From 375c40320883db46a373f5bb307006f662b9496f Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Mon, 5 Oct 2026 13:34:08 -0400 Subject: [PATCH 01/11] fix(ethreceipts): harden rollback, filter ownership and cancellation --- Makefile | 5 +- ethreceipts/ethreceipts.go | 378 +++++++-- ethreceipts/ethreceipts_test.go | 901 ++++++++------------ ethreceipts/filterer.go | 29 + ethreceipts/finalizer.go | 154 ++-- ethreceipts/hardening_test.go | 996 +++++++++++++++++++++++ ethreceipts/ownership_regression_test.go | 621 ++++++++++++++ ethreceipts/receipt.go | 24 +- ethreceipts/subscription.go | 596 ++++++++++---- 9 files changed, 2823 insertions(+), 881 deletions(-) create mode 100644 ethreceipts/hardening_test.go create mode 100644 ethreceipts/ownership_regression_test.go diff --git a/Makefile b/Makefile index 6621a01d..5697fe25 100644 --- a/Makefile +++ b/Makefile @@ -56,13 +56,14 @@ install: # Run baseline tests test: check-testchain-running go-test +# Packages share one testchain and deployment sender, so serialize package binaries. # Go test short-hand, and skip testing go-ethereum go-test: test-clean - GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -race -run=$(TEST) `go list ./... | grep -v go-ethereum` + GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -p=1 -race -run=$(TEST) `go list ./... | grep -v go-ethereum` # Go test short-hand, including go-ethereum go-test-all: test-clean - GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -run=$(TEST) ./... + GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -p=1 -run=$(TEST) ./... test-clean: GOGC=off go clean -testcache diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index cd7854f0..c7acc58e 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -98,10 +98,14 @@ type ReceiptsListener struct { registerFiltersCh chan registerFilters filterSem chan struct{} - ctx context.Context - ctxStop context.CancelFunc - running int32 - mu sync.RWMutex + ctx context.Context + ctxStop context.CancelFunc + running int32 + lifecycleMu sync.Mutex + receiptMu sync.Mutex + blockStates map[common.Hash]blockState + reorgRevision uint64 + mu sync.RWMutex } var ( @@ -160,6 +164,7 @@ func NewReceiptsListener(log *slog.Logger, provider ethrpc.Interface, monitor *e subscribers: make([]*subscriber, 0), registerFiltersCh: make(chan registerFilters, maxFiltersPerListener), filterSem: make(chan struct{}, opts.MaxConcurrentFilterWorkers), + blockStates: make(map[common.Hash]blockState), }, nil } @@ -186,32 +191,43 @@ func (l *ReceiptsListener) lazyInit(ctx context.Context) error { l.options.NumBlocksToFinality = ethrpc.DefaultNumBlocksToFinality } + for _, sub := range l.subscribers { + sub.finalizer.setFinality(l.options.NumBlocksToFinality) + } return nil } func (l *ReceiptsListener) Run(ctx context.Context) error { + l.lifecycleMu.Lock() if l.IsRunning() { + l.lifecycleMu.Unlock() return fmt.Errorf("ethreceipts: already running") } - - l.ctx, l.ctxStop = context.WithCancel(ctx) - + runCtx, cancel := context.WithCancel(ctx) + l.ctx, l.ctxStop = runCtx, cancel atomic.StoreInt32(&l.running, 1) - defer atomic.StoreInt32(&l.running, 0) - - if err := l.lazyInit(ctx); err != nil { - slog.Error("ethreceipts: lazyInit failed", slog.String("error", err.Error())) + l.lifecycleMu.Unlock() + defer func() { + cancel() + l.lifecycleMu.Lock() + l.ctxStop = nil + atomic.StoreInt32(&l.running, 0) + l.lifecycleMu.Unlock() + }() + if err := l.lazyInit(runCtx); err != nil { return err } - l.log.Info("ethreceipts: running") - - return l.listener() + return l.listener(runCtx) } - func (l *ReceiptsListener) Stop() { l.log.Info("ethreceipts: stop") - l.ctxStop() + l.lifecycleMu.Lock() + cancel := l.ctxStop + l.lifecycleMu.Unlock() + if cancel != nil { + cancel() + } } func (l *ReceiptsListener) IsRunning() bool { @@ -237,7 +253,7 @@ func (l *ReceiptsListener) Subscribe(filterQueries ...FilterQuery) Subscription finalizer: &finalizer{ numBlocksToFinality: big.NewInt(int64(l.options.NumBlocksToFinality)), queue: []finalTxn{}, - txns: map[common.Hash]struct{}{}, + txns: map[receiptKey]struct{}{}, }, } @@ -302,7 +318,11 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFinality(ctx context.Conte func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context, filter FilterQuery, optFilterFinalize ...bool) (*Receipt, WaitReceiptFinalityFunc, error) { // Fetch method searches for just a single filter match. If you'd like to keep the filter // open to listen to many similar receipts, use .Subscribe(filter) directly instead. - query := filter.LimitOne(true).SearchCache(true) + source, ok := filter.(Filterer) + if !ok { + return nil, nil, fmt.Errorf("ethreceipts: unable to cast Filterer from FilterQuery") + } + query := snapshotFilter(source).LimitOne(true).SearchCache(true) if len(optFilterFinalize) > 0 && optFilterFinalize[0] { query = query.Finalize(true) } @@ -327,6 +347,7 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // Use a WaitGroup to ensure the goroutine cleans up before the function returns var wg sync.WaitGroup + workerDone := make(chan struct{}) exhausted := make(chan struct{}) mined := make(chan Receipt, 2) finalized := make(chan Receipt, 1) @@ -335,7 +356,11 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context finalityFunc := func(ctx context.Context) (*Receipt, error) { // Wait for the goroutine to finish its cleanup before proceeding in finalityFunc, // ensuring Unsubscribe has been called if the goroutine exited. - wg.Wait() + select { + case <-workerDone: + case <-ctx.Done(): + return nil, ctx.Err() + } select { case <-ctx.Done(): return nil, ctx.Err() @@ -360,6 +385,7 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // don't think so, but we can double check. wg.Add(1) go func() { + defer close(workerDone) defer wg.Done() defer sub.Unsubscribe() defer close(mined) @@ -468,12 +494,17 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // fetchTransactionReceipt from the rpc provider, up to some amount of concurrency. When forceFetch is passed, // it indicates that we have high conviction that the receipt should be available, as the monitor has found // this transaction hash. -func (l *ReceiptsListener) fetchTransactionReceipt(ctx context.Context, txnHash common.Hash, forceFetch bool) (*types.Receipt, error) { +func (l *ReceiptsListener) fetchTransactionReceipt(ctx context.Context, txnHash common.Hash, forceFetch bool, expectedBlock ...blockRef) (*types.Receipt, error) { + l.receiptMu.Lock() + started := l.reorgRevision + l.receiptMu.Unlock() timeStart := time.Now() for { select { case l.fetchSem <- struct{}{}: goto start + case <-ctx.Done(): + return nil, ctx.Err() case <-time.After(1 * time.Minute): elapsed := time.Since(timeStart) l.log.Warn(fmt.Sprintf("fetchTransactionReceipt(%s) waiting for fetch semaphore for %s", txnHash.String(), elapsed)) @@ -496,7 +527,27 @@ start: txnHashHex := txnHash.String() + expected := blockRef{} + if len(expectedBlock) > 0 { + expected = expectedBlock[0] + } + l.receiptMu.Lock() + if expected.hash != (common.Hash{}) && !l.currentBlock(expected.hash, expected.generation) { + l.receiptMu.Unlock() + errCh <- ethereum.NotFound + return + } receipt, ok, _ := l.pastReceipts.Get(ctx, txnHashHex) + if ok && l.blockStates[receipt.BlockHash].removed { + ok = false + l.pastReceipts.Delete(ctx, txnHashHex) + } + if ok && !l.validFetchedBlock(receipt.BlockHash, expected, started) { + l.receiptMu.Unlock() + errCh <- ethereum.NotFound + return + } + l.receiptMu.Unlock() if ok { resultCh <- receipt return @@ -546,9 +597,20 @@ start: return superr.Wrap(fmt.Errorf("failed to fetch receipt %s", txnHash), err) } - l.pastReceipts.Set(ctx, txnHashHex, receipt) - l.notFoundTxnHashes.Delete(ctx, txnHashHex) - + if receipt == nil || receipt.TxHash != txnHash || receipt.BlockNumber == nil { + return fmt.Errorf("ethreceipts: invalid receipt for txn %s", txnHash) + } + l.receiptMu.Lock() + valid := l.validFetchedBlock(receipt.BlockHash, expected, started) + if valid { + l.pastReceipts.Set(ctx, txnHashHex, receipt) + l.notFoundTxnHashes.Delete(ctx, txnHashHex) + } + l.receiptMu.Unlock() + if !valid { + errCh <- ethereum.NotFound + return nil + } resultCh <- receipt return nil }) @@ -568,15 +630,25 @@ start: } } -func (l *ReceiptsListener) listener() error { +func (l *ReceiptsListener) listener(runCtx context.Context) error { + runCtx, cancel := context.WithCancel(runCtx) + defer cancel() monitor := l.monitor.Subscribe("ethreceipts") defer monitor.Unsubscribe() - latestBlockNum := l.LatestBlockNum().Uint64() + g, ctx := errgroup.WithContext(runCtx) + // Monitor closure must cancel RPC work even while both processing loops wait. + g.Go(func() error { + select { + case <-monitor.Done(): + cancel() + case <-ctx.Done(): + } + return nil + }) + latestBlockNum := l.latestBlockNum(ctx).Uint64() l.log.Debug(fmt.Sprintf("latestBlockNum %d", latestBlockNum)) - g, ctx := errgroup.WithContext(l.ctx) - // Listen on filter registration to search cached and on-chain receipts // At filter subscription registration time, we first search our local cache // of recent blocks retained by the monitor, and then we query on-chain for @@ -591,6 +663,7 @@ func (l *ReceiptsListener) listener() error { return nil case <-monitor.Done(): + cancel() l.log.Info("ethreceipts: receipt listener is stopped because monitor signaled its stopping") return nil @@ -623,7 +696,7 @@ func (l *ReceiptsListener) listener() error { // Search our local blocks cache from monitor retention list, and notify subscriber // of any matches found by publishing receipts. - matchedList, err := l.processBlocks(blocks, []*subscriber{reg.subscriber}, [][]Filterer{filters}) + matchedList, err := l.processCachedBlocksContext(ctx, blocks, []*subscriber{reg.subscriber}, [][]Filterer{filters}) if err != nil { l.log.Warn(fmt.Sprintf("ethreceipts: failed to process blocks during new filter registration: %v", err)) } @@ -649,6 +722,7 @@ func (l *ReceiptsListener) listener() error { return nil case <-monitor.Done(): + cancel() l.log.Info("ethreceipts: receipt listener is stopped because monitor signaled its stopping") return nil @@ -658,19 +732,15 @@ func (l *ReceiptsListener) listener() error { continue } - latestBlockNum = l.LatestBlockNum().Uint64() + latestBlockNum = l.latestBlockNum(ctx).Uint64() // pass blocks across filters of subscribers l.mu.Lock() - if len(l.subscribers) == 0 { - l.mu.Unlock() - continue - } subscribers := make([]*subscriber, len(l.subscribers)) copy(subscribers, l.subscribers) filters := make([][]Filterer, len(l.subscribers)) for i := 0; i < len(subscribers); i++ { - filters[i] = subscribers[i].Filters() + filters[i] = subscribers[i].filterers() } l.mu.Unlock() @@ -683,13 +753,7 @@ func (l *ReceiptsListener) listener() error { l.notFoundTxnHashes.Delete(ctx, txn.Hash().Hex()) } case ethmonitor.Removed: - // delete past receipts of removed blocks reorg = true - for _, txn := range block.Transactions() { - txnHashHex := txn.Hash().Hex() - l.pastReceipts.Delete(ctx, txnHashHex) - l.notFoundTxnHashes.Delete(ctx, txnHashHex) - } } } @@ -697,7 +761,7 @@ func (l *ReceiptsListener) listener() error { if reorg { for _, list := range filters { for _, filterer := range list { - if f, ok := filterer.(*filter); ok { + if f := builtinFilter(filterer); f != nil { f.setStartBlockNum(latestBlockNum) f.setLastMatchBlockNum(0) } @@ -706,7 +770,7 @@ func (l *ReceiptsListener) listener() error { } // Match blocks against subscribers[i] X filters[i][..] - matchedList, err := l.processBlocks(blocks, subscribers, filters) + matchedList, err := l.processBlocksContext(ctx, blocks, subscribers, filters) if err != nil { l.log.Warn(fmt.Sprintf("ethreceipts: failed to process blocks: %v", err)) } @@ -716,7 +780,7 @@ func (l *ReceiptsListener) listener() error { for y, matched := range list { filterer := filters[x][y] if matched || filterer.StartBlockNum() == 0 { - if f, ok := filterer.(*filter); ok { + if f := builtinFilter(filterer); f != nil { if f.StartBlockNum() == 0 { f.setStartBlockNum(latestBlockNum) } @@ -732,7 +796,7 @@ func (l *ReceiptsListener) listener() error { blockNum := max(filterer.StartBlockNum(), filterer.LastMatchBlockNum()) if maxWait != 0 && (latestBlockNum-blockNum) >= maxWait { - f, _ := filterer.(*filter) + f := builtinFilter(filterer) if f == nil { panic("ethreceipts: unexpected") } @@ -741,9 +805,9 @@ func (l *ReceiptsListener) listener() error { l.log.Debug(fmt.Sprintf("filter exhausted! last block matched:%d maxWait:%d filterID:%d", filterer.LastMatchBlockNum(), maxWait, filterer.FilterID())) subscriber := subscribers[x] - subscriber.RemoveFilter(filterer) + subscriber.exhaustFilter(filterer) - if f, ok := filterer.(*filter); ok { + if f := builtinFilter(filterer); f != nil { f.closeExhausted() } } @@ -769,20 +833,63 @@ func (l *ReceiptsListener) listener() error { // processBlocks attempts to match blocks against subscriber[i] X filterers[i].. list of filters. There is // a corresponding list of filters[i] for each subscriber[i]. func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { + l.lifecycleMu.Lock() + ctx := l.ctx + l.lifecycleMu.Unlock() + if ctx == nil { + ctx = context.Background() + } + return l.processCachedBlocksContext(ctx, blocks, subscribers, filterers) +} + +func (l *ReceiptsListener) processBlocksContext(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { + return l.processBlockEvents(ctx, blocks, subscribers, filterers, true) +} +func (l *ReceiptsListener) processCachedBlocksContext(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { + return l.processBlockEvents(ctx, blocks, subscribers, filterers, false) +} +func (l *ReceiptsListener) processBlockEvents(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer, canonicalEvents bool) ([][]bool, error) { // oks is the 'ok' match of the filterers [][]Filterer results oks := make([][]bool, len(filterers)) for i, f := range filterers { oks[i] = make([]bool, len(f)) } + // Invalidate every rollback before any retry, addition or finality advancement. + removed := make(map[common.Hash]blockRef) + for _, block := range blocks { + if block.Event == ethmonitor.Removed { + ref := l.invalidateBlock(ctx, block) + removed[block.Hash()] = ref + for _, sub := range subscribers { + sub.rollbackBlock(ref) + } + } + } if len(subscribers) == 0 || len(filterers) == 0 { + if canonicalEvents { + for _, block := range blocks { + if block.Event == ethmonitor.Added { + l.acceptBlock(block.Hash()) + } + } + } return oks, nil } - // check each block against each subscriber X filter for _, block := range blocks { + if ctx.Err() != nil { + return oks, ctx.Err() + } // report if the txn was removed reorged := block.Event == ethmonitor.Removed + if canonicalEvents && !reorged { + l.acceptBlock(block.Hash()) + } + generation := l.blockGeneration(block.Hash()) + if reorged { + generation = removed[block.Hash()].generation + } // TODOXXX: feels wasteful to build all receipts for all subscribers every time, but its okay for now receipts := make([]Receipt, len(block.Transactions())) @@ -800,6 +907,9 @@ func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers [ logs: txnLog, chainID: l.chainID, transaction: txn, + blockNum: block.Number(), + blockHash: block.Hash(), + generation: generation, } } @@ -807,8 +917,12 @@ func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers [ // and if there is a match notify the subscriber with the receipts. var wg sync.WaitGroup for i, sub := range subscribers { - l.filterSem <- struct{}{} - + select { + case l.filterSem <- struct{}{}: + case <-ctx.Done(): + wg.Wait() + return oks, ctx.Err() + } wg.Add(1) go func(i int, sub *subscriber) { defer func() { @@ -817,21 +931,24 @@ func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers [ }() // retry pending receipts first - retryCtx, cancel := context.WithTimeout(l.ctx, 5*time.Second) - sub.retryPendingReceipts(retryCtx) // TODOXXXPETER: what is this pending receipts thing..? hmpf.. - cancel() + if !reorged { + retryCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + sub.retryPendingReceipts(retryCtx) + cancel() + } // filter matcher and notify subscriber of receipts if matched - matched, err := sub.matchFiltersAndPublish(l.ctx, filterers[i], receipts) + matched, err := sub.matchFiltersAndPublish(ctx, filterers[i], receipts) if err != nil { l.log.Warn(fmt.Sprintf("error while processing filters: %s", err)) } oks[i] = matched // check subscriber to finalize any receipts - err = sub.finalizeReceipts(block.Number()) - if err != nil { - l.log.Error(fmt.Sprintf("finalizeReceipts failed: %v", err)) + if !reorged && ctx.Err() == nil { + if err = sub.finalizeReceipts(block.Number()); err != nil { + l.log.Error(fmt.Sprintf("finalizeReceipts failed: %v", err)) + } } }(i, sub) } @@ -885,6 +1002,9 @@ func (l *ReceiptsListener) queryFilterOnChain(ctx context.Context, subscriber *s return gctx.Err() } + l.receiptMu.Lock() + started := l.reorgRevision + l.receiptMu.Unlock() var r *types.Receipt var err error @@ -909,13 +1029,23 @@ func (l *ReceiptsListener) queryFilterOnChain(ctx context.Context, subscriber *s return nil } + l.receiptMu.Lock() + valid := l.validFetchedBlock(r.BlockHash, blockRef{}, started) + generation := l.blockStates[r.BlockHash].generation + l.receiptMu.Unlock() + if !valid { + return nil + } // Found the receipt, update last match block num and continue - if f, ok := item.filterer.(*filter); ok { - f.setLastMatchBlockNum(r.BlockNumber.Uint64()) + if f := builtinFilter(item.filterer); f != nil { + if r.BlockNumber != nil { + f.setLastMatchBlockNum(r.BlockNumber.Uint64()) + } } receipt := Receipt{ - receipt: r, + receipt: r, + generation: generation, // NOTE: we do not include the transaction at this point, as we don't have it. // transaction: txn, Final: l.isBlockFinal(r.BlockNumber), @@ -954,7 +1084,7 @@ func (l *ReceiptsListener) getMaxWaitBlocks(maxWait *int) uint64 { } func (l *ReceiptsListener) isBlockFinal(blockNum *big.Int) bool { - latestBlockNum := l.LatestBlockNum() + latestBlockNum := l.monitor.LatestBlockNum() if latestBlockNum == nil || blockNum == nil { return false } @@ -963,10 +1093,12 @@ func (l *ReceiptsListener) isBlockFinal(blockNum *big.Int) bool { l.mu.RLock() defer l.mu.RUnlock() - return diff.Cmp(big.NewInt(int64(l.options.NumBlocksToFinality))) >= 0 + return l.options.NumBlocksToFinality > 0 && diff.Cmp(big.NewInt(int64(l.options.NumBlocksToFinality))) >= 0 } -func (l *ReceiptsListener) LatestBlockNum() *big.Int { +func (l *ReceiptsListener) LatestBlockNum() *big.Int { return l.latestBlockNum(context.Background()) } + +func (l *ReceiptsListener) latestBlockNum(ctx context.Context) *big.Int { // return immediately if the monitor has a latest block number latestBlockNum := l.monitor.LatestBlockNum() if latestBlockNum != nil && latestBlockNum.Cmp(big.NewInt(0)) > 0 { @@ -977,7 +1109,11 @@ func (l *ReceiptsListener) LatestBlockNum() *big.Int { maxWaitTime := 30 * time.Second period := 250 * time.Millisecond for { - time.Sleep(period) + select { + case <-time.After(period): + case <-ctx.Done(): + return big.NewInt(0) + } latestBlockNum := l.monitor.LatestBlockNum() if latestBlockNum != nil && latestBlockNum.Cmp(big.NewInt(0)) > 0 { @@ -993,26 +1129,35 @@ func (l *ReceiptsListener) LatestBlockNum() *big.Int { } func getChainID(ctx context.Context, provider ethrpc.Interface) (*big.Int, error) { - var chainID *big.Int - - // provide plenty of time for breaker to succeed - err := breaker.Do(ctx, func() error { - ctx, cancel := context.WithTimeout(ctx, 4*time.Second) - defer cancel() - - id, err := provider.ChainID(ctx) - if err != nil { - return err + delay := time.Second + for attempt := 0; ; attempt++ { + if ctx.Err() != nil { + return nil, ctx.Err() } - chainID = id - return nil - }, nil, 1*time.Second, 2, 10) - - if err != nil { - return nil, err + requestCtx, cancel := context.WithTimeout(ctx, 4*time.Second) + id, err := provider.ChainID(requestCtx) + cancel() + if err == nil { + return id, nil + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + if errors.Is(err, breaker.ErrFatal) { + return nil, err + } + if attempt >= 10 { + return nil, superr.New(breaker.ErrHitMaxRetries, err) + } + timer := time.NewTimer(delay) + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + } + delay *= 2 } - - return chainID, nil } func collectOk[T any](in []T, oks []bool, okCond bool) []T { @@ -1065,3 +1210,74 @@ func groupLogsByTransaction(logs []types.Log) map[string][]*types.Log { // } // return max // } + +// A removed hash can later be canonical again. Generations invalidate work +// started before rollback even when its transaction and block hash are unchanged. +type blockRef struct { + hash common.Hash + generation uint64 +} +type blockState struct { + generation uint64 + removed bool + invalidatedAt uint64 +} + +func (l *ReceiptsListener) currentBlock(hash common.Hash, generation uint64) bool { + state := l.blockStates[hash] + return !state.removed && state.generation == generation +} +func (l *ReceiptsListener) isCurrentBlock(hash common.Hash, generation uint64) bool { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + return l.currentBlock(hash, generation) +} +func (l *ReceiptsListener) blockGeneration(hash common.Hash) uint64 { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + return l.blockStates[hash].generation +} + +// Caller holds receiptMu. Hash queries do not know their block before fetching, +// so reject results from blocks invalidated since the RPC started as well. +func (l *ReceiptsListener) validFetchedBlock(hash common.Hash, expected blockRef, started uint64) bool { + state := l.blockStates[hash] + if state.removed || state.invalidatedAt > started { + return false + } + return expected.hash == (common.Hash{}) || (hash == expected.hash && state.generation == expected.generation) +} +func (l *ReceiptsListener) acceptBlock(hash common.Hash) { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + state := l.blockStates[hash] + if state.removed { + canonical := l.monitor.GetBlock(hash) + if canonical != nil && canonical.Event == ethmonitor.Added { + state.removed = false + l.blockStates[hash] = state + } + } +} +func (l *ReceiptsListener) invalidateBlock(ctx context.Context, block *ethmonitor.Block) blockRef { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + state := l.blockStates[block.Hash()] + if !state.removed { + l.reorgRevision++ + state.generation++ + state.invalidatedAt = l.reorgRevision + state.removed = true + l.blockStates[block.Hash()] = state + } + removed := blockRef{block.Hash(), state.generation - 1} + for _, txn := range block.Transactions() { + hash := txn.Hash().Hex() + cached, found, _ := l.pastReceipts.Get(ctx, hash) + if found && cached.BlockHash == block.Hash() { + l.pastReceipts.Delete(ctx, hash) + } + l.notFoundTxnHashes.Delete(ctx, hash) + } + return removed +} diff --git a/ethreceipts/ethreceipts_test.go b/ethreceipts/ethreceipts_test.go index 9c508a4e..d2bfc3ae 100644 --- a/ethreceipts/ethreceipts_test.go +++ b/ethreceipts/ethreceipts_test.go @@ -20,10 +20,12 @@ import ( "github.com/0xsequence/ethkit/ethrpc" "github.com/0xsequence/ethkit/ethtest" "github.com/0xsequence/ethkit/ethtxn" + "github.com/0xsequence/ethkit/ethwallet" "github.com/0xsequence/ethkit/go-ethereum/common" "github.com/0xsequence/ethkit/go-ethereum/core/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) var ( @@ -72,6 +74,109 @@ func newProvider(t *testing.T) *ethrpc.Provider { return provider } +// Service errors cancel receipt waits and are checked after the worker exits. +func runReceiptTestService(t *testing.T, ctx context.Context, cancel context.CancelFunc, name string, run func(context.Context) error) { + t.Helper() + done := make(chan error, 1) + go func() { + err := run(ctx) + if ctx.Err() != nil && (err == nil || errors.Is(err, ctx.Err())) { + done <- nil + return + } + if err == nil { + err = fmt.Errorf("%s stopped unexpectedly", name) + } else { + err = fmt.Errorf("%s: %w", name, err) + } + done <- err + cancel() + }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Errorf("%s did not stop after cancellation", name) + } + }) +} + +func restoreReceiptTestProvider(t *testing.T, ctx context.Context, cancel context.CancelFunc, provider *ethrpc.Provider) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + select { + case <-ctx.Done(): + case <-time.After(20 * time.Second): + provider.SetHTTPClient(newFlakyHTTPClient(0.0)) + } + }() + t.Cleanup(func() { + cancel() + <-done + }) +} + +func waitForFlakyProviderReceipts(t *testing.T, ctx context.Context, provider *ethrpc.Provider, listener *ethreceipts.ReceiptsListener, sender *ethwallet.Wallet, recipient common.Address) { + t.Helper() + nonce, err := sender.GetNonce(ctx) + require.NoError(t, err) + + txns := make([]*types.Transaction, 50) + for i := range txns { + txns[i], err = sender.NewTransaction(ctx, ðtxn.TransactionRequest{ + To: &recipient, + ETHValue: ethtest.ETHValue(0.01), + GasLimit: 120_000, + Nonce: big.NewInt(int64(nonce + uint64(i))), + }) + require.NoError(t, err) + } + + g, workerCtx := errgroup.WithContext(ctx) + for _, txn := range txns { + ready := make(chan struct{}) + g.Go(func() error { + sub := listener.Subscribe(ethreceipts.FilterTxnHash(txn.Hash())) + defer sub.Unsubscribe() + close(ready) + + select { + case <-workerCtx.Done(): + return workerCtx.Err() + case <-sub.Done(): + return fmt.Errorf("subscription closed before receipt for %s", txn.Hash()) + case receipt, ok := <-sub.TransactionReceipt(): + if !ok { + return fmt.Errorf("receipt channel closed for %s", txn.Hash()) + } + if receipt.TransactionHash() != txn.Hash() || receipt.Status() != types.ReceiptStatusSuccessful || receipt.Reorged { + return fmt.Errorf("unexpected receipt for %s: hash=%s status=%d reorged=%t", txn.Hash(), receipt.TransactionHash(), receipt.Status(), receipt.Reorged) + } + return nil + case <-time.After(300 * time.Second): + return fmt.Errorf("timeout waiting for receipt for %s", txn.Hash()) + } + }) + g.Go(func() error { + select { + case <-workerCtx.Done(): + return workerCtx.Err() + case <-ready: + } + _, _, err := ethtxn.SendTransaction(workerCtx, provider, txn) + if err != nil { + return fmt.Errorf("send transaction %s: %w", txn.Hash(), err) + } + return nil + }) + } + require.NoError(t, g.Wait()) +} + func newFlakyHTTPClient(failureRate float32) *http.Client { return &http.Client{ Timeout: 10 * time.Second, @@ -118,12 +223,7 @@ func TestFetchTransactionReceiptBasic(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) require.Zero(t, monitor.NumSubscribers()) @@ -134,12 +234,7 @@ func TestFetchTransactionReceiptBasic(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) assert.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // // Setup test wallet @@ -174,54 +269,29 @@ func TestFetchTransactionReceiptBasic(t *testing.T) { txnHashes = append(txnHashes, txn.Hash()) } - // dispatch txns in the background - go func() { + workers, workerCtx := errgroup.WithContext(ctx) + workers.Go(func() error { for _, txn := range txns { - _, _, err = wallet.SendTransaction(ctx, txn) - require.NoError(t, err) - // time.Sleep(500 * time.Millisecond) + if _, _, err := wallet.SendTransaction(workerCtx, txn); err != nil { + return fmt.Errorf("send transaction %s: %w", txn.Hash(), err) + } } - }() - - // ensure all txns made it - // delay processing if we want to make sure SearchCache works - // time.Sleep(2 * time.Second) - // for _, txnHash := range txnHashes { - // receipt, err := provider.TransactionReceipt(context.Background(), txnHash) - // require.NoError(t, err) - // require.True(t, receipt.Status == 1) - // } - - // Let's listen for all the txns - var wg sync.WaitGroup - for i, txnHash := range txnHashes { - wg.Add(1) - go func(i int, txnHash common.Hash) { - defer wg.Done() + return nil + }) - receipt, err := receiptsListener.FetchTransactionReceipt(ctx, txnHash, 7) - require.NoError(t, err) - require.NotNil(t, receipt) - require.True(t, receipt.Status() == types.ReceiptStatusSuccessful) - require.False(t, receipt.Final) - t.Logf("=> MINED %d :: %s", i, receipt.TransactionHash().String()) - - // receipt, waitFinality, err := receiptsListener.FetchTransactionReceiptWithFinality(ctx, txnHash, 7) - // require.NoError(t, err) - // require.NotNil(t, receipt) - // require.True(t, receipt.Status() == types.ReceiptStatusSuccessful) - // require.False(t, receipt.Final) - // t.Logf("=> MINED %d :: %s", i, receipt.TransactionHash().String()) - // - // finalReceipt, err := waitFinality(context.Background()) - // require.NoError(t, err) - // require.NotNil(t, finalReceipt) - // require.True(t, finalReceipt.Status() == types.ReceiptStatusSuccessful) - // require.True(t, finalReceipt.Final) - // t.Logf("=> FINAL %d :: %s", i, receipt.TransactionHash().String()) - }(i, txnHash) + for _, txnHash := range txnHashes { + workers.Go(func() error { + receipt, err := receiptsListener.FetchTransactionReceipt(workerCtx, txnHash, 7) + if err != nil { + return fmt.Errorf("fetch receipt %s: %w", txnHash, err) + } + if receipt == nil || receipt.TransactionHash() != txnHash || receipt.Status() != types.ReceiptStatusSuccessful || receipt.Final { + return fmt.Errorf("unexpected mined receipt for %s: %v", txnHash, receipt) + } + return nil + }) } - wg.Wait() + require.NoError(t, workers.Wait()) time.Sleep(2 * time.Second) @@ -292,12 +362,7 @@ func TestFetchTransactionReceiptBlast(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 @@ -306,12 +371,7 @@ func TestFetchTransactionReceiptBlast(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) assert.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // // Setup wallets @@ -345,176 +405,193 @@ func TestFetchTransactionReceiptBlast(t *testing.T) { txns[5].Hash(), txns[2].Hash(), txns[8].Hash(), txns[3].Hash(), } - var count uint64 - - var wg sync.WaitGroup - for i, txnHash := range txnHashes { - wg.Add(1) - go func(i int, txnHash common.Hash) { - defer wg.Done() - - receipt, receiptFinality, err := receiptsListener.FetchTransactionReceiptWithFinality(ctx, txnHash) - assert.NoError(t, err) - assert.NotNil(t, receipt) - assert.True(t, receipt.Status() == types.ReceiptStatusSuccessful) - - finalReceipt, err := receiptFinality(context.Background()) - require.NoError(t, err) - require.True(t, finalReceipt.Status() == types.ReceiptStatusSuccessful) - - t.Logf("=> %d :: %s", i, receipt.TransactionHash().String()) - - atomic.AddUint64(&count, 1) - }(i, txnHash) + workers, workerCtx := errgroup.WithContext(ctx) + for _, txnHash := range txnHashes { + workers.Go(func() error { + receipt, waitFinality, err := receiptsListener.FetchTransactionReceiptWithFinality(workerCtx, txnHash) + if err != nil { + return fmt.Errorf("fetch receipt %s: %w", txnHash, err) + } + if receipt == nil || receipt.TransactionHash() != txnHash || receipt.Status() != types.ReceiptStatusSuccessful { + return fmt.Errorf("unexpected receipt for %s: %v", txnHash, receipt) + } + finalReceipt, err := waitFinality(workerCtx) + if err != nil { + return fmt.Errorf("final receipt %s: %w", txnHash, err) + } + if finalReceipt == nil || finalReceipt.TransactionHash() != txnHash || finalReceipt.Status() != types.ReceiptStatusSuccessful || !finalReceipt.Final { + return fmt.Errorf("unexpected final receipt for %s: %v", txnHash, finalReceipt) + } + return nil + }) } - wg.Wait() - - require.Equal(t, int(count), len(txnHashes)) + require.NoError(t, workers.Wait()) } func TestReceiptsListenerFilters(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() - // - // Setup ReceiptsListener - // provider := testchain.Provider - monitorOptions := ethmonitor.DefaultOptions - // monitorOptions.Logger = log monitorOptions.WithLogs = true monitorOptions.BlockRetentionLimit = 1000 monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) - assert.NoError(t, err) - - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + require.NoError(t, err) + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 listenerOptions.FilterMaxWaitNumBlocks = 4 - receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) - assert.NoError(t, err) - - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() - - // - // Setup wallets - // - - // create and fund a few wallets to send from - fromWallets, _ := testchain.DummyWallets(3, 100) - testchain.FundAddresses(ethtest.WalletAddresses(fromWallets), 10) - - // create a few wallets to send to - toWallets, _ := testchain.DummyWallets(3, 200) - - // prepare and sign bunch of txns - values := []*big.Int{} - for range fromWallets { - values = append(values, ethtest.ETHValue(0.1)) - } + require.NoError(t, err) + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) + fromWallets, err := testchain.DummyWallets(3, 100) + require.NoError(t, err) + require.NoError(t, testchain.FundAddresses(ethtest.WalletAddresses(fromWallets), 10)) + toWallets, err := testchain.DummyWallets(3, 200) + require.NoError(t, err) + values := []*big.Int{ethtest.ETHValue(0.1), ethtest.ETHValue(0.1), ethtest.ETHValue(0.1)} _, txns, err := ethtest.PrepareBlastSendTransactions(ctx, fromWallets, ethtest.WalletAddresses(toWallets), values) - assert.NoError(t, err) + require.NoError(t, err) - // send the txns -- these will be async, so we can just blast synchronously - // and not have to do it in a goroutine + fromFilter := ethreceipts.FilterFrom(fromWallets[1].Address()).LimitOne(true).ID(1111).MaxWait(0) + toFilter := ethreceipts.FilterTo(toWallets[1].Address()).ID(1112).MaxWait(0) + removedFilter := ethreceipts.FilterTxnHash(txns[2].Hash()).ID(2222) + addedFilter := ethreceipts.FilterTxnHash(txns[4].Hash()).ID(4444) + sub := receiptsListener.Subscribe(fromFilter, toFilter, removedFilter) + defer sub.Unsubscribe() + + sub2 := receiptsListener.Subscribe() + secondFilter := ethreceipts.FilterTxnHash(txns[3].Hash()) + sub2.AddFilter(secondFilter) + defer sub2.Unsubscribe() + + thirdFilter := ethreceipts.FilterTxnHash(txns[2].Hash()).ID(3333) + expiringFilter := ethreceipts.FilterFrom(ethkit.Address{4, 2, 4, 2}).ID(8888).MaxWait(4) + sub3 := receiptsListener.Subscribe(thirdFilter, expiringFilter) + defer sub3.Unsubscribe() + + // All live filters are installed before any transaction can be mined. + require.Eventually(t, func() bool { + return receiptsListener.IsRunning() && monitor.NumSubscribers() == 1 + }, 5*time.Second, 10*time.Millisecond) for _, txn := range txns { _, _, err := ethtxn.SendTransaction(ctx, provider, txn) - assert.NoError(t, err) + require.NoError(t, err) } - // - // Subscribe to a filter on the receipt listener - // - fmt.Println("listening for txns..") - - sub := receiptsListener.Subscribe( - ethreceipts.FilterFrom(fromWallets[1].Address()).LimitOne(true), - ethreceipts.FilterTo(toWallets[1].Address()), - ethreceipts.FilterTxnHash(txns[2].Hash()).ID(2222), //.Finalize(true) is set by default for FilterTxnHash - ) - - sub2 := receiptsListener.Subscribe() - sub2.AddFilter(ethreceipts.FilterTxnHash(txns[3].Hash())) - - sub3 := receiptsListener.Subscribe( - ethreceipts.FilterTxnHash(txns[2].Hash()).ID(3333), - - // will end up not being found and timeout after MaxWait - ethreceipts.FilterFrom(ethkit.Address{4, 2, 4, 2}).MaxWait(4), - ) - - go func() { - time.Sleep(5 * time.Second) - fmt.Println("==> delaying to find", txns[4].Hash().String()) - sub.AddFilter(ethreceipts.FilterTxnHash(txns[4].Hash()).ID(4444)) - }() - - go func() { - for r := range sub2.TransactionReceipt() { - fmt.Println("sub2, got receipt", r.TransactionHash(), "final?", r.Final) + type delivery struct { + subscription string + hash common.Hash + filterID uint64 + final bool + } + expected := make(map[delivery]ethreceipts.FilterQuery) + expect := func(subscription string, txnIndex int, filter ethreceipts.FilterQuery, states ...bool) { + for _, final := range states { + key := delivery{subscription, txns[txnIndex].Hash(), filter.(ethreceipts.Filterer).FilterID(), final} + expected[key] = filter } - }() - - go func() { - for r := range sub3.TransactionReceipt() { - fmt.Println("sub3, got receipt", r.TransactionHash(), "final?", r.Final, "id?", r.FilterID()) //, "maxWait hit?", r.Filter.IsExpired()) + } + // LimitOne selects the first matching nonce; the recipient filter selects all three senders. + expect("sub", 3, fromFilter, false) + for _, i := range []int{1, 4, 7} { + expect("sub", i, toFilter, false) + } + expect("sub", 2, removedFilter, false) + expect("sub", 4, addedFilter, false, true) + expect("sub2", 3, secondFilter, false, true) + expect("sub3", 2, thirdFilter, false, true) + + seen := make(map[delivery]bool) + added := false + removed := false + record := func(subscription string, receipt ethreceipts.Receipt) { + key := delivery{subscription, receipt.TransactionHash(), receipt.FilterID(), receipt.Final} + filter, ok := expected[key] + require.True(t, ok, "unexpected delivery: %+v", key) + require.False(t, seen[key], "duplicate delivery: %+v", key) + require.Same(t, filter, receipt.Filter, "wrong filter owner: %+v", key) + require.Equal(t, uint64(types.ReceiptStatusSuccessful), receipt.Status()) + require.False(t, receipt.Reorged) + require.NotZero(t, receipt.BlockHash()) + seen[key] = true + + if subscription == "sub" && receipt.FilterID() == 2222 { + // Remove only this owner after its mined event, before it can finalize. + sub.RemoveFilter(receipt.Filter) + require.NotContains(t, sub.Filters(), receipt.Filter) + removed = true } - }() + if subscription == "sub" && receipt.FilterID() == 1112 && receipt.TransactionHash() == txns[4].Hash() { + // This transaction is now in the monitor cache; registration must find it there. + sub.AddFilter(addedFilter) + require.Contains(t, sub.Filters(), addedFilter) + added = true + } + } -loop: - for { + exhausted := expiringFilter.(ethreceipts.Filterer).Exhausted() + expired := false + for len(seen) != len(expected) || !expired { select { - case <-ctx.Done(): - fmt.Println("ctx done") - break loop - + missing := make([]delivery, 0) + for key := range expected { + if !seen[key] { + missing = append(missing, key) + } + } + t.Fatalf("missing receipt deliveries: %v; expired=%t; context=%v", missing, expired, ctx.Err()) case <-sub.Done(): - fmt.Println("sub done") - break loop - + t.Fatal("subscription closed before expected deliveries") + case <-sub2.Done(): + t.Fatal("second subscription closed before expected deliveries") + case <-sub3.Done(): + t.Fatal("third subscription closed before expected deliveries") case receipt, ok := <-sub.TransactionReceipt(): - if !ok { - continue - } - - fmt.Println("=> sub, got receipt", receipt.TransactionHash(), "final?", receipt.Final, "id?", receipt.FilterID(), "status?", receipt.Status()) - - // txn := receipt.Transaction - // txnMsg := receipt.Message - - fmt.Println("=> filter matched!", receipt.From(), receipt.TransactionHash()) - fmt.Println("=> receipt status?", receipt.Status()) - - fmt.Println("==> len filters", len(sub.Filters())) - if receipt.TransactionHash() == txns[2].Hash() { - sub.RemoveFilter(receipt.Filter) - } - fmt.Println("==> len filters", len(sub.Filters())) - - fmt.Println("") - - // expecting to be finished with listening for events after a few seconds - case <-time.After(15 * time.Second): - sub.Unsubscribe() - + require.True(t, ok, "receipt channel closed before expected deliveries") + record("sub", receipt) + case receipt, ok := <-sub2.TransactionReceipt(): + require.True(t, ok, "second receipt channel closed before expected deliveries") + record("sub2", receipt) + case receipt, ok := <-sub3.TransactionReceipt(): + require.True(t, ok, "third receipt channel closed before expected deliveries") + record("sub3", receipt) + case <-exhausted: + expired = true + exhausted = nil } } + + require.True(t, added, "cached transaction filter was not added") + require.True(t, removed, "mined transaction filter was not removed") + require.Eventually(t, func() bool { + return len(sub.Filters()) == 1 && len(sub2.Filters()) == 0 && len(sub3.Filters()) == 0 + }, 5*time.Second, 10*time.Millisecond) + require.Same(t, toFilter, sub.Filters()[0]) + + // The unbounded subscription queue forwards asynchronously; observe it long enough + // to reject queued duplicates or finalization of the removed owner. + quiet := time.NewTimer(2 * time.Second) + defer quiet.Stop() + select { + case <-ctx.Done(): + t.Fatalf("context ended while checking for extra deliveries: %v", ctx.Err()) + case receipt, ok := <-sub.TransactionReceipt(): + require.True(t, ok, "receipt channel closed unexpectedly") + record("sub", receipt) + case receipt, ok := <-sub2.TransactionReceipt(): + require.True(t, ok, "second receipt channel closed unexpectedly") + record("sub2", receipt) + case receipt, ok := <-sub3.TransactionReceipt(): + require.True(t, ok, "third receipt channel closed unexpectedly") + record("sub3", receipt) + case <-quiet.C: + } } func TestReceiptsListenerERC20(t *testing.T) { @@ -544,12 +621,7 @@ func TestReceiptsListenerERC20(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 @@ -558,12 +630,7 @@ func TestReceiptsListenerERC20(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) assert.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // // Subscribe to a filter on the receipt listener @@ -602,26 +669,20 @@ func TestReceiptsListenerERC20(t *testing.T) { num := int64(2000) erc20Receipts := make([]*types.Receipt, 0) - var erc20ReceiptsMu sync.Mutex receipt := erc20Mock.Mint(t, wallet, num) erc20Receipts = append(erc20Receipts, receipt) erc20Mock.GetBalance(t, wallet.Address(), num) - go func() { - total := int64(0) - for i := 0; i < 5; i++ { - n := int64(40 + i) - total += n - - erc20ReceiptsMu.Lock() - receipt := erc20Mock.Transfer(t, wallet, wallet2.Address(), n) - erc20Receipts = append(erc20Receipts, receipt) - erc20ReceiptsMu.Unlock() + total := int64(0) + for i := 0; i < 5; i++ { + n := int64(40 + i) + total += n - erc20Mock.GetBalance(t, wallet2.Address(), total) - } - }() + receipt := erc20Mock.Transfer(t, wallet, wallet2.Address(), n) + erc20Receipts = append(erc20Receipts, receipt) + erc20Mock.GetBalance(t, wallet2.Address(), total) + } // // Listener loop @@ -667,20 +728,29 @@ loop: } } - // NOTE: expecting receipts twice. Once on mine, once on finalize. - for _, mr := range matchedReceipts { - found := false - for _, r := range erc20Receipts { - if mr.TransactionHash() == r.TxHash { - found = true - } - } - assert.True(t, found, "looking for matched receipt %s", mr.TransactionHash().String()) + // Each submitted transaction must have one mined and one final event, even + // when MaxWait expires before the queued finality threshold is reached. + expectedHashes := make(map[common.Hash]bool, len(erc20Receipts)) + for _, receipt := range erc20Receipts { + expectedHashes[receipt.TxHash] = true } - - t.Logf("matchedCount: %d", matchedCount) - t.Logf("erc20Receipts: %d", len(erc20Receipts)) - require.Equal(t, matchedCount, len(erc20Receipts)*2) + type receiptState struct { + hash common.Hash + final bool + } + counts := make(map[receiptState]int) + for _, receipt := range matchedReceipts { + require.True(t, expectedHashes[receipt.TransactionHash()], "unexpected transaction %s", receipt.TransactionHash()) + require.Equal(t, uint64(9999), receipt.FilterID()) + require.Equal(t, uint64(types.ReceiptStatusSuccessful), receipt.Status()) + require.False(t, receipt.Reorged) + counts[receiptState{receipt.TransactionHash(), receipt.Final}]++ + } + for hash := range expectedHashes { + require.Equal(t, 1, counts[receiptState{hash, false}], "mined delivery for %s", hash) + require.Equal(t, 1, counts[receiptState{hash, true}], "final delivery for %s", hash) + } + require.Equal(t, len(erc20Receipts)*2, matchedCount) } func TestFiltersAddDeadlock(t *testing.T) { @@ -696,12 +766,7 @@ func TestFiltersAddDeadlock(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 @@ -780,10 +845,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(goodProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -792,81 +854,18 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, goodProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) t.Run("Wait for txn receipts with a flaky monitor and a healthy provider", func(t *testing.T) { @@ -885,10 +884,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(flakyProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -897,10 +893,7 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, goodProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) @@ -909,81 +902,15 @@ func TestFlakyProvider(t *testing.T) { t.Logf("Setting provider to flaky state") flakyProvider.SetHTTPClient(newFlakyHTTPClient(1.0)) - go func() { - // After 20 seconds, restore the provider to more reliable state - time.Sleep(20 * time.Second) - t.Logf("Restoring provider to healthy state") - flakyProvider.SetHTTPClient(newFlakyHTTPClient(0.0)) - }() + restoreReceiptTestProvider(t, ctx, cancel, flakyProvider) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - // Add a bunch of subscribers that will each send a txn and wait for it - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) t.Run("Wait for txn receipts with a healthy monitor and a flaky provider", func(t *testing.T) { @@ -1002,10 +929,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(goodProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -1014,10 +938,7 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, flakyProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) @@ -1026,81 +947,15 @@ func TestFlakyProvider(t *testing.T) { t.Logf("Setting provider to flaky state") flakyProvider.SetHTTPClient(newFlakyHTTPClient(1.0)) - go func() { - // After 20 seconds, restore the provider to more reliable state - time.Sleep(20 * time.Second) - t.Logf("Restoring provider to healthy state") - flakyProvider.SetHTTPClient(newFlakyHTTPClient(0.0)) - }() + restoreReceiptTestProvider(t, ctx, cancel, flakyProvider) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - // Add a bunch of subscribers that will each send a txn and wait for it - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) t.Run("Wait for txn receipts with a flaky monitor and a flaky provider", func(t *testing.T) { @@ -1119,10 +974,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(flakyProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -1131,10 +983,7 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, flakyProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) @@ -1143,80 +992,14 @@ func TestFlakyProvider(t *testing.T) { t.Logf("Setting provider to flaky state") flakyProvider.SetHTTPClient(newFlakyHTTPClient(1.0)) - go func() { - // After 20 seconds, restore the provider to more reliable state - time.Sleep(20 * time.Second) - t.Logf("Restoring provider to healthy state") - flakyProvider.SetHTTPClient(newFlakyHTTPClient(0.0)) - }() + restoreReceiptTestProvider(t, ctx, cancel, flakyProvider) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - // Add a bunch of subscribers that will each send a txn and wait for it - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) } diff --git a/ethreceipts/filterer.go b/ethreceipts/filterer.go index fd48c20d..2a40a804 100644 --- a/ethreceipts/filterer.go +++ b/ethreceipts/filterer.go @@ -169,6 +169,7 @@ type FilterCond struct { type filter struct { options FilterOptions cond FilterCond + match func(context.Context, Receipt) (bool, error) // startBlockNum is the first block number observed once filter is active startBlockNum uint64 @@ -240,6 +241,9 @@ func (f *filter) Cond() FilterCond { } func (f *filter) Match(ctx context.Context, receipt Receipt) (bool, error) { + if f.match != nil { + return f.match(ctx, receipt) + } c := f.cond if c.TxnHash != nil { @@ -302,3 +306,28 @@ func (f *filter) closeExhausted() { func (f *filter) Exhausted() <-chan struct{} { return f.exhausted } + +// Fetch helpers own their options, counters, and exhaustion signal. A custom +// Filterer keeps its matching behavior without receiving helper option mutations. +func snapshotFilter(source Filterer) *filter { + options := source.Options() + if options.MaxWait != nil { + value := *options.MaxWait + options.MaxWait = &value + } + snapshot := &filter{options: options, cond: source.Cond(), exhausted: make(chan struct{})} + if builtIn, ok := source.(*filter); ok { + snapshot.match = builtIn.match + } else { + snapshot.match = source.Match + } + return snapshot +} + +func builtinFilter(filterer Filterer) *filter { + if owner, ok := filterer.(*filterOwner); ok { + filterer = owner.Filterer + } + builtIn, _ := filterer.(*filter) + return builtIn +} diff --git a/ethreceipts/finalizer.go b/ethreceipts/finalizer.go index c9ffa2db..40ee9f12 100644 --- a/ethreceipts/finalizer.go +++ b/ethreceipts/finalizer.go @@ -2,96 +2,132 @@ package ethreceipts import ( "math/big" - "sort" "sync" - "github.com/0xsequence/ethkit" + "github.com/0xsequence/ethkit/go-ethereum/common" ) +// Public filter IDs are labels; distinct filters can share one label. +type receiptKey struct { + txnHash common.Hash + blockHash common.Hash + generation uint64 + owner *filterOwner +} + +func receiptOwner(receipt Receipt, owner *filterOwner) receiptKey { + return receiptKey{receipt.TransactionHash(), receipt.BlockHash(), receipt.generation, owner} +} + type finalizer struct { queue []finalTxn - txns map[ethkit.Hash]struct{} + txns map[receiptKey]struct{} numBlocksToFinality *big.Int mu sync.Mutex } - type finalTxn struct { receipt Receipt blockNum *big.Int } -func (f *finalizer) len() int { +func (f *finalizer) setFinality(num int) { f.mu.Lock() defer f.mu.Unlock() - return len(f.queue) + f.numBlocksToFinality = big.NewInt(int64(num)) } - -// func (f *finalizer) lastBlockNum() *big.Int { -// f.mu.Lock() -// defer f.mu.Unlock() -// if len(f.queue) == 0 { -// return big.NewInt(0) -// } -// return f.queue[0].blockNum -// } - -func (f *finalizer) enqueue(filterID uint64, receipt Receipt, blockNum *big.Int) { +func (f *finalizer) enqueue(owner *filterOwner, receipt Receipt, blockNum *big.Int) { f.mu.Lock() defer f.mu.Unlock() - - if receipt.Final { - // do not enqueue if the receipt is already final + if receipt.Final || blockNum == nil { return } - - txnHash := receipt.TransactionHash() - - // txn id based on the hash + filterID to ensure we get finalize callback for any unique filterID - txnID := txnHash - if filterID > 0 { - for i := 0; i < 8; i++ { - txnID[i] = txnID[i] + byte(filterID>>uint(i)) - } - } - - if _, ok := f.txns[txnID]; ok { - // update the blockNum if we already have this txn, as it could have been included - // again after a reorg in a new block + key := receiptOwner(receipt, owner) + if _, ok := f.txns[key]; ok { for i, entry := range f.queue { - if entry.receipt.TransactionHash() == txnHash { - f.queue[i] = finalTxn{receipt, blockNum} + if receiptOwner(entry.receipt, entry.receipt.owner) == key { + f.queue[i] = finalTxn{receipt, new(big.Int).Set(blockNum)} + return } } - return } - - // append new - f.queue = append(f.queue, finalTxn{receipt, blockNum}) - f.txns[txnID] = struct{}{} - - // sort block order from oldest to newest in case of a reorg - if len(f.queue) >= 2 && f.queue[0].blockNum.Cmp(f.queue[1].blockNum) < 0 { - sort.SliceStable(f.queue, func(i, j int) bool { - return f.queue[i].blockNum.Cmp(f.queue[j].blockNum) < 0 - }) + f.queue = append(f.queue, finalTxn{receipt, new(big.Int).Set(blockNum)}) + f.txns[key] = struct{}{} +} +func (f *finalizer) invalidateBlock(block blockRef) []Receipt { + f.mu.Lock() + defer f.mu.Unlock() + var invalidated []Receipt + retained := f.queue[:0] + for _, txn := range f.queue { + if txn.receipt.BlockHash() == block.hash && txn.receipt.generation == block.generation { + invalidated = append(invalidated, txn.receipt) + delete(f.txns, receiptOwner(txn.receipt, txn.receipt.owner)) + } else { + retained = append(retained, txn) + } } + clear(f.queue[len(retained):]) + f.queue = retained + return invalidated } - -func (f *finalizer) dequeue(currentBlockNum *big.Int) []finalTxn { +func (f *finalizer) invalidateOwner(owner *filterOwner) { f.mu.Lock() defer f.mu.Unlock() - - finalTxns := []finalTxn{} - + retained := f.queue[:0] for _, txn := range f.queue { - if currentBlockNum.Cmp(big.NewInt(0).Add(txn.blockNum, f.numBlocksToFinality)) > 0 { - finalTxns = append(finalTxns, txn) + if txn.receipt.owner == owner { + delete(f.txns, receiptOwner(txn.receipt, owner)) + } else { + retained = append(retained, txn) } } - - if len(finalTxns) > 0 { - f.queue = f.queue[len(finalTxns):] + clear(f.queue[len(retained):]) + f.queue = retained +} +func (f *finalizer) hasOwner(owner *filterOwner) bool { + f.mu.Lock() + defer f.mu.Unlock() + for _, txn := range f.queue { + if txn.receipt.owner == owner { + return true + } } - - return finalTxns + return false +} +func (f *finalizer) findOwner(filter Filterer) *filterOwner { + f.mu.Lock() + defer f.mu.Unlock() + for _, txn := range f.queue { + if sameFilter(txn.receipt.Filter, filter) { + return txn.receipt.owner + } + } + return nil +} +func (f *finalizer) clear() { + f.mu.Lock() + defer f.mu.Unlock() + f.queue = nil + clear(f.txns) +} +func (f *finalizer) dequeue(currentBlockNum *big.Int) []finalTxn { + f.mu.Lock() + defer f.mu.Unlock() + // Zero is unresolved until Run selects the network's finality policy. + if f.numBlocksToFinality == nil || f.numBlocksToFinality.Sign() <= 0 { + return nil + } + var finalized []finalTxn + retained := f.queue[:0] + for _, txn := range f.queue { + if currentBlockNum.Cmp(new(big.Int).Add(txn.blockNum, f.numBlocksToFinality)) > 0 { + finalized = append(finalized, txn) + delete(f.txns, receiptOwner(txn.receipt, txn.receipt.owner)) + } else { + retained = append(retained, txn) + } + } + clear(f.queue[len(retained):]) + f.queue = retained + return finalized } diff --git a/ethreceipts/hardening_test.go b/ethreceipts/hardening_test.go new file mode 100644 index 00000000..74a34570 --- /dev/null +++ b/ethreceipts/hardening_test.go @@ -0,0 +1,996 @@ +package ethreceipts + +import ( + "context" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/ethrpc" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/0xsequence/ethkit/go-ethereum/crypto" + "github.com/goware/breaker" +) + +// Only the RPC methods used by the listener are provided. No external node is needed. +type hardeningProvider struct { + ethrpc.RawInterface + chainID func(context.Context) (*big.Int, error) + receipt func(context.Context, common.Hash) (*types.Receipt, error) +} + +func (p *hardeningProvider) ChainID(ctx context.Context) (*big.Int, error) { + if p.chainID != nil { + return p.chainID(ctx) + } + return big.NewInt(1), nil +} +func (p *hardeningProvider) TransactionReceipt(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if p.receipt != nil { + return p.receipt(ctx, h) + } + return nil, ethereum.NotFound +} +func hardeningBlock(num int64, txns ...*types.Transaction) *ethmonitor.Block { + b := types.NewBlockWithHeader(&types.Header{Number: big.NewInt(num), BlockHash: common.BigToHash(big.NewInt(num)), ParentHash: common.BigToHash(big.NewInt(num - 1)), GasLimit: 30_000_000, Time: uint64(num)}) + return ðmonitor.Block{Block: b.WithBody(types.Body{Transactions: txns}), Event: ethmonitor.Added, OK: true} +} +func hardeningTxn(t *testing.T, nonce uint64) (*types.Transaction, common.Address, common.Address) { + t.Helper() + key, err := crypto.HexToECDSA("1111111111111111111111111111111111111111111111111111111111111111") + if err != nil { + t.Fatal(err) + } + from := crypto.PubkeyToAddress(key.PublicKey) + to := common.HexToAddress("0x2222222222222222222222222222222222222222") + tx, err := types.SignTx(types.NewTransaction(nonce, to, big.NewInt(1), 21000, big.NewInt(1), nil), types.NewEIP155Signer(big.NewInt(1)), key) + if err != nil { + t.Fatal(err) + } + return tx, from, to +} +func hardeningReceipt(b *ethmonitor.Block, tx *types.Transaction) *types.Receipt { + return &types.Receipt{TxHash: tx.Hash(), BlockHash: b.Hash(), BlockNumber: b.Number(), Status: 1} +} +func hardeningListener(t *testing.T, p *hardeningProvider, opts Options, blocks ...*ethmonitor.Block) *ReceiptsListener { + t.Helper() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger = log + mo.WithLogs = true + mo.BlockRetentionLimit = 100 + mo.Bootstrap = true + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if len(blocks) == 0 { + blocks = []*ethmonitor.Block{hardeningBlock(100)} + } + if err = m.Chain().BootstrapFromBlocks(blocks); err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + l.ctx = context.Background() + l.br = breaker.New(log, 0, 1, 0) + return l +} +func hardeningStart(t *testing.T, l *ReceiptsListener) { + t.Helper() + done := make(chan error, 1) + go func() { done <- l.Run(context.Background()) }() + deadline := time.After(time.Second) + for l.monitor.NumSubscribers() != 1 { + select { + case err := <-done: + t.Fatalf("Run exited before subscription: %v", err) + case <-deadline: + t.Fatal("Run did not subscribe to monitor") + default: + time.Sleep(time.Millisecond) + } + } + t.Cleanup(func() { + l.Stop() + select { + case err := <-done: + if err != nil { + t.Errorf("Run: %v", err) + } + case <-time.After(time.Second): + t.Error("Run did not stop") + } + }) +} +func hardeningRead(t *testing.T, s Subscription) Receipt { + t.Helper() + select { + case r := <-s.TransactionReceipt(): + return r + case <-time.After(time.Second): + t.Fatal("missing receipt") + return Receipt{} + } +} +func hardeningNoReceipt(t *testing.T, s Subscription) { + t.Helper() + select { + case r := <-s.TransactionReceipt(): + t.Errorf("unexpected receipt: txn=%s block=%s final=%v reorged=%v owner=%p", r.TransactionHash(), r.BlockHash(), r.Final, r.Reorged, r.Filter) + case <-time.After(30 * time.Millisecond): + } +} +func hardeningProcess(t *testing.T, l *ReceiptsListener, s *subscriber, blocks ...*ethmonitor.Block) { + t.Helper() + if _, err := l.processBlocks(blocks, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + t.Fatal(err) + } +} +func hardeningOptions() Options { opts := DefaultOptions; opts.NumBlocksToFinality = 2; return opts } + +func TestHardeningRollbackRemining(t *testing.T) { + tx, _, _ := hardeningTxn(t, 1) + old, newBlock := hardeningBlock(100, tx), hardeningBlock(102, tx) + var current atomic.Pointer[types.Receipt] + current.Store(hardeningReceipt(old, tx)) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { calls.Add(1); return current.Load(), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, old) + first := hardeningRead(t, s) + if first.Final || first.Reorged || first.BlockHash() != old.Hash() { + t.Fatalf("initial receipt: %+v", first) + } + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case rollback := <-s.TransactionReceipt(): + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.TransactionHash() != tx.Hash() { + t.Fatalf("rollback: %+v", rollback) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing rollback receipt") + } + if calls.Load() != 1 { + t.Errorf("rollback fetched orphan receipt: %d RPC calls", calls.Load()) + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + current.Store(hardeningReceipt(newBlock, tx)) + hardeningProcess(t, l, s, newBlock) + mined := hardeningRead(t, s) + if mined.BlockHash() != newBlock.Hash() || mined.Reorged || mined.Final { + t.Fatalf("re-mined receipt owns wrong block: %+v", mined) + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Reorged || final.BlockHash() != newBlock.Hash() || final.Filter != q { + t.Fatalf("canonical final receipt: %+v", final) + } +} + +func TestHardeningSubscribeBeforeRunFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 2) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, DefaultOptions, b, hardeningBlock(101)) + s := l.Subscribe(FilterTxnHash(tx.Hash())).(*subscriber) + defer s.Unsubscribe() + hardeningStart(t, l) + if r := hardeningRead(t, s); r.Final { + t.Fatal("receipt final before network threshold") + } + processed := make(chan struct{}) + s.AddFilter(FilterLogs(func([]*types.Log) bool { return false }).SearchCache(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { close(processed); return nil, nil })) + select { + case <-processed: + case <-time.After(time.Second): + t.Fatal("registration did not complete") + } + hardeningNoReceipt(t, s) + if len(s.Filters()) != 2 { + t.Fatal("premature finality removed transaction filter") + } +} + +func TestHardeningAddressFilters(t *testing.T) { + tx, from, to := hardeningTxn(t, 3) + b := hardeningBlock(100, tx) + for _, tc := range []struct { + name string + q FilterQuery + }{{"from", FilterFrom(from)}, {"to", FilterTo(to)}} { + t.Run(tc.name, func(t *testing.T) { + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions(), b) + s := l.Subscribe(tc.q.SearchCache(true)) + defer s.Unsubscribe() + hardeningStart(t, l) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() { + t.Fatalf("wrong matching transaction: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("signed address filter missed cached transaction") + } + }) + } +} + +func TestHardeningFinalizerOrderAndOwners(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 4) + tx2, _, _ := hardeningTxn(t, 5) + newer, older := hardeningBlock(101, tx1), hardeningBlock(100, tx2) + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + return hardeningReceipt(newer, tx1), nil + } + return hardeningReceipt(older, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) + q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + for _, r := range []*types.Receipt{hardeningReceipt(newer, tx1), hardeningReceipt(older, tx2)} { + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: r}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + hardeningRead(t, s) + } + for _, tc := range []struct { + head int64 + tx *types.Transaction + b *ethmonitor.Block + }{{103, tx2, older}, {104, tx1, newer}} { + if err := s.finalizeReceipts(big.NewInt(tc.head)); err != nil { + t.Fatal(err) + } + owners := map[Filterer]bool{} + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.TransactionHash() != tc.tx.Hash() || r.BlockHash() != tc.b.Hash() { + t.Errorf("wrong final at %d: %+v", tc.head, r) + } + owners[r.Filter] = true + case <-time.After(100 * time.Millisecond): + t.Errorf("missing owner final at %d", tc.head) + } + } + if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { + t.Errorf("finality lost distinct owners with same public ID at %d", tc.head) + } + } + hardeningNoReceipt(t, s) +} + +func TestHardeningPendingOwners(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6) + b := hardeningBlock(100, tx) + var failing atomic.Bool + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if failing.Load() { + return nil, errors.New("temporary provider failure") + } + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(0).Finalize(true) + q2 := FilterTxnHash(tx.Hash()).ID(0) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, chainID: big.NewInt(1)}}) + if err == nil { + t.Fatal("expected transient fetch failure") + } + failing.Store(false) + s.retryMu.Lock() + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + } + s.retryMu.Unlock() + s.retryPendingReceipts(context.Background()) + owners := map[Filterer]bool{} + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if r.Final || r.BlockHash() != b.Hash() { + t.Errorf("wrong recovered receipt: %+v", r) + } + owners[r.Filter] = true + case <-time.After(100 * time.Millisecond): + t.Error("missing recovered owner") + } + } + if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { + t.Error("retry lost an owner") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.BlockHash() != b.Hash() { + t.Errorf("wrong recovered final: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing recovered final") + } + } +} + +func TestHardeningLimitOneSnapshots(t *testing.T) { + for _, sameBlock := range []bool{true, false} { + t.Run(map[bool]string{true: "same_block", false: "retained_blocks"}[sameBlock], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 7) + tx2, _, _ := hardeningTxn(t, 8) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + blocks := ethmonitor.Blocks{b1, b2} + if sameBlock { + b1 = hardeningBlock(100, tx1, tx2) + b2 = b1 + blocks = ethmonitor.Blocks{b1} + } + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + return hardeningReceipt(b1, tx1), nil + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true)).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + if _, err := l.processBlocks(blocks, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + r := hardeningRead(t, s) + if r.TransactionHash() != tx1.Hash() { + t.Errorf("LimitOne did not select first match: %s", r.TransactionHash()) + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestHardeningQueryOnChainUsesCompleteReceipt(t *testing.T) { + tx, _, _ := hardeningTxn(t, 9) + b := hardeningBlock(90, tx) + complete := hardeningReceipt(b, tx) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + return nil, ethereum.NotFound + }} + l := hardeningListener(t, p, hardeningOptions()) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil })).(*subscriber) + defer s.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.Receipt() != complete || r.BlockHash() != b.Hash() { + t.Fatal("callback receipt replaced") + } + case <-time.After(100 * time.Millisecond): + t.Error("complete callback receipt was not delivered") + } + if calls.Load() != 0 { + t.Errorf("complete callback caused %d redundant RPCs", calls.Load()) + } +} + +func TestHardeningFetchPreservesSharedQuery(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }) + s := l.Subscribe(q) + defer s.Unsubscribe() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { defer wg.Done(); _, _, _ = l.FetchTransactionReceiptWithFilter(ctx, q) }() + } + wg.Wait() + opts := s.Filters()[0].Options() + if opts.LimitOne || opts.SearchCache { + t.Fatalf("Fetch rewrote active subscription: %+v", opts) + } +} + +func TestHardeningStopCancelsStartup(t *testing.T) { + entered := make(chan struct{}) + p := &hardeningProvider{chainID: func(ctx context.Context) (*big.Int, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() }} + l := hardeningListener(t, p, DefaultOptions) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + <-entered + l.Stop() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + cancel() + <-done + t.Error("Stop did not cancel startup ChainID request") + } +} +func TestHardeningMonitorStopCancelsRegistration(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + entered := make(chan struct{}) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return false }).QueryOnChain(func(ctx context.Context) (*types.Receipt, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() })) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { done <- l.Run(context.Background()) }() + <-entered + l.monitor.UnsubscribeAll(errors.New("monitor closed")) + select { + case <-done: + case <-time.After(100 * time.Millisecond): + l.Stop() + <-done + t.Error("monitor closure did not cancel registration") + } +} +func TestHardeningFetchSemaphoreCancellation(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + close(entered) + select { + case <-release: + return &types.Receipt{TxHash: h, BlockNumber: big.NewInt(100)}, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + }} + opts := hardeningOptions() + opts.MaxConcurrentFetchReceiptWorkers = 1 + l := hardeningListener(t, p, opts) + first := make(chan error, 1) + go func() { _, err := l.fetchTransactionReceipt(context.Background(), common.Hash{1}, true); first <- err }() + <-entered + ctx, cancel := context.WithCancel(context.Background()) + cancel() + second := make(chan error, 1) + go func() { _, err := l.fetchTransactionReceipt(ctx, common.Hash{2}, true); second <- err }() + blocked := false + select { + case err := <-second: + if !errors.Is(err, context.Canceled) { + t.Errorf("cancellation error: %v", err) + } + case <-time.After(100 * time.Millisecond): + blocked = true + } + close(release) + <-first + if blocked { + <-second + t.Error("canceled fetch waited for unrelated fetch slot") + } +} +func TestHardeningFinalityWaitCancellation(t *testing.T) { + tx, _, _ := hardeningTxn(t, 10) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + hardeningStart(t, l) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + mined, wait, err := l.FetchTransactionReceiptWithFinality(parent, tx.Hash()) + if err != nil { + t.Fatal(err) + } + if mined.Final { + t.Fatal("initial receipt final") + } + ctx, finalCancel := context.WithCancel(context.Background()) + finalCancel() + done := make(chan error, 1) + go func() { _, err := wait(ctx); done <- err }() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("wait error: %v", err) + } + case <-time.After(100 * time.Millisecond): + cancel() + <-done + t.Error("finality waiter ignored its own context") + } +} +func TestHardeningConcurrentRunStop(t *testing.T) { + for i := 0; i < 5; i++ { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + l.ctx, l.ctxStop = context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var wg sync.WaitGroup + start := make(chan struct{}) + for j := 0; j < 16; j++ { + wg.Add(1) + go func() { defer wg.Done(); <-start; _ = l.Run(ctx) }() + } + wg.Add(1) + go func() { + defer wg.Done() + <-start + for j := 0; j < 16; j++ { + l.Stop() + } + }() + close(start) + wg.Wait() + } +} + +func TestHardeningInflightRollbackRemining(t *testing.T) { + tx, _, _ := hardeningTxn(t, 11) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + select { + case <-release: + return hardeningReceipt(old, tx), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(canonical, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + snapshot := s.Filters() + oldDone := make(chan error, 1) + go func() { + _, err := l.processBlocks(ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}) + oldDone <- err + }() + <-entered + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case rollback := <-s.TransactionReceipt(): + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.Filter != q { + t.Errorf("wrong in-flight rollback: %+v", rollback) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing in-flight rollback") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + hardeningProcess(t, l, s, canonical) + select { + case mined := <-s.TransactionReceipt(): + if mined.BlockHash() != canonical.Hash() || mined.Reorged || mined.Final || mined.Filter != q { + t.Errorf("wrong canonical mined receipt: %+v", mined) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing canonical mined receipt") + } + close(release) + select { + case <-oldDone: + case <-time.After(time.Second): + t.Fatal("old fetch did not finish") + } + // A registration snapshot retained before rollback must also remain invalid. + if _, err := l.processBlocks(ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + select { + case final := <-s.TransactionReceipt(): + if !final.Final || final.Reorged || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Errorf("wrong canonical final: %+v", final) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing canonical final") + } + cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || cached.BlockHash != canonical.Hash() { + t.Errorf("late orphan RPC poisoned receipt cache: receipt=%+v error=%v", cached, err) + } + if calls.Load() != 2 { + t.Errorf("unexpected fetch count: %d", calls.Load()) + } +} + +func TestHardeningConcurrentLimitOne(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 12) + tx2, _, _ := hardeningTxn(t, 13) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + close(entered) + select { + case <-release: + return hardeningReceipt(b1, tx1), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + done := make(chan error, 1) + go func() { + _, err := l.processBlocks(ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{fs}) + done <- err + }() + <-entered + if _, err := l.processBlocks(ethmonitor.Blocks{b2}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + close(release) + <-done + r := hardeningRead(t, s) + if r.TransactionHash() != tx1.Hash() || r.Filter != q { + t.Errorf("concurrent LimitOne selected different transaction: %+v", r) + } + hardeningNoReceipt(t, s) + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.TransactionHash() != tx1.Hash() || final.Filter != q { + t.Errorf("wrong claimed final: %+v", final) + } + hardeningNoReceipt(t, s) +} + +// This matcher deliberately needs the fetched receipt, which removed block data +// cannot supply. Rollback must use its previously delivered owner and receipt. +type hardeningStatusFilter struct{ Filterer } + +func (f *hardeningStatusFilter) Match(_ context.Context, r Receipt) (bool, error) { + return r.Status() == 1, nil +} +func TestHardeningRollbackRetainsCustomOwner(t *testing.T) { + tx, _, _ := hardeningTxn(t, 14) + b := hardeningBlock(100, tx) + complete := hardeningReceipt(b, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return complete, nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := &hardeningStatusFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil }).(Filterer)} + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { + t.Fatal(err) + } + first := hardeningRead(t, s) + if first.Filter != q || first.Receipt() != complete { + t.Fatal("wrong custom filter owner or receipt") + } + removed := *b + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case r := <-s.TransactionReceipt(): + if !r.Reorged || r.Final || r.Filter != q || r.Receipt() != complete { + t.Errorf("rollback lost retained data/owner: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("custom complete-receipt matcher lost rollback") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} + +func TestHardeningRollbackInvalidatesPending(t *testing.T) { + tx, _, _ := hardeningTxn(t, 15) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + var failing atomic.Bool + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if failing.Load() { + return nil, errors.New("transient receipt error") + } + return hardeningReceipt(canonical, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, old) + s.retryMu.Lock() + pendingBefore := len(s.pendingReceipts) + s.retryMu.Unlock() + if pendingBefore != 1 { + t.Fatalf("expected one pending owner, got %d", pendingBefore) + } + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + s.retryMu.Lock() + pendingAfter := len(s.pendingReceipts) + s.retryMu.Unlock() + if pendingAfter != 0 { + t.Errorf("rollback retained %d orphan pending deliveries", pendingAfter) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Reorged || r.Filter != q || r.BlockHash() != old.Hash() { + t.Errorf("wrong pending rollback: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing pending rollback") + } + failing.Store(false) + hardeningProcess(t, l, s, canonical) + mined := hardeningRead(t, s) + if mined.BlockHash() != canonical.Hash() || mined.Reorged { + t.Fatal("retry ownership interfered with re-mining") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Fatal("wrong re-mined final owner") + } +} + +func TestHardeningExplicitRemovalCancelsOnlyOwner(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove_one", true: "clear_all"}[clearAll], func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 16) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(1).Finalize(true) + q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(2).Finalize(true) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + hardeningRead(t, s) + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q1.(Filterer)) + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + if !clearAll { + r := hardeningRead(t, s) + if !r.Final || r.Filter != q2 || r.BlockHash() != b.Hash() { + t.Errorf("removal lost surviving owner: %+v", r) + } + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestHardeningExhaustedOwnerKeepsQueuedFinal(t *testing.T) { + tx, _, _ := hardeningTxn(t, 17) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).MaxWait(1) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + // Exhaustion removes matching and closes its signal, but documented pending + // finality remains owned. Exercise that state without relying on a live chain. + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + q.(*filter).closeExhausted() + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Filter != q || final.BlockHash() != b.Hash() { + t.Fatal("exhaustion canceled a queued final") + } + hardeningNoReceipt(t, s) +} + +func TestHardeningMonitorClosureCancelsHeadWait(t *testing.T) { + p := &hardeningProvider{} + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger = log + mo.WithLogs = true + mo.BlockRetentionLimit = 100 + mo.Bootstrap = true + monitor, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, monitor, hardeningOptions()) + if err != nil { + t.Fatal(err) + } + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + deadline := time.After(time.Second) + for monitor.NumSubscribers() != 1 { + select { + case <-deadline: + t.Fatal("listener did not subscribe") + default: + time.Sleep(time.Millisecond) + } + } + monitor.UnsubscribeAll(errors.New("monitor closed before first block")) + select { + case <-done: + case <-time.After(100 * time.Millisecond): + cancel() + // Baseline LatestBlockNum has a fixed 30-second wait; let it finish + // without mutating the monitor's bootstrap state during concurrent reads. + select { + case <-done: + case <-time.After(32 * time.Second): + t.Fatal("Run did not clean up") + } + t.Error("monitor closure did not cancel waiting for first head") + } +} + +func TestHardeningClearCancelsExhaustedFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 18) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + q.(*filter).closeExhausted() + s.ClearFilters() + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} + +func TestHardeningSameHashReadoptionRejectsOldWork(t *testing.T) { + tx, _, _ := hardeningTxn(t, 19) + block := hardeningBlock(100, tx) + oldReceipt, canonicalReceipt := hardeningReceipt(block, tx), hardeningReceipt(block, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + select { + case <-release: + return oldReceipt, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return canonicalReceipt, nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + oldDone := make(chan error, 1) + go func() { + _, err := l.processBlocks(ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}) + oldDone <- err + }() + <-entered + removed := *block + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.BlockHash() != block.Hash() || rollback.Filter != q { + t.Fatalf("wrong rollback: %+v", rollback) + } + // A cache snapshot taken before removal cannot authoritatively re-adopt it. + hardeningProcess(t, l, s, block) + hardeningNoReceipt(t, s) + if calls.Load() != 1 { + t.Errorf("stale snapshot cleared removal marker: %d fetches", calls.Load()) + } + // The live block-processing entry point receives the canonical Added event. + if _, err := l.processBlocksContext(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { + t.Errorf("wrong readopted receipt: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("canonical same-hash re-adoption did not reopen delivery") + } + close(release) + <-oldDone + hardeningNoReceipt(t, s) + if calls.Load() != 2 { + t.Errorf("expected fresh readoption fetch, got %d", calls.Load()) + } + cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || cached != canonicalReceipt { + t.Errorf("old in-flight work replaced canonical cache: receipt=%p error=%v", cached, err) + } + // A callback with missing block data must acquire the fresh block generation + // when completing its receipt from cache, rather than inheriting generation zero. + partial := &types.Receipt{TxHash: tx.Hash()} + completion := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return partial, nil })).(*subscriber) + defer completion.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), completion, completion.Filters()); err != nil { + t.Fatal(err) + } + select { + case r := <-completion.TransactionReceipt(): + if r.Receipt() != canonicalReceipt || r.BlockHash() != block.Hash() { + t.Error("incomplete callback lost readopted receipt") + } + case <-time.After(100 * time.Millisecond): + t.Error("missing-data callback lost canonical generation") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { + t.Errorf("wrong readopted final: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing readopted final") + } + hardeningNoReceipt(t, s) +} diff --git a/ethreceipts/ownership_regression_test.go b/ethreceipts/ownership_regression_test.go new file mode 100644 index 00000000..51059845 --- /dev/null +++ b/ethreceipts/ownership_regression_test.go @@ -0,0 +1,621 @@ +package ethreceipts + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +type ownershipValueFilter struct { + Filterer + labels []string + match func(context.Context, Receipt) (bool, error) +} + +func (f ownershipValueFilter) Match(ctx context.Context, r Receipt) (bool, error) { + return f.match(ctx, r) +} +func TestOwnershipCustomValueFilter(t *testing.T) { + tx, _, _ := hardeningTxn(t, 100) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions(), b) + base := FilterLogs(func([]*types.Log) bool { return false }).SearchCache(true).(Filterer) + var matches atomic.Int32 + q := ownershipValueFilter{base, []string{"custom"}, func(_ context.Context, r Receipt) (bool, error) { + matches.Add(1) + return r.TransactionHash() == tx.Hash(), nil + }} + s := l.Subscribe(q) + defer s.Unsubscribe() + gotFilter, ok := s.Filters()[0].(ownershipValueFilter) + if !ok || gotFilter.Filterer != base || gotFilter.labels[0] != "custom" { + t.Fatal("public Filters lost original custom value") + } + hardeningStart(t, l) + r := hardeningRead(t, s) + original, ok := r.Filter.(ownershipValueFilter) + if !ok || original.Filterer != base || original.labels[0] != "custom" || original.match == nil { + t.Fatal("receipt lost original custom Filterer value") + } + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || matches.Load() == 0 { + t.Fatal("custom Match behavior lost") + } + s.ClearFilters() +} + +type ownershipCollectionFilter struct { + Filterer + cancel context.CancelFunc +} + +func (f *ownershipCollectionFilter) Match(context.Context, Receipt) (bool, error) { + if f.cancel != nil { + f.cancel() + return false, nil + } + return false, errors.New("temporary match failure") +} +func TestOwnershipCollectionAbort(t *testing.T) { + for _, canceled := range []bool{false, true} { + t.Run(map[bool]string{false: "matcher_error", true: "cancellation"}[canceled], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 101) + tx2, _, _ := hardeningTxn(t, 102) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + bad := &ownershipCollectionFilter{Filterer: FilterLogs(func([]*types.Log) bool { return false }).(Filterer)} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if canceled { + bad.cancel = cancel + } + s := l.Subscribe(q, bad).(*subscriber) + defer s.Unsubscribe() + _, err := s.matchFiltersAndPublish(ctx, s.Filters(), []Receipt{{receipt: hardeningReceipt(b1, tx1)}}) + if err == nil { + t.Error("expected matching abort") + } + if canceled && !errors.Is(err, context.Canceled) { + t.Errorf("cancellation error: %v", err) + } + if len(s.inFlight) != 0 || len(s.claims) != 0 { + t.Errorf("aborted collection retained work: inFlight=%d claims=%d", len(s.inFlight), len(s.claims)) + } + hardeningNoReceipt(t, s) + s.RemoveFilter(bad) + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b2, tx2)}}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("unused claim blocked later transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("aborted reservation blocked later processing") + } + }) + } +} + +func TestOwnershipQueryReuse(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove", true: "clear"}[clearAll], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 103) + tx2, _, _ := hardeningTxn(t, 104) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b1, tx1)}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q.(Filterer)) + } + if len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 { + t.Error("explicit removal retained canceled ownership") + } + s.AddFilter(q) + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b2, tx2)}}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q || r.Final { + t.Error("reused query kept old selection") + } + case <-time.After(100 * time.Millisecond): + t.Error("reused LimitOne query missed second transaction") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("canceled owner finalized in new lifetime") + } + case <-time.After(100 * time.Millisecond): + t.Error("new owner did not finalize") + } + hardeningNoReceipt(t, s) + }) + } +} +func TestOwnershipReaddRejectsOldWorker(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove", true: "clear"}[clearAll], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 105) + tx2, _, _ := hardeningTxn(t, 106) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + close(entered) + select { + case <-release: + return hardeningReceipt(b1, tx1), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { + _, err := l.processBlocks(ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{s.Filters()}) + done <- err + }() + <-entered + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q.(Filterer)) + } + s.AddFilter(q) + hardeningProcess(t, l, s, b2) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("new lifetime selected old transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("new lifetime suppressed by old claim") + } + close(release) + <-done + hardeningNoReceipt(t, s) + if len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 { + t.Error("completed/canceled lifetime retained state") + } + }) + } +} +func TestOwnershipCompletionReleasesState(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + s := l.Subscribe().(*subscriber) + defer s.Unsubscribe() + for i := 0; i < 32; i++ { + tx, _, _ := hardeningTxn(t, uint64(200+i)) + b := hardeningBlock(100, tx) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(i%4 != 0) + s.AddFilter(q) + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + switch i % 4 { + case 1: + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + case 2: + s.RemoveFilter(q.(Filterer)) + case 3: + s.ClearFilters() + } + if len(s.Filters()) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 { + t.Fatalf("completion %d retained state: active=%d claims=%d inFlight=%d deliveries=%d", i, len(s.Filters()), len(s.claims), len(s.inFlight), len(s.deliveries)) + } + if len(s.finalizer.queue) != 0 || len(s.finalizer.txns) != 0 || len(s.pendingReceipts) != 0 { + t.Fatalf("completion %d retained queued ownership", i) + } + } +} + +func TestOwnershipPendingDropReleasesClaim(t *testing.T) { + for _, notFound := range []bool{false, true} { + t.Run(map[bool]string{false: "retry_limit", true: "not_found"}[notFound], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 108) + tx2, _, _ := hardeningTxn(t, 109) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx2.Hash() { + return hardeningReceipt(b2, tx2), nil + } + return nil, errors.New("temporary receipt failure") + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx1, blockHash: b1.Hash(), blockNum: b1.Number()}}); err == nil { + t.Fatal("expected initial pending fetch failure") + } + if len(s.pendingReceipts) != 1 || len(s.claims) != 1 { + t.Fatal("pending fetch lost its selection") + } + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + pending.attempts = maxReceiptRetryAttempts - 1 + } + if notFound { + p.receipt = func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx2.Hash() { + return hardeningReceipt(b2, tx2), nil + } + return nil, ethereum.NotFound + } + } + s.retryPendingReceipts(context.Background()) + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 { + t.Error("discarded pending work retained an undelivered claim") + } + hardeningProcess(t, l, s, b2) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("released pending claim selected wrong transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("discarded pending work blocked a new transaction") + } + }) + } +} + +type ownershipGateFilter struct { + Filterer + entered, release chan struct{} + calls atomic.Int32 +} + +func (f *ownershipGateFilter) Match(ctx context.Context, _ Receipt) (bool, error) { + if f.calls.Add(1) == 1 { + close(f.entered) + select { + case <-f.release: + case <-ctx.Done(): + return false, ctx.Err() + } + } + return false, nil +} +func TestOwnershipStaleFetchPreservesCache(t *testing.T) { + for _, sameHash := range []bool{false, true} { + t.Run(map[bool]string{false: "remined_block", true: "readopted_generation"}[sameHash], func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 107) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + if sameHash { + canonical = old + } + complete := hardeningReceipt(canonical, tx) + var calls atomic.Int32 + var unavailable atomic.Bool + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if unavailable.Load() { + return nil, ethereum.NotFound + } + return complete, nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + gate := &ownershipGateFilter{Filterer: FilterLogs(func([]*types.Log) bool { return false }).(Filterer), entered: make(chan struct{}), release: make(chan struct{})} + s := l.Subscribe(q, gate).(*subscriber) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { + _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, blockHash: old.Hash(), blockNum: old.Number()}}) + done <- err + }() + <-gate.entered + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + hardeningRead(t, s) + if sameHash { + if _, err := l.processBlocksContext(context.Background(), ethmonitor.Blocks{canonical}, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + t.Fatal(err) + } + } else { + hardeningProcess(t, l, s, canonical) + } + hardeningRead(t, s) + cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Fatal("canonical receipt was not cached") + } + close(gate.release) + <-done + cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Error("late collection destroyed canonical cache") + } + unavailable.Store(true) + if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{old.Hash(), 0}); !errors.Is(err, ethereum.NotFound) { + t.Errorf("stale request error: %v", err) + } + cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Error("stale expected block/generation deleted current cache") + } + r, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || r != complete { + t.Errorf("fresh cache query depended on unavailable origin: %v", err) + } + if calls.Load() != 1 { + t.Errorf("stale work refetched a current canonical result: calls=%d", calls.Load()) + } + }) + } +} + +// The real monitor polls this provider and builds/broadcasts reorg events. Tests +// advance its canonical RPC responses without changing monitor production APIs. +type ownershipLiveProvider struct { + *hardeningProvider + mu sync.Mutex + canonical map[uint64]*types.Block + blocks map[common.Hash]*types.Block +} + +func (p *ownershipLiveProvider) advance(blocks ...*types.Block) { + p.mu.Lock() + defer p.mu.Unlock() + for _, b := range blocks { + p.canonical[b.NumberU64()] = b + p.blocks[b.Hash()] = b + } +} + +func ownershipBlockPayload(b *types.Block) (json.RawMessage, error) { + if b == nil { + return nil, ethereum.NotFound + } + header := b.Header() + header.Difficulty = big.NewInt(0) + payload, err := json.Marshal(header) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err = json.Unmarshal(payload, &fields); err != nil { + return nil, err + } + fields["transactions"], err = json.Marshal(b.Transactions()) + if err != nil { + return nil, err + } + fields["uncles"] = json.RawMessage(`[]`) + return json.Marshal(fields) +} + +func (p *ownershipLiveProvider) RawBlockByNumber(_ context.Context, num *big.Int) (json.RawMessage, error) { + p.mu.Lock() + b := p.canonical[num.Uint64()] + p.mu.Unlock() + return ownershipBlockPayload(b) +} + +func (p *ownershipLiveProvider) RawBlockByHash(_ context.Context, hash common.Hash) (json.RawMessage, error) { + p.mu.Lock() + b := p.blocks[hash] + p.mu.Unlock() + return ownershipBlockPayload(b) +} + +func (p *ownershipLiveProvider) RawFilterLogs(context.Context, ethereum.FilterQuery) (json.RawMessage, error) { + return json.RawMessage(`[]`), nil +} + +func TestOwnershipLiveExhaustedRollbackReleasesOwner(t *testing.T) { + tx, _, _ := hardeningTxn(t, 110) + b := hardeningBlock(100, tx) + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }}, + canonical: make(map[uint64]*types.Block), + blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 100 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99)}); err != nil { + t.Fatal(err) + } + opts := hardeningOptions() + opts.NumBlocksToFinality = 10 + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true).MaxWait(1) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningStart(t, l) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor Run did not stop") + } + }) + wait := func(what string, condition func() bool) { + t.Helper() + deadline := time.Now().Add(12 * time.Second) + for !condition() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } + } + alternate := func(num, fork int64, parent common.Hash) *types.Block { + header := hardeningBlock(num).Header() + header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+fork)), parent + return types.NewBlockWithHeader(header) + } + p.advance(b.Block) + mined := hardeningRead(t, s) + if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Final || mined.Reorged { + t.Fatal("wrong initial mined owner") + } + wait("first live match", func() bool { return q.(Filterer).LastMatchBlockNum() == 100 }) + old101 := hardeningBlock(101) + p.advance(old101.Block) + wait("block 101", func() bool { return m.LatestBlockNum().Int64() == 101 }) + alt101 := alternate(101, 10000, b.Hash()) + alt102 := alternate(102, 10000, alt101.Hash()) + p.advance(alt101, alt102) + wait("live reorg counter reset", func() bool { + return q.(Filterer).StartBlockNum() == 102 && q.(Filterer).LastMatchBlockNum() == 0 + }) + alt103 := alternate(103, 10000, alt102.Hash()) + p.advance(alt103) + select { + case <-q.(Filterer).Exhausted(): + case <-time.After(time.Second): + t.Fatal("live MaxWait did not exhaust owner") + } + s.deliveryMu.Lock() + queued := len(s.finalizer.queue) == 1 && len(s.claims) == 1 + s.deliveryMu.Unlock() + if !queued || len(s.Filters()) != 0 { + t.Fatal("exhaustion lost queued finality ownership") + } + current := hardeningBlock(99).Block + for num := int64(100); num <= 104; num++ { + next := alternate(num, 20000, current.Hash()) + p.advance(next) + current = next + } + wait("rollback invalidation", func() bool { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + return m.LatestBlock().Hash() == current.Hash() && len(s.finalizer.queue) == 0 + }) + select { + case rollback := <-s.TransactionReceipt(): + if rollback.TransactionHash() != tx.Hash() || rollback.BlockHash() != b.Hash() || rollback.BlockNumber().Int64() != 100 || rollback.Filter != q || !rollback.Reorged || rollback.Final || rollback.Status() != 1 { + t.Error("wrong exhausted-owner rollback identity/state") + } + case <-time.After(100 * time.Millisecond): + t.Error("exhausted queued owner missed its delivered rollback") + } + checkRetired := func() { + t.Helper() + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.inFlight) != 0 || len(s.pendingReceipts) != 0 || len(s.finalizer.queue) != 0 || len(s.finalizer.txns) != 0 { + t.Errorf("inactive rollback owner retained state: claims=%d deliveries=%d", len(s.claims), len(s.deliveries)) + } + } + checkRetired() + for num := int64(105); num <= 112; num++ { + next := alternate(num, 20000, current.Hash()) + p.advance(next) + current = next + } + wait("finality advancement", func() bool { return m.LatestBlock().Hash() == current.Hash() }) + s.RemoveFilter(q.(Filterer)) + checkRetired() + hardeningNoReceipt(t, s) +} + +func TestOwnershipExhaustedRollbackKeepsOtherCandidates(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 111) + tx2, _, _ := hardeningTxn(t, 112) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + other := FilterTxnHash(tx2.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q, other).(*subscriber) + defer s.Unsubscribe() + for _, b := range []*ethmonitor.Block{b1, b2} { + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b, b.Transactions()[0])}}); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 3; i++ { + hardeningRead(t, s) + } + s.exhaustFilter(q.(Filterer)) + q.(*filter).closeExhausted() + removed := *b1 + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx1.Hash() || r.BlockHash() != b1.Hash() || r.Filter != q || !r.Reorged || r.Final { + t.Error("wrong exhausted retained rollback") + } + case <-time.After(100 * time.Millisecond): + t.Error("inactive owner missed rollback while another candidate remained") + } + if len(s.finalizer.queue) != 2 || len(s.Filters()) != 1 { + t.Fatal("rollback removed unrelated ownership") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + seen := make(map[Filterer]bool) + for i := 0; i < 2; i++ { + r := hardeningRead(t, s) + if r.TransactionHash() != tx2.Hash() || r.BlockHash() != b2.Hash() || !r.Final || r.Reorged || seen[r.Filter] { + t.Error("rollback lost or duplicated another owned final") + } + seen[r.Filter] = true + } + if !seen[q.(Filterer)] || !seen[other.(Filterer)] || len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.Filters()) != 0 { + t.Error("finished owners were not retired") + } + hardeningNoReceipt(t, s) +} diff --git a/ethreceipts/receipt.go b/ethreceipts/receipt.go index 01ad8b96..9cfb918b 100644 --- a/ethreceipts/receipt.go +++ b/ethreceipts/receipt.go @@ -21,6 +21,10 @@ type Receipt struct { transaction *types.Transaction receipt *types.Receipt logs []*types.Log + blockNum *big.Int + blockHash common.Hash + generation uint64 + owner *filterOwner // TODOXXX: this intermediate type is lame.. with new ethrpc we can remove // NOTE: we only use this for From/To address resolution currently @@ -61,7 +65,7 @@ func (r *Receipt) BlockNumber() *big.Int { if r.receipt != nil { return r.receipt.BlockNumber } else { - return nil + return r.blockNum } } @@ -69,7 +73,7 @@ func (r *Receipt) BlockHash() ethkit.Hash { if r.receipt != nil { return r.receipt.BlockHash } else { - return ethkit.Hash{} + return r.blockHash } } @@ -175,12 +179,18 @@ func (r *Receipt) To() common.Address { } func (r *Receipt) AsMessage() (*core.Message, error) { - msg, ok := r.message.Load().(*core.Message) - if !ok { - return nil, fmt.Errorf("ethreceipts: Receipt.message type-assertion fail, unexpected") + cached := r.message.Load() + if cached != nil { + msg, ok := cached.(*core.Message) + if !ok { + return nil, fmt.Errorf("ethreceipts: Receipt.message type-assertion fail, unexpected") + } + if msg != nil { + return msg, nil + } } - if msg != nil { - return msg, nil + if r.transaction == nil || r.chainID == nil { + return nil, fmt.Errorf("ethreceipts: transaction and chainID required to resolve message") } // TODOXXX: avoid using AsMessage as its fairly expensive operation, especially diff --git a/ethreceipts/subscription.go b/ethreceipts/subscription.go index 0d59f0a5..c1b55b12 100644 --- a/ethreceipts/subscription.go +++ b/ethreceipts/subscription.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/big" + "reflect" "sync" "time" @@ -48,17 +49,21 @@ type subscriber struct { ch channel.Channel[Receipt] done chan struct{} unsubscribe func() - filters []Filterer + filters []*filterOwner finalizer *finalizer mu sync.Mutex - pendingReceipts map[common.Hash]*pendingReceipt + pendingReceipts map[receiptKey]*pendingReceipt retryMu sync.Mutex + deliveryMu sync.Mutex + deliveries map[receiptKey]Receipt + inFlight map[receiptKey]struct{} + claims map[*filterOwner]common.Hash } type pendingReceipt struct { receipt Receipt - filterer Filterer + filterer *filterOwner attempts int nextRetryAt time.Time } @@ -80,225 +85,487 @@ func (s *subscriber) Unsubscribe() { s.unsubscribe() } +// Every registration owns a comparable token; the public filter may contain +// slices, maps or functions and may be reused in a later registration. +type filterOwner struct{ Filterer } + +func sameFilter(a, b Filterer) bool { + if reflect.ValueOf(a).Comparable() && reflect.ValueOf(b).Comparable() { + return a == b + } + return reflect.DeepEqual(a, b) +} + func (s *subscriber) Filters() []Filterer { s.mu.Lock() defer s.mu.Unlock() filters := make([]Filterer, len(s.filters)) - copy(filters, s.filters) + for i, owner := range s.filters { + filters[i] = owner.Filterer + } return filters } -func (s *subscriber) AddFilter(filterQueries ...FilterQuery) { - if len(filterQueries) == 0 { - return +func (s *subscriber) filterers() []Filterer { + s.mu.Lock() + defer s.mu.Unlock() + filters := make([]Filterer, len(s.filters)) + for i, owner := range s.filters { + filters[i] = owner } + return filters +} - filters := make([]Filterer, len(filterQueries)) - for i, query := range filterQueries { +func (s *subscriber) AddFilter(queries ...FilterQuery) { + if len(queries) == 0 { + return + } + owners := make([]*filterOwner, len(queries)) + filters := make([]Filterer, len(queries)) + for i, query := range queries { filterer, ok := query.(Filterer) if !ok { panic("ethreceipts: unexpected") } - filters[i] = filterer + owners[i] = &filterOwner{filterer} + filters[i] = owners[i] } - s.mu.Lock() - if len(s.filters)+len(filters) > maxFiltersPerListener { - // too many filters, ignore the extra filter. not ideal, but better than - // deadlocking - s.listener.log.Warn(fmt.Sprintf("ethreceipts: subscriber has too many filters (%d), ignoring extra", len(s.filters)+len(filters))) - // TODO: maybe return an error or force-unsubscribe instead? + if len(s.filters)+len(owners) > maxFiltersPerListener { + s.listener.log.Warn(fmt.Sprintf("ethreceipts: subscriber has too many filters (%d), ignoring extra", len(s.filters)+len(owners))) s.mu.Unlock() return } - s.filters = append(s.filters, filters...) + s.filters = append(s.filters, owners...) s.mu.Unlock() - - // TODO: maybe add non-blocking push structure like in relayer queue select { case s.listener.registerFiltersCh <- registerFilters{subscriber: s, filters: filters}: - // ok default: s.listener.log.Warn("ethreceipts: listener registerFiltersCh full, dropping filter register") } } -func (s *subscriber) RemoveFilter(filter Filterer) { +func (s *subscriber) owner(filter Filterer) *filterOwner { + if owner, ok := filter.(*filterOwner); ok { + return owner + } s.mu.Lock() defer s.mu.Unlock() + for _, owner := range s.filters { + if sameFilter(owner.Filterer, filter) { + return owner + } + } + return nil +} +func (s *subscriber) RemoveFilter(filter Filterer) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + owner := s.owner(filter) + if owner == nil { + owner = s.finalizer.findOwner(filter) + } + if owner != nil { + s.retireOwner(owner, true) + } +} + +func (s *subscriber) removeActive(owner *filterOwner) { + s.mu.Lock() + defer s.mu.Unlock() for i, f := range s.filters { - if f == filter { - s.filters = append(s.filters[:i], s.filters[i+1:]...) + if f == owner { + copy(s.filters[i:], s.filters[i+1:]) + s.filters[len(s.filters)-1] = nil + s.filters = s.filters[:len(s.filters)-1] return } } } +// Caller holds deliveryMu. Exhaustion retains queued finals, while explicit +// cancellation and automatic completion release the complete owner lifetime. +func (s *subscriber) retireOwner(owner *filterOwner, cancelFinality bool) { + s.removeActive(owner) + if cancelFinality { + s.finalizer.invalidateOwner(owner) + } + if cancelFinality || !s.finalizer.hasOwner(owner) { + delete(s.claims, owner) + for key := range s.deliveries { + if key.owner == owner { + delete(s.deliveries, key) + } + } + } + for key := range s.inFlight { + if key.owner == owner { + delete(s.inFlight, key) + } + } + s.retryMu.Lock() + for key := range s.pendingReceipts { + if key.owner == owner { + delete(s.pendingReceipts, key) + } + } + s.retryMu.Unlock() +} + +func (s *subscriber) exhaustFilter(filter Filterer) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if owner := s.owner(filter); owner != nil { + s.retireOwner(owner, false) + } +} + func (s *subscriber) ClearFilters() { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + s.finalizer.clear() + clear(s.claims) + clear(s.inFlight) + clear(s.deliveries) + s.retryMu.Lock() + clear(s.pendingReceipts) + s.retryMu.Unlock() +} + +// Reserve matches before fetching so concurrent registration and live processing +// cannot select different transactions for one LimitOne filter. +func (s *subscriber) reserve(receipt Receipt, owner *filterOwner) bool { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if !s.hasFilter(owner) || !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + return false + } + if s.claims == nil { + s.claims = make(map[*filterOwner]common.Hash) + } + if owner.Options().LimitOne { + if txn, claimed := s.claims[owner]; claimed && txn != receipt.TransactionHash() { + return false + } + s.claims[owner] = receipt.TransactionHash() + } + key := receiptOwner(receipt, owner) + if _, exists := s.deliveries[key]; exists { + return false + } + if s.inFlight == nil { + s.inFlight = make(map[receiptKey]struct{}) + } + if _, exists := s.inFlight[key]; exists { + return false + } + s.retryMu.Lock() + _, pending := s.pendingReceipts[key] + s.retryMu.Unlock() + if pending { + return false + } + s.inFlight[key] = struct{}{} + return true +} + +func (s *subscriber) releaseReservation(receipt Receipt, owner *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + delete(s.inFlight, receiptOwner(receipt, owner)) + if s.finalizer.hasOwner(owner) { + return + } + for key := range s.inFlight { + if key.owner == owner { + return + } + } + for key := range s.deliveries { + if key.owner == owner { + return + } + } + s.retryMu.Lock() + defer s.retryMu.Unlock() + for key := range s.pendingReceipts { + if key.owner == owner { + return + } + } + delete(s.claims, owner) +} + +func (s *subscriber) hasFilter(owner *filterOwner) bool { s.mu.Lock() defer s.mu.Unlock() - s.filters = s.filters[:0] + for _, f := range s.filters { + if f == owner { + return true + } + } + return false } -// matchFiltersAndPublish matches the given receipts against the provided filterers, -// fetches any missing receipt data as needed, and notifies the subscriber of matches. -func (s *subscriber) matchFiltersAndPublish(ctx context.Context, filterers []Filterer, receipts []Receipt) ([]bool, error) { - oks := make([]bool, len(filterers)) +// RPC waits happen outside deliveryMu. Rollback, delivery and finalization share +// this lock so an invalidated in-flight receipt cannot be published afterwards. +func (s *subscriber) publish(ctx context.Context, receipt Receipt, owner *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if ctx.Err() != nil || !s.hasFilter(owner) { + return + } + s.listener.receiptMu.Lock() + defer s.listener.receiptMu.Unlock() + if !s.listener.currentBlock(receipt.BlockHash(), receipt.generation) { + return + } + key := receiptOwner(receipt, owner) + if _, delivered := s.deliveries[key]; delivered { + return + } + if owner.Options().LimitOne { + if txn, claimed := s.claims[owner]; claimed && txn != receipt.TransactionHash() { + return + } + } + receipt.Filter = owner.Filterer + receipt.owner = owner + receipt.Final = s.listener.isBlockFinal(receipt.BlockNumber()) + if !receipt.Final && owner.Options().Finalize { + s.finalizer.enqueue(owner, receipt, receipt.BlockNumber()) + } + if s.deliveries == nil { + s.deliveries = make(map[receiptKey]Receipt) + } + s.deliveries[key] = receipt + s.ch.Send(receipt) + if owner.Options().LimitOne && (!owner.Options().Finalize || receipt.Final) { + s.retireOwner(owner, true) + } +} - // Collect matches that need receipt fetching - type matchedReceipt struct { - receipt Receipt - filtererIdx int - filterer Filterer +func (s *subscriber) rollbackBlock(block blockRef) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + for _, receipt := range s.finalizer.invalidateBlock(block) { + // Queued finality still owns the delivered receipt after exhaustion, + // including when finalized-block history has already been pruned. + key := receiptOwner(receipt, receipt.owner) + if _, retained := s.deliveries[key]; !retained { + if s.deliveries == nil { + s.deliveries = make(map[receiptKey]Receipt) + } + s.deliveries[key] = receipt + } + } + s.retryMu.Lock() + for key := range s.pendingReceipts { + if key.blockHash == block.hash && key.generation == block.generation { + delete(s.pendingReceipts, key) + } } - var toFetch []matchedReceipt + s.retryMu.Unlock() + // A custom matcher may require fields that lightweight block receipts lack. + // Previously published owners already matched, so use their retained data. + inactive := make(map[*filterOwner]struct{}) + for key, receipt := range s.deliveries { + if key.blockHash != block.hash || key.generation != block.generation { + continue + } + if !receipt.Reorged { + receipt.Final = false + receipt.Reorged = true + s.deliveries[key] = receipt + s.ch.Send(receipt) + } + if !s.hasFilter(key.owner) { + inactive[key.owner] = struct{}{} + } + } + // Notify every delivered receipt before retiring an exhausted lifetime. + for owner := range inactive { + if !s.finalizer.hasOwner(owner) { + s.retireOwner(owner, false) + } + } +} - // First pass: find all matches +func (s *subscriber) rollback(receipt Receipt, owner *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + key := receiptOwner(receipt, owner) + if !s.hasFilter(owner) { + return + } + if txn, claimed := s.claims[owner]; owner.Options().LimitOne && claimed && txn != receipt.TransactionHash() { + return + } + if retained, ok := s.deliveries[key]; ok { + if retained.Reorged { + return + } + receipt = retained + } + receipt.Filter = owner.Filterer + receipt.owner = owner + receipt.Final = false + receipt.Reorged = true + if s.deliveries == nil { + s.deliveries = make(map[receiptKey]Receipt) + } + s.deliveries[key] = receipt + s.ch.Send(receipt) +} + +func (s *subscriber) fetchReceipt(ctx context.Context, receipt Receipt) (Receipt, error) { + l := s.listener + l.receiptMu.Lock() + started := l.reorgRevision + l.receiptMu.Unlock() + expected := blockRef{receipt.BlockHash(), receipt.generation} + r, err := l.fetchTransactionReceipt(ctx, receipt.TransactionHash(), true, expected) + if err != nil { + return receipt, err + } + if expected.hash == (common.Hash{}) { + l.receiptMu.Lock() + valid := l.validFetchedBlock(r.BlockHash, blockRef{}, started) + receipt.generation = l.blockStates[r.BlockHash].generation + l.receiptMu.Unlock() + if !valid { + return receipt, ethereum.NotFound + } + } + receipt.receipt = r + receipt.logs = r.Logs + return receipt, nil +} + +func (s *subscriber) matchFiltersAndPublish(ctx context.Context, filterers []Filterer, receipts []Receipt) ([]bool, error) { + oks := make([]bool, len(filterers)) + type match struct { + receipt Receipt + owner *filterOwner + } + var matches []match + owners := make([]*filterOwner, len(filterers)) + for i, filter := range filterers { + owners[i] = s.owner(filter) + } for _, receipt := range receipts { - for i, filterer := range filterers { - matched, err := filterer.Match(ctx, receipt) + for i, owner := range owners { + if ctx.Err() != nil { + return oks, ctx.Err() + } + if owner == nil || !s.hasFilter(owner) { + continue + } + matched, err := owner.Match(ctx, receipt) if err != nil { return oks, superr.New(ErrFilterMatch, err) } - if !matched { - // skip, not a match continue } - - // its a match oks[i] = true - - if !receipt.Reorged { - toFetch = append(toFetch, matchedReceipt{ - receipt: receipt, - filtererIdx: i, - filterer: filterer, - }) + if receipt.Reorged { + s.rollback(receipt, owner) + continue } + matches = append(matches, match{receipt, owner}) } } - - if len(toFetch) == 0 { - return oks, nil - } - - // Fetch receipts concurrently sem := make(chan struct{}, maxConcurrentReceiptFetches) - g, gctx := errgroup.WithContext(ctx) - - for _, item := range toFetch { - item := item // capture loop variable + // One owner's fetch failure must not cancel another owner's delivery or retry. + var g errgroup.Group + for _, item := range matches { + if ctx.Err() != nil { + break + } + if !s.reserve(item.receipt, item.owner) { + continue + } g.Go(func() error { + defer s.releaseReservation(item.receipt, item.owner) select { case sem <- struct{}{}: defer func() { <-sem }() - case <-gctx.Done(): - return gctx.Err() + case <-ctx.Done(): + return ctx.Err() } - - // Fetch transaction receipt - r, err := s.listener.fetchTransactionReceipt(gctx, item.receipt.TransactionHash(), true) - if err != nil { - if errors.Is(err, ethereum.NotFound) { - // not found, don't retry - return superr.Wrap(fmt.Errorf("txn %s not found", item.receipt.TransactionHash()), err) + receipt := item.receipt + if receipt.receipt == nil || receipt.receipt.BlockNumber == nil || receipt.receipt.TxHash == (common.Hash{}) || receipt.receipt.BlockHash == (common.Hash{}) { + completed, err := s.fetchReceipt(ctx, receipt) + if err != nil { + if !errors.Is(err, ethereum.NotFound) && ctx.Err() == nil { + s.addPendingReceipt(receipt, item.owner) + } + return superr.Wrap(fmt.Errorf("failed to fetch txn %s receipt", receipt.TransactionHash()), err) } - - // might be a provider issue, add to pending receipts for retry - s.addPendingReceipt(item.receipt, item.filterer) - return superr.Wrap(fmt.Errorf("failed to fetch txn %s receipt due to node issue", item.receipt.TransactionHash()), err) + receipt = completed } - - // Update receipt with fetched data - item.receipt.receipt = r - item.receipt.logs = r.Logs - item.receipt.Filter = item.filterer - - // Finality enqueue if filter asked to Finalize, and receipt isn't already final - if !item.receipt.Final && item.filterer.Options().Finalize { - s.finalizer.enqueue(item.filterer.FilterID(), item.receipt, item.receipt.BlockNumber()) - } - - // LimitOne will auto unsubscribe now if were not also waiting for finalizer, - // and if the returned txn isn't one that has been reorged - // - // NOTE: when Finalize is set, we don't want to remove this filter until the txn finalizes, - // because its possible that it can reorg and we have to fetch it again after being re-mined. - // So we only remove the filter now if the filter finalizer isn't used, otherwise the - // finalizer will remove the LimitOne filter - toFinalize := item.filterer.Options().Finalize && !item.receipt.Final - if item.filterer.Options().LimitOne && !toFinalize { - s.RemoveFilter(item.receipt.Filter) - } - - // Check if receipt is already final, in case comes from cache when - // previously final was not toggled. - if s.listener.isBlockFinal(item.receipt.BlockNumber()) { - item.receipt.Final = true - } - - // Broadcast to subscribers (needs mutex as multiple goroutines may send) - s.ch.Send(item.receipt) - + s.publish(ctx, receipt, item.owner) return nil }) } - - // Wait for all fetches to complete - if err := g.Wait(); err != nil { - return oks, err + err := g.Wait() + if err == nil && ctx.Err() != nil { + err = ctx.Err() } - - return oks, nil + return oks, err } func (s *subscriber) finalizeReceipts(blockNum *big.Int) error { - // check subscriber finalizer - finalizer := s.finalizer - if finalizer.len() == 0 { - return nil - } - - finalTxns := finalizer.dequeue(blockNum) - if len(finalTxns) == 0 { - // no matching txns which have been finalized - return nil - } - - // dispatch to subscriber finalized receipts - for _, x := range finalTxns { - if x.receipt.Reorged { - // for removed receipts, just skip - continue + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + for _, txn := range s.finalizer.dequeue(blockNum) { + receipt := txn.receipt + owner := receipt.owner + s.listener.receiptMu.Lock() + if !receipt.Reorged && s.listener.currentBlock(receipt.BlockHash(), receipt.generation) { + receipt.Final = true + s.ch.Send(receipt) + if owner != nil && (owner.Cond().TxnHash != nil || owner.Options().LimitOne) { + s.retireOwner(owner, true) + } } - - // mark receipt as final, and send the receipt payload to the subscriber - x.receipt.Final = true - - // send to the subscriber - s.ch.Send(x.receipt) - - // Automatically remove filters for finalized txn hashes, as they won't come up again. - filter := x.receipt.Filter - if filter != nil && (filter.Cond().TxnHash != nil || filter.Options().LimitOne) { - s.RemoveFilter(filter) + s.listener.receiptMu.Unlock() + if !s.hasFilter(owner) && !s.finalizer.hasOwner(owner) { + s.retireOwner(owner, false) + } + } + // Retain mined data until its block is final, for receipt-free rollback. + for key, receipt := range s.deliveries { + if receipt.BlockNumber() != nil && s.listener.isBlockFinal(receipt.BlockNumber()) { + delete(s.deliveries, key) } } - return nil } -func (s *subscriber) addPendingReceipt(receipt Receipt, filterer Filterer) { +func (s *subscriber) addPendingReceipt(receipt Receipt, filterer *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if !s.hasFilter(filterer) { + return + } s.retryMu.Lock() defer s.retryMu.Unlock() txnHash := receipt.TransactionHash() + key := receiptOwner(receipt, filterer) + if !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + return + } if s.pendingReceipts == nil { // lazy init - s.pendingReceipts = make(map[common.Hash]*pendingReceipt) + s.pendingReceipts = make(map[receiptKey]*pendingReceipt) } if len(s.pendingReceipts) >= maxPendingReceipts { @@ -310,12 +577,12 @@ func (s *subscriber) addPendingReceipt(receipt Receipt, filterer Filterer) { return } - if _, exists := s.pendingReceipts[txnHash]; exists { + if _, exists := s.pendingReceipts[key]; exists { // already pending, skip return } - s.pendingReceipts[txnHash] = &pendingReceipt{ + s.pendingReceipts[key] = &pendingReceipt{ receipt: receipt, filterer: filterer, attempts: 1, @@ -359,6 +626,7 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { wg.Add(1) go func(p *pendingReceipt) { defer wg.Done() + defer s.releaseReservation(p.receipt, p.filterer) select { case sem <- struct{}{}: @@ -366,7 +634,7 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { case <-ctx.Done(): // If context is cancelled, release the claim so the item can be retried later. s.retryMu.Lock() - if current, ok := s.pendingReceipts[p.receipt.TransactionHash()]; ok && current == p { + if current, ok := s.pendingReceipts[receiptOwner(p.receipt, p.filterer)]; ok && current == p { current.nextRetryAt = time.Now().Add(100 * time.Millisecond) // small delay to avoid immediate retry } s.retryMu.Unlock() @@ -375,22 +643,24 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { // Attempt to fetch the receipt txnHash := p.receipt.TransactionHash() - r, err := s.listener.fetchTransactionReceipt(ctx, txnHash, true) + key := receiptOwner(p.receipt, p.filterer) + receipt, err := s.fetchReceipt(ctx, p.receipt) s.retryMu.Lock() - defer s.retryMu.Unlock() // Check if the item still exists and is the same one we claimed. - currentPending, exists := s.pendingReceipts[txnHash] + currentPending, exists := s.pendingReceipts[key] if !exists || currentPending != p { + s.retryMu.Unlock() s.listener.log.Debug("Pending receipt is stale or already processed, skipping retry", "txnHash", txnHash.String()) return } if err != nil { + defer s.retryMu.Unlock() if errors.Is(err, ethereum.NotFound) { // Transaction genuinely doesn't exist - remove from queue - delete(s.pendingReceipts, txnHash) + delete(s.pendingReceipts, key) s.listener.log.Debug("Receipt not found after retry, removing from queue", "txnHash", txnHash.String()) return } @@ -398,7 +668,7 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { // Provider error - update retry state directly on the pointer. currentPending.attempts++ if currentPending.attempts >= maxReceiptRetryAttempts { - delete(s.pendingReceipts, txnHash) + delete(s.pendingReceipts, key) s.listener.log.Error( "Failed to fetch receipt after max retries", "txnHash", txnHash.String(), @@ -427,36 +697,16 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { } // Remove from pending list - delete(s.pendingReceipts, txnHash) - - // Update receipt with fetched data - p.receipt.receipt = r - p.receipt.logs = r.Logs - p.receipt.Filter = p.filterer - - // Check finality - if s.listener.isBlockFinal(r.BlockNumber) { - p.receipt.Final = true - } - - // Handle finalization queue if needed - if !p.receipt.Final && p.filterer.Options().Finalize { - s.finalizer.enqueue(p.filterer.FilterID(), p.receipt, r.BlockNumber) - } - - // Handle LimitOne filter removal - toFinalize := p.filterer.Options().Finalize && !p.receipt.Final - if p.filterer.Options().LimitOne && !toFinalize { - s.RemoveFilter(p.filterer) - } + delete(s.pendingReceipts, key) - // Send to subscriber - s.ch.Send(p.receipt) + attempts := currentPending.attempts + s.retryMu.Unlock() + s.publish(ctx, receipt, p.filterer) s.listener.log.Info( "Successfully fetched receipt after retry", "txnHash", txnHash.String(), - "attempts", currentPending.attempts, + "attempts", attempts, ) }(pending) } From 9ec19fb5aa9f30cc10285c059924d63f41194d87 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Mon, 5 Oct 2026 16:21:40 -0400 Subject: [PATCH 02/11] fix(ethreceipts): preserve retry candidates and filter identity Keep successful canceled retries tied to their learned block generation, reopen canonical adopted blocks after retention, and support stable custom value filter removal. Document supported limits and add regression coverage. --- FIXES.md | 25 + ethreceipts/codegenie_regression_test.go | 709 +++++++++++++++++++++++ ethreceipts/ethreceipts.go | 17 +- ethreceipts/subscription.go | 91 ++- 4 files changed, 823 insertions(+), 19 deletions(-) create mode 100644 FIXES.md create mode 100644 ethreceipts/codegenie_regression_test.go diff --git a/FIXES.md b/FIXES.md new file mode 100644 index 00000000..51ec9c82 --- /dev/null +++ b/FIXES.md @@ -0,0 +1,25 @@ +# Receipt listener fixes + +These changes correct three receipt-listener findings and the additional partial-candidate issue found during independent review. Receipt work remains owned by its exact filter registration, transaction, block hash and generation. Public filter values and custom matching remain intact. + +- **Successful retry followed by cancellation.** A receipt fetch could succeed, lose its pending entry, then miss publication because the attempt expired while waiting for delivery. Pending removal now commits with publication under the delivery lock. A canceled attempt sends nothing and retains valid work for a prompt retry using the completed cache entry. RemoveFilter, ClearFilters and rollback remain terminal for the affected owner or candidate. +- **Partial callback completion.** A QueryOnChain callback may initially supply only TxHash. If its successful completion is retained after cancellation, the pending item now keeps the learned block/generation separately from its immutable queue identity. Rollback matches that completed candidate and releases its never-delivered LimitOne selection. Later retries validate the learned generation, so same-hash re-adoption cannot revive it. A fresh canonical transaction can be selected; an already-mined LimitOne transaction still keeps its selection through rollback and re-mining. +- **Same-hash re-adoption after retention eviction.** A delayed live Added event can outlast the monitor's retained chain. Receipt acceptance now uses positive evidence for that exact monitor incarnation, preserving valid adoption after deep eviction. Removed incarnations, cached registration snapshots and earlier RPC generations cannot reopen stale work. Missing hash lookup or height alone is insufficient; untracked/manual events require a positive retained canonical lookup. +- **Individual custom-value removal.** Comparable values and pointers retain Go equality. Otherwise unresolved non-comparable values of the same concrete type use a stable nonnil Exhausted channel as base identity. Shared-base values are aliases: removal chooses the first matching active registration, otherwise its first queued finality owner. Distinct bases or pointer identities provide independent removal, including through Filters() and Receipt.Filter copies. Nil/unstable signals cannot identify arbitrary callback value copies; register those as pointers for individual cancellation. Callback code pointers, unsafe identity and public filter IDs are not used as owner identity. + +The monitor support belongs to [PR219](https://github.com/0xsequence/ethkit/pull/219); receipt changes remain in [PR220](https://github.com/0xsequence/ethkit/pull/220), based directly on parallel-monitor. The parent confirms cached parent mismatches even with local prefetch disabled and exposes Block.CanonicalState: acceptance is owned per incarnation, actual removal invalidates it, and retention eviction preserves it. This evidence covers known monitor removals; reorgs beyond retained known history remain the monitor's existing limit. + +The parent's private Block state requires external positional literals to migrate to keyed literals, for example: + +```go +ethmonitor.Block{Block: block, Event: ethmonitor.Added, Logs: logs, OK: true} +``` + +Keyed construction and the existing JSON schema remain supported; serialized blocks carry no runtime incarnation evidence. No parent implementation files are included in the receipt correction delta. + +Final validation on actual synchronized sources: + +- All 39 permanent receipt regression groups passed three repetitions with the race detector: 156.481s. Eight new groups contain 30 cases; failing pre-fix evidence was captured for the confirmed defects. +- Full local receipt suite passed with the race detector: 252.573s, including existing integration tests. No overlay, chain reset or race warning. +- Parent monitor suite passed with the race detector: 73.007s. Only the external live-mainnet fee-history test was excluded; no external live RPC was used. +- go vet ./ethreceipts ./ethmonitor, go build ./..., Go formatting and diff checks passed. The requested documentation addition changes no runtime or test source. diff --git a/ethreceipts/codegenie_regression_test.go b/ethreceipts/codegenie_regression_test.go new file mode 100644 index 00000000..d69f7083 --- /dev/null +++ b/ethreceipts/codegenie_regression_test.go @@ -0,0 +1,709 @@ +package ethreceipts + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func receiptsFixCollect(s Subscription) []Receipt { + var receipts []Receipt + timer := time.NewTimer(40 * time.Millisecond) + defer timer.Stop() + for { + select { + case r := <-s.TransactionReceipt(): + receipts = append(receipts, r) + case <-timer.C: + return receipts + } + } +} + +func TestReceiptsFixRetryDeadlineAfterFetch(t *testing.T) { + for _, terminal := range []string{"retry", "remove", "clear", "rollback"} { + t.Run(terminal, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 800) + b := hardeningBlock(100, tx) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), b) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + seed := Receipt{transaction: tx, blockHash: b.Hash(), blockNum: b.Number()} + if !s.reserve(seed, owner) { + t.Fatal("initial reservation failed") + } + s.addPendingReceipt(seed, owner) + s.releaseReservation(seed, owner) + key := receiptOwner(seed, owner) + s.pendingReceipts[key].nextRetryAt = time.Time{} + pending := s.pendingReceipts[key] + parent := context.Background() + attempt, cancel := context.WithTimeout(parent, 200*time.Millisecond) + defer cancel() + s.deliveryMu.Lock() + done := make(chan struct{}) + go func() { s.retryPendingReceipts(attempt); close(done) }() + deadline := time.Now().Add(time.Second) + for { + _, cached, _ := l.pastReceipts.Get(parent, tx.Hash().Hex()) + if cached && len(l.fetchSem) == 0 { + break + } + if time.Now().After(deadline) { + s.deliveryMu.Unlock() + t.Fatal("receipt fetch did not complete before publication") + } + time.Sleep(time.Millisecond) + } + <-attempt.Done() + terminalDone := make(chan struct{}) + go func() { + defer close(terminalDone) + switch terminal { + case "remove": + s.RemoveFilter(q.(Filterer)) + case "clear": + s.ClearFilters() + case "rollback": + ref := l.invalidateBlock(parent, b) + s.rollbackBlock(ref) + } + }() + s.deliveryMu.Unlock() + <-done + <-terminalDone + hardeningNoReceipt(t, s) + if parent.Err() != nil { + t.Fatal("retry child canceled its listener parent") + } + if terminal == "retry" { + if s.pendingReceipts[key] != pending || len(s.claims) != 1 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Errorf("expired successful retry lost exact ownership: pending=%d claims=%d", len(s.pendingReceipts), len(s.claims)) + } + if s.pendingReceipts[key] == nil { + return + } + if pending.attempts != 1 || pending.nextRetryAt.After(time.Now().Add(time.Second)) { + t.Error("successful expired attempt was not promptly released for retry") + } + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(parent) + r := hardeningRead(t, s) + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q || r.Final || r.Reorged { + t.Error("later active retry lost mined identity") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + r = hardeningRead(t, s) + if !r.Final || r.Reorged || r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q { + t.Error("later active retry lost finality identity") + } + if calls.Load() != 1 { + t.Error("later retry discarded its successful cached receipt") + } + } else { + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Error("expired retry resurrected terminal owner/block state") + } + s.retryPendingReceipts(parent) + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestReceiptsFixPartialPendingCandidate(t *testing.T) { + for _, terminal := range []string{"retry", "rollback", "readoption", "remove", "clear"} { + t.Run(terminal, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 808) + replacement, _, _ := hardeningTxn(t, 809) + old, next := hardeningBlock(100, tx), hardeningBlock(101, replacement) + var ready atomic.Bool + var calls atomic.Int32 + l := hardeningListener(t, &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if !ready.Load() { + return nil, errors.New("provider temporarily unavailable") + } + return hardeningReceipt(old, tx), nil + }}, hardeningOptions(), old) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true). + QueryOnChain(func(context.Context) (*types.Receipt, error) { return &types.Receipt{TxHash: tx.Hash()}, nil }) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + if err := l.queryFilterOnChain(context.Background(), s, s.filterers()); err != nil { + t.Fatal(err) + } + if len(s.pendingReceipts) != 1 || s.claims[owner] != tx.Hash() { + t.Fatal("partial public callback did not enter the pending completion path") + } + var key receiptKey + var pending *pendingReceipt + for k, p := range s.pendingReceipts { + key, pending = k, p + } + if key.blockHash != (common.Hash{}) { + t.Fatal("partial callback already knew its block") + } + pending.nextRetryAt = time.Time{} + ready.Store(true) + attempt, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + s.deliveryMu.Lock() + done := make(chan struct{}) + go func() { s.retryPendingReceipts(attempt); close(done) }() + deadline := time.Now().Add(time.Second) + for { + _, cached, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if cached && len(l.fetchSem) == 0 { + break + } + if time.Now().After(deadline) { + s.deliveryMu.Unlock() + t.Fatal("partial pending fetch did not complete") + } + time.Sleep(time.Millisecond) + } + <-attempt.Done() + s.deliveryMu.Unlock() + <-done + hardeningNoReceipt(t, s) + if s.pendingReceipts[key] != pending || pending.attempts != 1 || calls.Load() != 2 { + t.Fatal("successful canceled completion did not retain its exact pending owner") + } + if terminal == "retry" { + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(context.Background()) + mined := hardeningRead(t, s) + if mined.Filter != q || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != old.Hash() || mined.generation != 0 || mined.Final || mined.Reorged { + t.Fatal("partial pending live retry lost its learned candidate") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if final.Filter != q || final.TransactionHash() != tx.Hash() || final.BlockHash() != old.Hash() || !final.Final || final.Reorged || calls.Load() != 2 { + t.Fatal("partial retry lost cached completion or canonical finality") + } + hardeningNoReceipt(t, s) + return + } + switch terminal { + case "rollback", "readoption": + ref := l.invalidateBlock(context.Background(), old) + if terminal == "readoption" { + // Even if authoritative same-hash acceptance wins before subscriber + // cleanup, the learned generation-0 candidate must stay invalid. + l.acceptBlock(l.monitor.GetBlock(old.Hash())) + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(context.Background()) + hardeningNoReceipt(t, s) + } + s.rollbackBlock(ref) + case "remove": + s.RemoveFilter(q.(Filterer)) + case "clear": + s.ClearFilters() + } + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Error("terminal invalidation retained a learned pending candidate or selection") + } + s.retryPendingReceipts(context.Background()) + hardeningNoReceipt(t, s) + if terminal == "remove" || terminal == "clear" { + s.AddFilter(q) + if s.owner(q.(Filterer)) == owner { + t.Fatal("new registration reused the canceled owner") + } + } + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: hardeningReceipt(next, replacement)}}); err != nil { + t.Fatal(err) + } + select { + case mined := <-s.TransactionReceipt(): + if mined.Filter != q || mined.TransactionHash() != replacement.Hash() || mined.BlockHash() != next.Hash() || mined.Final || mined.Reorged { + t.Error("fresh replacement lost its exact mined identity") + } + case <-time.After(100 * time.Millisecond): + t.Error("orphan pending claim blocked a fresh canonical replacement") + return + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if final.Filter != q || final.TransactionHash() != replacement.Hash() || final.BlockHash() != next.Hash() || !final.Final || final.Reorged { + t.Error("terminal invalidation published an orphan final") + } + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.Filters()) != 0 || calls.Load() != 2 { + t.Error("terminal candidate or completed owner was retained/re-fetched") + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestReceiptsFixCustomValueRemoval(t *testing.T) { + for _, pointer := range []bool{false, true} { + for _, exhausted := range []bool{false, true} { + for _, source := range []string{"original", "Filters", "Receipt.Filter"} { + t.Run(fmt.Sprintf("pointer=%v/exhausted=%v/%s", pointer, exhausted, source), func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 801) + tx2, _, _ := hardeningTxn(t, 802) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + custom := func(label string) Filterer { + return ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).ID(7).(Filterer), []string{label}, func(context.Context, Receipt) (bool, error) { return label != "", nil }} + } + value := custom("selected").(ownershipValueFilter) + var q Filterer = value + if pointer { + q = &value + } + other := custom("surviving") + s := l.Subscribe(q, other).(*subscriber) + defer s.Unsubscribe() + first, second := s.filterers()[0].(*filterOwner), s.filterers()[1].(*filterOwner) + fromFilters := s.Filters()[0] + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(b1, tx1)) + var mined Receipt + for i := 0; i < 2; i++ { + r := hardeningRead(t, s) + if r.owner == first { + mined = r + if pointer { + if r.Filter != q || fromFilters != q { + t.Fatal("public pointer identity changed") + } + } else if r.Filter.(ownershipValueFilter).Filterer != value.Filterer || fromFilters.(ownershipValueFilter).Filterer != value.Filterer { + t.Fatal("original public custom value changed") + } + } + } + if exhausted { + s.exhaustFilter(first) + value.Filterer.(*filter).closeExhausted() + } + candidate := q + switch source { + case "Filters": + candidate = fromFilters + case "Receipt.Filter": + candidate = mined.Filter + } + s.RemoveFilter(candidate) + if s.hasFilter(first) || s.finalizer.hasOwner(first) || !s.hasFilter(second) || !s.finalizer.hasOwner(second) { + t.Error("public removal did not isolate selected registration/finality") + } + process(hardeningReceipt(b2, tx2)) + late := receiptsFixCollect(s) + if len(late) != 1 || late[0].owner != second || late[0].TransactionHash() != tx2.Hash() || late[0].Final || late[0].Reorged { + t.Error("selected cancellation changed later surviving delivery") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(s) + seen := make(map[common.Hash]bool) + for _, r := range finals { + if r.owner != second || !r.Final || r.Reorged || seen[r.TransactionHash()] { + t.Error("removed custom value still finalized or surviving final duplicated") + } + seen[r.TransactionHash()] = true + } + if len(finals) != 2 || !seen[tx1.Hash()] || !seen[tx2.Hash()] { + t.Error("custom removal lost surviving finals") + } + }) + } + } + } +} + +func TestReceiptsFixSharedBaseAliases(t *testing.T) { + tx, _, _ := hardeningTxn(t, 803) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + base := FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).(Filterer) + first := ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + second := ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + other := ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer), []string{"other"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + s := l.Subscribe(first, second, other).(*subscriber) + defer s.Unsubscribe() + owners := s.filterers() + if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + hardeningRead(t, s) + } + // Shared base signals define aliases: old RemoveFilter removes the first + // matching registration. Independent callback identities use distinct bases. + s.RemoveFilter(second) + if s.hasFilter(owners[0].(*filterOwner)) || s.finalizer.hasOwner(owners[0].(*filterOwner)) || !s.hasFilter(owners[1].(*filterOwner)) || !s.hasFilter(owners[2].(*filterOwner)) { + t.Error("shared-base removal did not preserve first-match alias semantics") + } + s.RemoveFilter(s.Filters()[0]) + if s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) || !s.finalizer.hasOwner(owners[2].(*filterOwner)) { + t.Error("removing second base alias affected independent owner") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(s) + if len(finals) != 1 || finals[0].owner != owners[2] || !finals[0].Final || finals[0].TransactionHash() != tx.Hash() { + t.Error("shared aliases canceled an unrelated queued final") + } +} + +type receiptsFixComparableFilter struct { + Filterer + label string +} + +func (f receiptsFixComparableFilter) Match(context.Context, Receipt) (bool, error) { + return f.label != "", nil +} + +func TestReceiptsFixComparableSharedBaseIsolation(t *testing.T) { + for _, pointer := range []bool{false, true} { + t.Run(fmt.Sprintf("pointer=%v", pointer), func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 804) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + base := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer) + var first, second Filterer = receiptsFixComparableFilter{base, "first"}, receiptsFixComparableFilter{base, "second"} + if pointer { + first = &ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + second = &ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + } + s := l.Subscribe(first, second).(*subscriber) + defer s.Unsubscribe() + owners := s.filterers() + if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + hardeningRead(t, s) + s.RemoveFilter(second) + if !s.hasFilter(owners[0].(*filterOwner)) || !s.finalizer.hasOwner(owners[0].(*filterOwner)) || s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) { + t.Fatal("base fallback overrode definitive comparable identity") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(s) + if len(finals) != 1 || finals[0].owner != owners[0] || !finals[0].Final { + t.Fatal("comparable cancellation lost surviving owner") + } + }) + } +} + +type receiptsFixNilSignalFilter struct{ ownershipValueFilter } + +func (receiptsFixNilSignalFilter) Exhausted() <-chan struct{} { return nil } + +func TestReceiptsFixCustomValueWithoutIdentity(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + value := receiptsFixNilSignalFilter{ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).(Filterer), []string{"no identity"}, func(context.Context, Receipt) (bool, error) { return true, nil }}} + other := FilterLogs(func([]*types.Log) bool { return true }).(Filterer) + s := l.Subscribe(value, other).(*subscriber) + defer s.Unsubscribe() + s.RemoveFilter(s.Filters()[0]) + if len(s.Filters()) != 2 { + t.Fatal("unidentifiable callback value removed another registration") + } + s.ClearFilters() + s.AddFilter(&value, other) + s.RemoveFilter(s.Filters()[0]) + if len(s.Filters()) != 1 || s.Filters()[0] != other { + t.Fatal("pointer identity did not isolate custom nil-signal removal") + } +} + +func TestReceiptsFixRollbackPreservesMinedSelection(t *testing.T) { + tx, _, _ := hardeningTxn(t, 805) + other, _, _ := hardeningTxn(t, 806) + old, replacement, canonical := hardeningBlock(100, tx), hardeningBlock(101, other), hardeningBlock(102, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(old, tx)) + hardeningRead(t, s) + s.rollbackBlock(l.invalidateBlock(context.Background(), old)) + rollback := hardeningRead(t, s) + if s.claims[owner] != tx.Hash() || !rollback.Reorged || rollback.Filter != q || rollback.BlockHash() != old.Hash() { + t.Fatal("rollback released an already-mined selection") + } + process(hardeningReceipt(replacement, other)) + hardeningNoReceipt(t, s) + process(hardeningReceipt(canonical, tx)) + mined := hardeningRead(t, s) + if mined.Reorged || mined.Final || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != canonical.Hash() || mined.Filter != q { + t.Fatal("retained selection did not re-mine correctly") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Fatal("retained mined selection finalized an orphan") + } + hardeningNoReceipt(t, s) +} + +func TestReceiptsFixReadoptionAfterRetention(t *testing.T) { + for _, scenario := range []string{"canonical_retention", "obsolete_incarnation", "later_removal", "alternate_retention"} { + t.Run(scenario, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 807) + b := hardeningBlock(100, tx) + old, canonical := hardeningReceipt(b, tx), hardeningReceipt(b, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + var releaseOnce sync.Once + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + // Deliberately finish an old RPC after its request timeout, + // to verify invalidation of late provider completions. + <-release + return old, nil + } + return canonical, nil + }}, + canonical: make(map[uint64]*types.Block), blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 50 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99), b}); err != nil { + t.Fatal(err) + } + initialSnapshot := *m.LatestBlock() + opts := hardeningOptions() + opts.NumBlocksToFinality = 200 + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + stream := m.Subscribe("delayed receipt events") + defer stream.Unsubscribe() + oldDone := make(chan error, 1) + go func() { + _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{b.Hash(), 0}) + oldDone <- err + }() + <-entered + releaseOld := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(releaseOld) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor did not stop") + } + }) + consume := func() ethmonitor.Blocks { + t.Helper() + select { + case batch := <-stream.Blocks(): + return batch + case <-time.After(10 * time.Second): + t.Fatal("missing canonical monitor event") + return nil + } + } + process := func(batch ethmonitor.Blocks, cached bool) { + t.Helper() + var err error + if cached { + _, err = l.processCachedBlocksContext(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + } else { + _, err = l.processBlocksContext(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + } + if err != nil { + t.Fatal(err) + } + } + fork := func(from, through, offset int64, parent common.Hash) { + for num := from; num <= through; num++ { + header := hardeningBlock(num).Header() + header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+offset)), parent + next := types.NewBlockWithHeader(header) + p.advance(next) + parent = next.Hash() + } + } + waitHeight := func(height int64) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for m.LatestBlockNum().Int64() != height { + if time.Now().After(deadline) { + t.Fatal("monitor did not advance canonical height") + } + time.Sleep(time.Millisecond) + } + } + // The first real reorg removes H, invalidating the held generation-0 RPC. + fork(100, 101, 10000, hardeningBlock(99).Hash()) + process(consume(), false) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != b.Hash() || rollback.TransactionHash() != tx.Hash() || rollback.Filter != q { + t.Fatal("wrong original removal identity") + } + process(ethmonitor.Blocks{&initialSnapshot}, true) + hardeningNoReceipt(t, s) + // Rebuild the same hash H in a fresh, owned monitor incarnation. + p.advance(b.Block, hardeningBlock(101).Block, hardeningBlock(102).Block) + readoption := consume() + var added *ethmonitor.Block + for _, event := range readoption { + if event.Event == ethmonitor.Added && event.Hash() == b.Hash() { + added = event + } + } + if added == nil { + t.Fatal("monitor omitted readopted incarnation") + } + incarnation, valid := added.CanonicalState() + if incarnation == 0 || !valid { + t.Fatal("Added input was not the owned canonical monitor event") + } + process(ethmonitor.Blocks{added}, true) + hardeningNoReceipt(t, s) + negative := scenario == "later_removal" || scenario == "alternate_retention" + var removed *ethmonitor.Block + if negative { + fork(100, 103, 20000, hardeningBlock(99).Hash()) + for _, event := range consume() { + if event.Event == ethmonitor.Removed && event.Hash() == b.Hash() { + removed = event + } + } + if removed == nil { + t.Fatal("monitor omitted later removal") + } + } + if scenario == "canonical_retention" { + fork(103, 155, 0, hardeningBlock(102).Hash()) + waitHeight(155) + } else if scenario == "alternate_retention" { + fork(104, 155, 20000, common.BigToHash(big.NewInt(20103))) + waitHeight(155) + } + if scenario == "canonical_retention" || scenario == "alternate_retention" { + if m.GetBlock(b.Hash()) != nil || m.OldestBlockNum().Int64() != 106 { + t.Fatal("H was not evicted more than one block behind retention") + } + } + if scenario == "obsolete_incarnation" { + // A removed tracked snapshot cannot fall back to a newer retained + // incarnation of the same hash, even during authoritative processing. + if m.GetBlock(b.Hash()) == nil { + t.Fatal("newer same-hash incarnation is missing") + } + process(ethmonitor.Blocks{&initialSnapshot}, false) + hardeningNoReceipt(t, s) + if !l.blockStates[b.Hash()].removed { + t.Error("obsolete tracked incarnation used same-hash fallback") + } + } + process(readoption, false) + if negative { + hardeningNoReceipt(t, s) + if !l.blockStates[b.Hash()].removed { + t.Error("later-removed Added reopened canonical marker") + } + process(ethmonitor.Blocks{added, removed}, false) + for _, r := range receiptsFixCollect(s) { + if !r.Reorged || r.Final { + t.Error("same-batch removal published stale mined/final receipt") + } + } + if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false); !errors.Is(err, ethereum.NotFound) { + t.Errorf("later removal reopened hash lookup: %v", err) + } + } else { + select { + case mined := <-s.TransactionReceipt(): + if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Reorged || mined.Final || mined.generation != 1 { + t.Error("readopted incarnation lost exact mined ownership") + } + case <-time.After(100 * time.Millisecond): + t.Error("canonical readoption after retention was rejected") + } + result, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || result != canonical { + t.Errorf("canonical readoption hash lookup failed: %v", err) + } + } + releaseOld() + if err := <-oldDone; !errors.Is(err, ethereum.NotFound) { + t.Errorf("old generation RPC escaped invalidation: %v", err) + } + if !negative { + cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != canonical { + t.Error("old RPC replaced fresh canonical cache") + } + } + hardeningNoReceipt(t, s) + }) + } +} diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index c7acc58e..19fa0515 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -870,7 +870,7 @@ func (l *ReceiptsListener) processBlockEvents(ctx context.Context, blocks ethmon if canonicalEvents { for _, block := range blocks { if block.Event == ethmonitor.Added { - l.acceptBlock(block.Hash()) + l.acceptBlock(block) } } } @@ -884,7 +884,7 @@ func (l *ReceiptsListener) processBlockEvents(ctx context.Context, blocks ethmon // report if the txn was removed reorged := block.Event == ethmonitor.Removed if canonicalEvents && !reorged { - l.acceptBlock(block.Hash()) + l.acceptBlock(block) } generation := l.blockGeneration(block.Hash()) if reorged { @@ -1247,13 +1247,20 @@ func (l *ReceiptsListener) validFetchedBlock(hash common.Hash, expected blockRef } return expected.hash == (common.Hash{}) || (hash == expected.hash && state.generation == expected.generation) } -func (l *ReceiptsListener) acceptBlock(hash common.Hash) { +func (l *ReceiptsListener) acceptBlock(block *ethmonitor.Block) { l.receiptMu.Lock() defer l.receiptMu.Unlock() + hash := block.Hash() state := l.blockStates[hash] if state.removed { - canonical := l.monitor.GetBlock(hash) - if canonical != nil && canonical.Event == ethmonitor.Added { + incarnation, canonical := block.CanonicalState() + if incarnation == 0 { + // Legacy/manual blocks have no incarnation evidence; only a + // positive retained lookup can authorize their re-adoption. + retained := l.monitor.GetBlock(hash) + canonical = retained != nil && retained.Event == ethmonitor.Added + } + if canonical { state.removed = false l.blockStates[hash] = state } diff --git a/ethreceipts/subscription.go b/ethreceipts/subscription.go index c1b55b12..33249a12 100644 --- a/ethreceipts/subscription.go +++ b/ethreceipts/subscription.go @@ -38,6 +38,12 @@ type Subscription interface { Filters() []Filterer AddFilter(filters ...FilterQuery) + // RemoveFilter removes the first matching active registration, or its queued + // finality owner if inactive. Comparable filters retain Go equality. Otherwise + // callback-bearing values of the same type use a stable nonnil Exhausted signal + // as base identity. Values sharing a base are aliases; distinct callbacks need + // distinct bases or pointer identities for independent removal. Values without + // a stable signal should be registered as pointers for individual removal. RemoveFilter(filter Filterer) ClearFilters() } @@ -63,6 +69,7 @@ type subscriber struct { type pendingReceipt struct { receipt Receipt + completed *Receipt // Learned candidate; receipt retains the original queue key. filterer *filterOwner attempts int nextRetryAt time.Time @@ -93,7 +100,14 @@ func sameFilter(a, b Filterer) bool { if reflect.ValueOf(a).Comparable() && reflect.ValueOf(b).Comparable() { return a == b } - return reflect.DeepEqual(a, b) + if reflect.DeepEqual(a, b) { + return true + } + if reflect.TypeOf(a) != reflect.TypeOf(b) { + return false + } + identity := a.Exhausted() + return identity != nil && identity == b.Exhausted() } func (s *subscriber) Filters() []Filterer { @@ -277,6 +291,11 @@ func (s *subscriber) releaseReservation(receipt Receipt, owner *filterOwner) { s.deliveryMu.Lock() defer s.deliveryMu.Unlock() delete(s.inFlight, receiptOwner(receipt, owner)) + s.releaseClaim(owner) +} + +// Caller holds deliveryMu. Keep selection only while real owner work remains. +func (s *subscriber) releaseClaim(owner *filterOwner) { if s.finalizer.hasOwner(owner) { return } @@ -313,24 +332,30 @@ func (s *subscriber) hasFilter(owner *filterOwner) bool { // RPC waits happen outside deliveryMu. Rollback, delivery and finalization share // this lock so an invalidated in-flight receipt cannot be published afterwards. -func (s *subscriber) publish(ctx context.Context, receipt Receipt, owner *filterOwner) { +func (s *subscriber) publish(ctx context.Context, receipt Receipt, owner *filterOwner) bool { s.deliveryMu.Lock() defer s.deliveryMu.Unlock() + return s.publishLocked(ctx, receipt, owner) +} + +// Caller holds deliveryMu so retry publication and pending ownership commit +// together with Remove/Clear and rollback invalidation. +func (s *subscriber) publishLocked(ctx context.Context, receipt Receipt, owner *filterOwner) bool { if ctx.Err() != nil || !s.hasFilter(owner) { - return + return false } s.listener.receiptMu.Lock() defer s.listener.receiptMu.Unlock() - if !s.listener.currentBlock(receipt.BlockHash(), receipt.generation) { - return + if ctx.Err() != nil || !s.listener.currentBlock(receipt.BlockHash(), receipt.generation) { + return false } key := receiptOwner(receipt, owner) if _, delivered := s.deliveries[key]; delivered { - return + return true } if owner.Options().LimitOne { if txn, claimed := s.claims[owner]; claimed && txn != receipt.TransactionHash() { - return + return false } } receipt.Filter = owner.Filterer @@ -347,6 +372,7 @@ func (s *subscriber) publish(ctx context.Context, receipt Receipt, owner *filter if owner.Options().LimitOne && (!owner.Options().Finalize || receipt.Final) { s.retireOwner(owner, true) } + return true } func (s *subscriber) rollbackBlock(block blockRef) { @@ -364,8 +390,14 @@ func (s *subscriber) rollbackBlock(block blockRef) { } } s.retryMu.Lock() - for key := range s.pendingReceipts { - if key.blockHash == block.hash && key.generation == block.generation { + invalidatedPending := make(map[*filterOwner]struct{}) + for key, pending := range s.pendingReceipts { + candidate := pending.receipt + if pending.completed != nil { + candidate = *pending.completed + } + if candidate.BlockHash() == block.hash && candidate.generation == block.generation { + invalidatedPending[key.owner] = struct{}{} delete(s.pendingReceipts, key) } } @@ -393,6 +425,9 @@ func (s *subscriber) rollbackBlock(block blockRef) { s.retireOwner(owner, false) } } + for owner := range invalidatedPending { + s.releaseClaim(owner) + } } func (s *subscriber) rollback(receipt Receipt, owner *filterOwner) { @@ -644,7 +679,13 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { // Attempt to fetch the receipt txnHash := p.receipt.TransactionHash() key := receiptOwner(p.receipt, p.filterer) - receipt, err := s.fetchReceipt(ctx, p.receipt) + s.retryMu.Lock() + candidate := p.receipt + if p.completed != nil { + candidate = *p.completed + } + s.retryMu.Unlock() + receipt, err := s.fetchReceipt(ctx, candidate) s.retryMu.Lock() @@ -696,12 +737,34 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { return } - // Remove from pending list - delete(s.pendingReceipts, key) - attempts := currentPending.attempts s.retryMu.Unlock() - s.publish(ctx, receipt, p.filterer) + + s.deliveryMu.Lock() + s.retryMu.Lock() + currentPending, exists = s.pendingReceipts[key] + s.retryMu.Unlock() + if !exists || currentPending != p { + s.deliveryMu.Unlock() + return + } + published := s.publishLocked(ctx, receipt, p.filterer) + s.retryMu.Lock() + if current, exists := s.pendingReceipts[key]; exists && current == p { + if published || !s.hasFilter(p.filterer) || !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + delete(s.pendingReceipts, key) + } else { + // A successful fetch can outlive its child deadline while waiting + // for delivery. Keep this exact valid owner available to a live retry. + current.completed = &receipt + current.nextRetryAt = time.Now().Add(100 * time.Millisecond) + } + } + s.retryMu.Unlock() + s.deliveryMu.Unlock() + if !published { + return + } s.listener.log.Info( "Successfully fetched receipt after retry", From 90392047bd186f5578884cb3c3068943898846ba Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Mon, 5 Oct 2026 18:58:26 -0400 Subject: [PATCH 03/11] ci: test Go 1.25 and 1.26 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11046a93..6bf49709 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - go-version: [1.24.x,1.25.x] + go-version: [1.25.x,1.26.x] os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} From 654c14950655c9c04b541e86d82ec84d8c1eb716 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 08:15:44 -0400 Subject: [PATCH 04/11] fix(ethreceipts): retry current candidates after stale RPC receipts --- FIXES.md | 9 +- ethreceipts/ethreceipts.go | 8 +- ethreceipts/stale_receipt_test.go | 170 ++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 ethreceipts/stale_receipt_test.go diff --git a/FIXES.md b/FIXES.md index 51ec9c82..af19cf20 100644 --- a/FIXES.md +++ b/FIXES.md @@ -1,9 +1,10 @@ # Receipt listener fixes -These changes correct three receipt-listener findings and the additional partial-candidate issue found during independent review. Receipt work remains owned by its exact filter registration, transaction, block hash and generation. Public filter values and custom matching remain intact. +These changes correct receipt-listener findings and issues found during review. Receipt work remains owned by its exact filter registration, transaction, block hash and generation. Public filter values and custom matching remain intact. - **Successful retry followed by cancellation.** A receipt fetch could succeed, lose its pending entry, then miss publication because the attempt expired while waiting for delivery. Pending removal now commits with publication under the delivery lock. A canceled attempt sends nothing and retains valid work for a prompt retry using the completed cache entry. RemoveFilter, ClearFilters and rollback remain terminal for the affected owner or candidate. - **Partial callback completion.** A QueryOnChain callback may initially supply only TxHash. If its successful completion is retained after cancellation, the pending item now keeps the learned block/generation separately from its immutable queue identity. Rollback matches that completed candidate and releases its never-delivered LimitOne selection. Later retries validate the learned generation, so same-hash re-adoption cannot revive it. A fresh canonical transaction can be selected; an already-mined LimitOne transaction still keeps its selection through rollback and re-mining. +- **Stale RPC receipt after re-mining.** A provider can return the removed block A's receipt while the listener processes the same transaction in canonical block B. The invalid response is still rejected and never cached. Its rejection is retryable only when B's expected hash/generation remains current under the receipt lock, keeping B in the existing bounded pending queue. Obsolete generations and hash queries without a current expected block remain terminal. Mock-provider regressions cover initial processing and pending retries, recovery on an unrelated block, mined/final delivery, and obsolete work preserving a newer cache entry. - **Same-hash re-adoption after retention eviction.** A delayed live Added event can outlast the monitor's retained chain. Receipt acceptance now uses positive evidence for that exact monitor incarnation, preserving valid adoption after deep eviction. Removed incarnations, cached registration snapshots and earlier RPC generations cannot reopen stale work. Missing hash lookup or height alone is insufficient; untracked/manual events require a positive retained canonical lookup. - **Individual custom-value removal.** Comparable values and pointers retain Go equality. Otherwise unresolved non-comparable values of the same concrete type use a stable nonnil Exhausted channel as base identity. Shared-base values are aliases: removal chooses the first matching active registration, otherwise its first queued finality owner. Distinct bases or pointer identities provide independent removal, including through Filters() and Receipt.Filter copies. Nil/unstable signals cannot identify arbitrary callback value copies; register those as pointers for individual cancellation. Callback code pointers, unsafe identity and public filter IDs are not used as owner identity. @@ -19,7 +20,7 @@ Keyed construction and the existing JSON schema remain supported; serialized blo Final validation on actual synchronized sources: -- All 39 permanent receipt regression groups passed three repetitions with the race detector: 156.481s. Eight new groups contain 30 cases; failing pre-fix evidence was captured for the confirmed defects. -- Full local receipt suite passed with the race detector: 252.573s, including existing integration tests. No overlay, chain reset or race warning. +- The two stale-RPC regression groups (four cases) passed five repetitions with the race detector: 1.656s. The same permanent tests fail against the pre-fix production source: current candidates are discarded and never retried, while obsolete-work controls pass. +- Full local receipt suite passed with the race detector: 254.839s, including all 41 permanent regression groups and existing integration tests. No overlay, chain reset or race warning. - Parent monitor suite passed with the race detector: 73.007s. Only the external live-mainnet fee-history test was excluded; no external live RPC was used. -- go vet ./ethreceipts ./ethmonitor, go build ./..., Go formatting and diff checks passed. The requested documentation addition changes no runtime or test source. +- go vet ./ethreceipts, go build ./..., Go formatting and diff checks passed for the stale-RPC follow-up. The unchanged monitor package retains its earlier passing vet and race checks. diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index 19fa0515..d33febfe 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -602,13 +602,19 @@ start: } l.receiptMu.Lock() valid := l.validFetchedBlock(receipt.BlockHash, expected, started) + // A stale provider response does not invalidate a current monitored candidate. + retryable := !valid && expected.hash != (common.Hash{}) && l.currentBlock(expected.hash, expected.generation) if valid { l.pastReceipts.Set(ctx, txnHashHex, receipt) l.notFoundTxnHashes.Delete(ctx, txnHashHex) } l.receiptMu.Unlock() if !valid { - errCh <- ethereum.NotFound + if retryable { + errCh <- fmt.Errorf("ethreceipts: stale receipt for txn %s in block %s, expected current block %s", txnHash, receipt.BlockHash, expected.hash) + } else { + errCh <- ethereum.NotFound + } return nil } resultCh <- receipt diff --git a/ethreceipts/stale_receipt_test.go b/ethreceipts/stale_receipt_test.go new file mode 100644 index 00000000..9fea2901 --- /dev/null +++ b/ethreceipts/stale_receipt_test.go @@ -0,0 +1,170 @@ +package ethreceipts + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestReceiptsStaleRPCRecoversCurrentCandidate(t *testing.T) { + for _, pendingFirst := range []bool{false, true} { + name := "initial_response" + if pendingFirst { + name = "pending_retry" + } + t.Run(name, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6000) + a, b := hardeningBlock(100, tx), hardeningBlock(102, tx) + var current atomic.Pointer[types.Receipt] + current.Store(hardeningReceipt(a, tx)) + var providerError atomic.Bool + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if providerError.Load() { + return nil, errors.New("temporary provider failure") + } + return current.Load(), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + ctx := context.Background() + hardeningProcess(t, l, s, a) + mined := hardeningRead(t, s) + if mined.BlockHash() != a.Hash() || mined.Reorged || mined.Final { + t.Fatalf("wrong initial mined receipt: %+v", mined) + } + removed := *a + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.BlockHash() != a.Hash() { + t.Fatalf("wrong rollback notification: %+v", rollback) + } + providerError.Store(pendingFirst) + hardeningProcess(t, l, s, b) + providerError.Store(false) + pendingCount := func() int { + s.retryMu.Lock() + defer s.retryMu.Unlock() + return len(s.pendingReceipts) + } + makeDue := func() { + s.retryMu.Lock() + defer s.retryMu.Unlock() + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Now().Add(-time.Second) + } + } + if pendingFirst { + if pendingCount() != 1 { + t.Fatal("provider failure did not queue B") + } + makeDue() + hardeningProcess(t, l, s, hardeningBlock(103)) + } + if pendingCount() != 1 { + t.Errorf("stale A response discarded current B candidate: pending=%d", pendingCount()) + } + if _, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); found { + t.Fatal("stale A receipt was cached") + } + hardeningNoReceipt(t, s) + + // The provider recovers; only an unrelated later block triggers retry. + canonical := hardeningReceipt(b, tx) + current.Store(canonical) + makeDue() + callsBefore := calls.Load() + hardeningProcess(t, l, s, hardeningBlock(104)) + if calls.Load() <= callsBefore { + t.Fatal("provider recovered but B was never retried") + } + mined = hardeningRead(t, s) + if mined.BlockHash() != b.Hash() || mined.TransactionHash() != tx.Hash() || mined.Reorged || mined.Final || mined.Filter != q { + t.Fatalf("wrong recovered mined receipt: %+v", mined) + } + hardeningProcess(t, l, s, hardeningBlock(105)) + final := hardeningRead(t, s) + if final.BlockHash() != b.Hash() || final.TransactionHash() != tx.Hash() || final.Reorged || !final.Final || final.Filter != q { + t.Fatalf("wrong recovered final receipt: %+v", final) + } + if pendingCount() != 0 { + t.Error("delivered B remained pending") + } + if cached, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); !found || cached != canonical { + t.Error("canonical B receipt was not cached") + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestReceiptsStaleRPCKeepsObsoleteWorkTerminal(t *testing.T) { + t.Run("expected_generation_removed_during_fetch", func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6001) + b := hardeningBlock(100, tx) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + close(entered) + select { + case <-release: + return hardeningReceipt(b, tx), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + }} + l := hardeningListener(t, p, hardeningOptions(), b) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := l.fetchTransactionReceipt(ctx, tx.Hash(), true, blockRef{b.Hash(), 0}) + done <- err + }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("RPC did not start") + } + l.invalidateBlock(ctx, b) + l.acceptBlock(b) + if !l.isCurrentBlock(b.Hash(), 1) { + t.Fatal("same-hash re-adoption did not advance the generation") + } + fresh := hardeningReceipt(b, tx) + l.pastReceipts.Set(ctx, tx.Hash().Hex(), fresh) + close(release) + if err := <-done; !errors.Is(err, ethereum.NotFound) { + t.Fatalf("obsolete expected generation became retryable: %v", err) + } + if cached, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); !found || cached != fresh { + t.Error("obsolete completion replaced the current cache entry") + } + }) + t.Run("hash_query_without_current_candidate", func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6002) + a := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(a, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), a) + ctx := context.Background() + l.invalidateBlock(ctx, a) + if _, err := l.fetchTransactionReceipt(ctx, tx.Hash(), true); !errors.Is(err, ethereum.NotFound) { + t.Fatalf("unscoped orphan query became retryable: %v", err) + } + if _, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); found { + t.Error("unscoped orphan query cached stale data") + } + }) +} From b33c95faeee305c80fedf563f60b8e71f187b6a1 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 08:52:41 -0400 Subject: [PATCH 05/11] refactor(ethreceipts): pass block processing contexts explicitly --- FIXES.md | 26 --------------------- ethreceipts/codegenie_regression_test.go | 4 ++-- ethreceipts/ethreceipts.go | 29 +++++++++--------------- ethreceipts/hardening_test.go | 19 ++++++++-------- ethreceipts/ownership_regression_test.go | 4 ++-- 5 files changed, 24 insertions(+), 58 deletions(-) delete mode 100644 FIXES.md diff --git a/FIXES.md b/FIXES.md deleted file mode 100644 index af19cf20..00000000 --- a/FIXES.md +++ /dev/null @@ -1,26 +0,0 @@ -# Receipt listener fixes - -These changes correct receipt-listener findings and issues found during review. Receipt work remains owned by its exact filter registration, transaction, block hash and generation. Public filter values and custom matching remain intact. - -- **Successful retry followed by cancellation.** A receipt fetch could succeed, lose its pending entry, then miss publication because the attempt expired while waiting for delivery. Pending removal now commits with publication under the delivery lock. A canceled attempt sends nothing and retains valid work for a prompt retry using the completed cache entry. RemoveFilter, ClearFilters and rollback remain terminal for the affected owner or candidate. -- **Partial callback completion.** A QueryOnChain callback may initially supply only TxHash. If its successful completion is retained after cancellation, the pending item now keeps the learned block/generation separately from its immutable queue identity. Rollback matches that completed candidate and releases its never-delivered LimitOne selection. Later retries validate the learned generation, so same-hash re-adoption cannot revive it. A fresh canonical transaction can be selected; an already-mined LimitOne transaction still keeps its selection through rollback and re-mining. -- **Stale RPC receipt after re-mining.** A provider can return the removed block A's receipt while the listener processes the same transaction in canonical block B. The invalid response is still rejected and never cached. Its rejection is retryable only when B's expected hash/generation remains current under the receipt lock, keeping B in the existing bounded pending queue. Obsolete generations and hash queries without a current expected block remain terminal. Mock-provider regressions cover initial processing and pending retries, recovery on an unrelated block, mined/final delivery, and obsolete work preserving a newer cache entry. -- **Same-hash re-adoption after retention eviction.** A delayed live Added event can outlast the monitor's retained chain. Receipt acceptance now uses positive evidence for that exact monitor incarnation, preserving valid adoption after deep eviction. Removed incarnations, cached registration snapshots and earlier RPC generations cannot reopen stale work. Missing hash lookup or height alone is insufficient; untracked/manual events require a positive retained canonical lookup. -- **Individual custom-value removal.** Comparable values and pointers retain Go equality. Otherwise unresolved non-comparable values of the same concrete type use a stable nonnil Exhausted channel as base identity. Shared-base values are aliases: removal chooses the first matching active registration, otherwise its first queued finality owner. Distinct bases or pointer identities provide independent removal, including through Filters() and Receipt.Filter copies. Nil/unstable signals cannot identify arbitrary callback value copies; register those as pointers for individual cancellation. Callback code pointers, unsafe identity and public filter IDs are not used as owner identity. - -The monitor support belongs to [PR219](https://github.com/0xsequence/ethkit/pull/219); receipt changes remain in [PR220](https://github.com/0xsequence/ethkit/pull/220), based directly on parallel-monitor. The parent confirms cached parent mismatches even with local prefetch disabled and exposes Block.CanonicalState: acceptance is owned per incarnation, actual removal invalidates it, and retention eviction preserves it. This evidence covers known monitor removals; reorgs beyond retained known history remain the monitor's existing limit. - -The parent's private Block state requires external positional literals to migrate to keyed literals, for example: - -```go -ethmonitor.Block{Block: block, Event: ethmonitor.Added, Logs: logs, OK: true} -``` - -Keyed construction and the existing JSON schema remain supported; serialized blocks carry no runtime incarnation evidence. No parent implementation files are included in the receipt correction delta. - -Final validation on actual synchronized sources: - -- The two stale-RPC regression groups (four cases) passed five repetitions with the race detector: 1.656s. The same permanent tests fail against the pre-fix production source: current candidates are discarded and never retried, while obsolete-work controls pass. -- Full local receipt suite passed with the race detector: 254.839s, including all 41 permanent regression groups and existing integration tests. No overlay, chain reset or race warning. -- Parent monitor suite passed with the race detector: 73.007s. Only the external live-mainnet fee-history test was excluded; no external live RPC was used. -- go vet ./ethreceipts, go build ./..., Go formatting and diff checks passed for the stale-RPC follow-up. The unchanged monitor package retains its earlier passing vet and race checks. diff --git a/ethreceipts/codegenie_regression_test.go b/ethreceipts/codegenie_regression_test.go index d69f7083..15d80e6f 100644 --- a/ethreceipts/codegenie_regression_test.go +++ b/ethreceipts/codegenie_regression_test.go @@ -573,9 +573,9 @@ func TestReceiptsFixReadoptionAfterRetention(t *testing.T) { t.Helper() var err error if cached { - _, err = l.processCachedBlocksContext(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + _, err = l.processCachedBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) } else { - _, err = l.processBlocksContext(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + _, err = l.processBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) } if err != nil { t.Fatal(err) diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index d33febfe..449fb549 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -98,13 +98,12 @@ type ReceiptsListener struct { registerFiltersCh chan registerFilters filterSem chan struct{} - ctx context.Context ctxStop context.CancelFunc running int32 - lifecycleMu sync.Mutex + lifecycleMu sync.Mutex // protects Run/Stop cancellation handle and running transitions receiptMu sync.Mutex blockStates map[common.Hash]blockState - reorgRevision uint64 + reorgRevision uint64 // removal sequence used to reject hash queries started before invalidation mu sync.RWMutex } @@ -204,7 +203,7 @@ func (l *ReceiptsListener) Run(ctx context.Context) error { return fmt.Errorf("ethreceipts: already running") } runCtx, cancel := context.WithCancel(ctx) - l.ctx, l.ctxStop = runCtx, cancel + l.ctxStop = cancel atomic.StoreInt32(&l.running, 1) l.lifecycleMu.Unlock() defer func() { @@ -702,7 +701,7 @@ func (l *ReceiptsListener) listener(runCtx context.Context) error { // Search our local blocks cache from monitor retention list, and notify subscriber // of any matches found by publishing receipts. - matchedList, err := l.processCachedBlocksContext(ctx, blocks, []*subscriber{reg.subscriber}, [][]Filterer{filters}) + matchedList, err := l.processCachedBlocks(ctx, blocks, []*subscriber{reg.subscriber}, [][]Filterer{filters}) if err != nil { l.log.Warn(fmt.Sprintf("ethreceipts: failed to process blocks during new filter registration: %v", err)) } @@ -776,7 +775,7 @@ func (l *ReceiptsListener) listener(runCtx context.Context) error { } // Match blocks against subscribers[i] X filters[i][..] - matchedList, err := l.processBlocksContext(ctx, blocks, subscribers, filters) + matchedList, err := l.processBlocks(ctx, blocks, subscribers, filters) if err != nil { l.log.Warn(fmt.Sprintf("ethreceipts: failed to process blocks: %v", err)) } @@ -838,20 +837,12 @@ func (l *ReceiptsListener) listener(runCtx context.Context) error { // processBlocks attempts to match blocks against subscriber[i] X filterers[i].. list of filters. There is // a corresponding list of filters[i] for each subscriber[i]. -func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { - l.lifecycleMu.Lock() - ctx := l.ctx - l.lifecycleMu.Unlock() - if ctx == nil { - ctx = context.Background() - } - return l.processCachedBlocksContext(ctx, blocks, subscribers, filterers) -} - -func (l *ReceiptsListener) processBlocksContext(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { +func (l *ReceiptsListener) processBlocks(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { return l.processBlockEvents(ctx, blocks, subscribers, filterers, true) } -func (l *ReceiptsListener) processCachedBlocksContext(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { + +// Cached snapshots may match receipts, but cannot authorize canonical re-adoption. +func (l *ReceiptsListener) processCachedBlocks(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { return l.processBlockEvents(ctx, blocks, subscribers, filterers, false) } func (l *ReceiptsListener) processBlockEvents(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer, canonicalEvents bool) ([][]bool, error) { @@ -1135,6 +1126,8 @@ func (l *ReceiptsListener) latestBlockNum(ctx context.Context) *big.Int { } func getChainID(ctx context.Context, provider ethrpc.Interface) (*big.Int, error) { + // breaker v0.2.0 sleeps without checking cancellation between attempts. + // Preserve its retry policy here while making startup stoppable during backoff. delay := time.Second for attempt := 0; ; attempt++ { if ctx.Err() != nil { diff --git a/ethreceipts/hardening_test.go b/ethreceipts/hardening_test.go index 74a34570..25262ff7 100644 --- a/ethreceipts/hardening_test.go +++ b/ethreceipts/hardening_test.go @@ -82,7 +82,6 @@ func hardeningListener(t *testing.T, p *hardeningProvider, opts Options, blocks if err != nil { t.Fatal(err) } - l.ctx = context.Background() l.br = breaker.New(log, 0, 1, 0) return l } @@ -133,7 +132,7 @@ func hardeningNoReceipt(t *testing.T, s Subscription) { } func hardeningProcess(t *testing.T, l *ReceiptsListener, s *subscriber, blocks ...*ethmonitor.Block) { t.Helper() - if _, err := l.processBlocks(blocks, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { t.Fatal(err) } } @@ -365,7 +364,7 @@ func TestHardeningLimitOneSnapshots(t *testing.T) { s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true)).(*subscriber) defer s.Unsubscribe() fs := s.Filters() - if _, err := l.processBlocks(blocks, []*subscriber{s}, [][]Filterer{fs}); err != nil { + if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{fs}); err != nil { t.Fatal(err) } r := hardeningRead(t, s) @@ -529,7 +528,7 @@ func TestHardeningFinalityWaitCancellation(t *testing.T) { func TestHardeningConcurrentRunStop(t *testing.T) { for i := 0; i < 5; i++ { l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - l.ctx, l.ctxStop = context.WithCancel(context.Background()) + _, l.ctxStop = context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background()) cancel() var wg sync.WaitGroup @@ -575,7 +574,7 @@ func TestHardeningInflightRollbackRemining(t *testing.T) { snapshot := s.Filters() oldDone := make(chan error, 1) go func() { - _, err := l.processBlocks(ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}) + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}) oldDone <- err }() <-entered @@ -610,7 +609,7 @@ func TestHardeningInflightRollbackRemining(t *testing.T) { t.Fatal("old fetch did not finish") } // A registration snapshot retained before rollback must also remain invalid. - if _, err := l.processBlocks(ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}); err != nil { + if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}); err != nil { t.Fatal(err) } hardeningNoReceipt(t, s) @@ -658,11 +657,11 @@ func TestHardeningConcurrentLimitOne(t *testing.T) { fs := s.Filters() done := make(chan error, 1) go func() { - _, err := l.processBlocks(ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{fs}) + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{fs}) done <- err }() <-entered - if _, err := l.processBlocks(ethmonitor.Blocks{b2}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b2}, []*subscriber{s}, [][]Filterer{fs}); err != nil { t.Fatal(err) } close(release) @@ -926,7 +925,7 @@ func TestHardeningSameHashReadoptionRejectsOldWork(t *testing.T) { fs := s.Filters() oldDone := make(chan error, 1) go func() { - _, err := l.processBlocks(ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}) + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}) oldDone <- err }() <-entered @@ -944,7 +943,7 @@ func TestHardeningSameHashReadoptionRejectsOldWork(t *testing.T) { t.Errorf("stale snapshot cleared removal marker: %d fetches", calls.Load()) } // The live block-processing entry point receives the canonical Added event. - if _, err := l.processBlocksContext(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}); err != nil { t.Fatal(err) } select { diff --git a/ethreceipts/ownership_regression_test.go b/ethreceipts/ownership_regression_test.go index 51059845..ecff1b87 100644 --- a/ethreceipts/ownership_regression_test.go +++ b/ethreceipts/ownership_regression_test.go @@ -185,7 +185,7 @@ func TestOwnershipReaddRejectsOldWorker(t *testing.T) { defer s.Unsubscribe() done := make(chan error, 1) go func() { - _, err := l.processBlocks(ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{s.Filters()}) + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{s.Filters()}) done <- err }() <-entered @@ -348,7 +348,7 @@ func TestOwnershipStaleFetchPreservesCache(t *testing.T) { hardeningProcess(t, l, s, &removed) hardeningRead(t, s) if sameHash { - if _, err := l.processBlocksContext(context.Background(), ethmonitor.Blocks{canonical}, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{canonical}, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { t.Fatal(err) } } else { From b0a6de8e5522640ca4bbcfd06c8a5c0111838645 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 09:41:52 -0400 Subject: [PATCH 06/11] refactor(ethreceipts): restore breaker retries with v0.3.2 --- ethreceipts/ethreceipts.go | 36 +++++++------------ ethreceipts/hardening_test.go | 35 ++++++++++++++++++ ..._regression_test.go => regression_test.go} | 18 ++++++---- go.mod | 2 +- go.sum | 5 ++- 5 files changed, 61 insertions(+), 35 deletions(-) rename ethreceipts/{codegenie_regression_test.go => regression_test.go} (98%) diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index 449fb549..41ed5775 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -1126,37 +1126,25 @@ func (l *ReceiptsListener) latestBlockNum(ctx context.Context) *big.Int { } func getChainID(ctx context.Context, provider ethrpc.Interface) (*big.Int, error) { - // breaker v0.2.0 sleeps without checking cancellation between attempts. - // Preserve its retry policy here while making startup stoppable during backoff. - delay := time.Second - for attempt := 0; ; attempt++ { - if ctx.Err() != nil { - return nil, ctx.Err() - } + var chainID *big.Int + err := breaker.Do(ctx, func() error { requestCtx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + id, err := provider.ChainID(requestCtx) - cancel() - if err == nil { - return id, nil + if err != nil { + return err } + chainID = id + return nil + }, nil, time.Second, 2, 10) + if err != nil { if ctx.Err() != nil { return nil, ctx.Err() } - if errors.Is(err, breaker.ErrFatal) { - return nil, err - } - if attempt >= 10 { - return nil, superr.New(breaker.ErrHitMaxRetries, err) - } - timer := time.NewTimer(delay) - select { - case <-timer.C: - case <-ctx.Done(): - timer.Stop() - return nil, ctx.Err() - } - delay *= 2 + return nil, err } + return chainID, nil } func collectOk[T any](in []T, oks []bool, okCond bool) []T { diff --git a/ethreceipts/hardening_test.go b/ethreceipts/hardening_test.go index 25262ff7..e7587d8b 100644 --- a/ethreceipts/hardening_test.go +++ b/ethreceipts/hardening_test.go @@ -441,6 +441,41 @@ func TestHardeningStopCancelsStartup(t *testing.T) { t.Error("Stop did not cancel startup ChainID request") } } + +func TestHardeningStopCancelsStartupBackoff(t *testing.T) { + entered := make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{chainID: func(context.Context) (*big.Int, error) { + if calls.Add(1) == 1 { + close(entered) + return nil, errors.New("transient ChainID failure") + } + return nil, breaker.ErrFatal + }} + l := hardeningListener(t, p, DefaultOptions) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + <-entered + // Allow the failed attempt to enter its one-second backoff. + time.Sleep(100 * time.Millisecond) + l.Stop() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("startup cancellation error: %v", err) + } + case <-time.After(500 * time.Millisecond): + cancel() + <-done + t.Error("Stop waited for startup ChainID backoff") + } + if calls.Load() != 1 { + t.Errorf("ChainID retried after cancellation: %d calls", calls.Load()) + } +} + func TestHardeningMonitorStopCancelsRegistration(t *testing.T) { l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) entered := make(chan struct{}) diff --git a/ethreceipts/codegenie_regression_test.go b/ethreceipts/regression_test.go similarity index 98% rename from ethreceipts/codegenie_regression_test.go rename to ethreceipts/regression_test.go index 15d80e6f..024d32c5 100644 --- a/ethreceipts/codegenie_regression_test.go +++ b/ethreceipts/regression_test.go @@ -18,13 +18,17 @@ import ( "github.com/0xsequence/ethkit/go-ethereum/core/types" ) -func receiptsFixCollect(s Subscription) []Receipt { +func receiptsFixCollect(t *testing.T, s Subscription) []Receipt { + t.Helper() var receipts []Receipt timer := time.NewTimer(40 * time.Millisecond) defer timer.Stop() for { select { - case r := <-s.TransactionReceipt(): + case r, ok := <-s.TransactionReceipt(): + if !ok { + t.Fatal("receipt channel closed unexpectedly") + } receipts = append(receipts, r) case <-timer.C: return receipts @@ -326,14 +330,14 @@ func TestReceiptsFixCustomValueRemoval(t *testing.T) { t.Error("public removal did not isolate selected registration/finality") } process(hardeningReceipt(b2, tx2)) - late := receiptsFixCollect(s) + late := receiptsFixCollect(t, s) if len(late) != 1 || late[0].owner != second || late[0].TransactionHash() != tx2.Hash() || late[0].Final || late[0].Reorged { t.Error("selected cancellation changed later surviving delivery") } if err := s.finalizeReceipts(big.NewInt(105)); err != nil { t.Fatal(err) } - finals := receiptsFixCollect(s) + finals := receiptsFixCollect(t, s) seen := make(map[common.Hash]bool) for _, r := range finals { if r.owner != second || !r.Final || r.Reorged || seen[r.TransactionHash()] { @@ -380,7 +384,7 @@ func TestReceiptsFixSharedBaseAliases(t *testing.T) { if err := s.finalizeReceipts(big.NewInt(103)); err != nil { t.Fatal(err) } - finals := receiptsFixCollect(s) + finals := receiptsFixCollect(t, s) if len(finals) != 1 || finals[0].owner != owners[2] || !finals[0].Final || finals[0].TransactionHash() != tx.Hash() { t.Error("shared aliases canceled an unrelated queued final") } @@ -422,7 +426,7 @@ func TestReceiptsFixComparableSharedBaseIsolation(t *testing.T) { if err := s.finalizeReceipts(big.NewInt(103)); err != nil { t.Fatal(err) } - finals := receiptsFixCollect(s) + finals := receiptsFixCollect(t, s) if len(finals) != 1 || finals[0].owner != owners[0] || !finals[0].Final { t.Fatal("comparable cancellation lost surviving owner") } @@ -671,7 +675,7 @@ func TestReceiptsFixReadoptionAfterRetention(t *testing.T) { t.Error("later-removed Added reopened canonical marker") } process(ethmonitor.Blocks{added, removed}, false) - for _, r := range receiptsFixCollect(s) { + for _, r := range receiptsFixCollect(t, s) { if !r.Reorged || r.Final { t.Error("same-batch removal published stale mined/final receipt") } diff --git a/go.mod b/go.mod index a8a1b939..7f279041 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/google/gofuzz v1.2.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 - github.com/goware/breaker v0.2.0 + github.com/goware/breaker v0.3.2 github.com/goware/cachestore-mem v0.2.2 github.com/goware/cachestore-redis v0.2.1 github.com/goware/cachestore2 v0.12.3 diff --git a/go.sum b/go.sum index 4eeddb69..258c1801 100644 --- a/go.sum +++ b/go.sum @@ -96,8 +96,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/goware/breaker v0.2.0 h1:MJOaFjHwQ7h5/nCtjAUuOHhJeLCFF2GeUnXsBDMXq1k= -github.com/goware/breaker v0.2.0/go.mod h1:ijCEfXAa0j6w7IoHA4v6Sox2W6U9HUbI/t+5x0zGaug= +github.com/goware/breaker v0.3.2 h1:rVT/0UqQ70b3PN78Gj9dtZRDFJ1ZK8sm/fA8FkpOyCc= +github.com/goware/breaker v0.3.2/go.mod h1:CfgulI2zAaSLSnIeT8EN3mOJeWKSUbC26mGaveebT3E= github.com/goware/cachestore-mem v0.2.2 h1:toE6/1QMQQcQLJQpTIiTDAIHWLN4zvihoqZHq41cPns= github.com/goware/cachestore-mem v0.2.2/go.mod h1:KpXr+yVajbeN0s+CX/08hdSww2WhjtAL1pmNy69S03w= github.com/goware/cachestore-redis v0.2.1 h1:bMdkzGuy6Dsybq2chv3kRwu8UGoJz3aQBO9Cu41vEqQ= @@ -160,7 +160,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= From 2465f4a26f3a7524099450e671dc81cfa43391b3 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 10:25:31 -0400 Subject: [PATCH 07/11] test(ethreceipts): organize regressions by behavior --- ethreceipts/finalizer_test.go | 132 +++ ethreceipts/hardening_test.go | 1030 ---------------------- ethreceipts/helpers_test.go | 224 +++++ ethreceipts/lifecycle_test.go | 228 +++++ ethreceipts/ownership_regression_test.go | 653 +++++++------- ethreceipts/pending_receipts_test.go | 357 ++++++++ ethreceipts/receipt_fetch_test.go | 150 ++++ ethreceipts/regression_test.go | 713 --------------- ethreceipts/reorg_test.go | 773 ++++++++++++++++ 9 files changed, 2166 insertions(+), 2094 deletions(-) create mode 100644 ethreceipts/finalizer_test.go delete mode 100644 ethreceipts/hardening_test.go create mode 100644 ethreceipts/helpers_test.go create mode 100644 ethreceipts/lifecycle_test.go create mode 100644 ethreceipts/pending_receipts_test.go create mode 100644 ethreceipts/receipt_fetch_test.go delete mode 100644 ethreceipts/regression_test.go create mode 100644 ethreceipts/reorg_test.go diff --git a/ethreceipts/finalizer_test.go b/ethreceipts/finalizer_test.go new file mode 100644 index 00000000..407abd67 --- /dev/null +++ b/ethreceipts/finalizer_test.go @@ -0,0 +1,132 @@ +package ethreceipts + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningSubscribeBeforeRunFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 2) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, DefaultOptions, b, hardeningBlock(101)) + s := l.Subscribe(FilterTxnHash(tx.Hash())).(*subscriber) + defer s.Unsubscribe() + hardeningStart(t, l) + if r := hardeningRead(t, s); r.Final { + t.Fatal("receipt final before network threshold") + } + processed := make(chan struct{}) + s.AddFilter(FilterLogs(func([]*types.Log) bool { return false }).SearchCache(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { close(processed); return nil, nil })) + select { + case <-processed: + case <-time.After(time.Second): + t.Fatal("registration did not complete") + } + hardeningNoReceipt(t, s) + if len(s.Filters()) != 2 { + t.Fatal("premature finality removed transaction filter") + } +} + +func TestHardeningFinalizerOrderAndOwners(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 4) + tx2, _, _ := hardeningTxn(t, 5) + newer, older := hardeningBlock(101, tx1), hardeningBlock(100, tx2) + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + return hardeningReceipt(newer, tx1), nil + } + return hardeningReceipt(older, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) + q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + for _, r := range []*types.Receipt{hardeningReceipt(newer, tx1), hardeningReceipt(older, tx2)} { + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: r}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + hardeningRead(t, s) + } + for _, tc := range []struct { + head int64 + tx *types.Transaction + b *ethmonitor.Block + }{{103, tx2, older}, {104, tx1, newer}} { + if err := s.finalizeReceipts(big.NewInt(tc.head)); err != nil { + t.Fatal(err) + } + owners := map[Filterer]bool{} + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.TransactionHash() != tc.tx.Hash() || r.BlockHash() != tc.b.Hash() { + t.Errorf("wrong final at %d: %+v", tc.head, r) + } + owners[r.Filter] = true + case <-time.After(100 * time.Millisecond): + t.Errorf("missing owner final at %d", tc.head) + } + } + if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { + t.Errorf("finality lost distinct owners with same public ID at %d", tc.head) + } + } + hardeningNoReceipt(t, s) +} + +func TestHardeningExhaustedOwnerKeepsQueuedFinal(t *testing.T) { + tx, _, _ := hardeningTxn(t, 17) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).MaxWait(1) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + // Exhaustion removes matching and closes its signal, but documented pending + // finality remains owned. Exercise that state without relying on a live chain. + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + q.(*filter).closeExhausted() + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Filter != q || final.BlockHash() != b.Hash() { + t.Fatal("exhaustion canceled a queued final") + } + hardeningNoReceipt(t, s) +} + +func TestHardeningClearCancelsExhaustedFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 18) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + q.(*filter).closeExhausted() + s.ClearFilters() + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} diff --git a/ethreceipts/hardening_test.go b/ethreceipts/hardening_test.go deleted file mode 100644 index e7587d8b..00000000 --- a/ethreceipts/hardening_test.go +++ /dev/null @@ -1,1030 +0,0 @@ -package ethreceipts - -import ( - "context" - "errors" - "io" - "log/slog" - "math/big" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/0xsequence/ethkit/ethmonitor" - "github.com/0xsequence/ethkit/ethrpc" - "github.com/0xsequence/ethkit/go-ethereum" - "github.com/0xsequence/ethkit/go-ethereum/common" - "github.com/0xsequence/ethkit/go-ethereum/core/types" - "github.com/0xsequence/ethkit/go-ethereum/crypto" - "github.com/goware/breaker" -) - -// Only the RPC methods used by the listener are provided. No external node is needed. -type hardeningProvider struct { - ethrpc.RawInterface - chainID func(context.Context) (*big.Int, error) - receipt func(context.Context, common.Hash) (*types.Receipt, error) -} - -func (p *hardeningProvider) ChainID(ctx context.Context) (*big.Int, error) { - if p.chainID != nil { - return p.chainID(ctx) - } - return big.NewInt(1), nil -} -func (p *hardeningProvider) TransactionReceipt(ctx context.Context, h common.Hash) (*types.Receipt, error) { - if p.receipt != nil { - return p.receipt(ctx, h) - } - return nil, ethereum.NotFound -} -func hardeningBlock(num int64, txns ...*types.Transaction) *ethmonitor.Block { - b := types.NewBlockWithHeader(&types.Header{Number: big.NewInt(num), BlockHash: common.BigToHash(big.NewInt(num)), ParentHash: common.BigToHash(big.NewInt(num - 1)), GasLimit: 30_000_000, Time: uint64(num)}) - return ðmonitor.Block{Block: b.WithBody(types.Body{Transactions: txns}), Event: ethmonitor.Added, OK: true} -} -func hardeningTxn(t *testing.T, nonce uint64) (*types.Transaction, common.Address, common.Address) { - t.Helper() - key, err := crypto.HexToECDSA("1111111111111111111111111111111111111111111111111111111111111111") - if err != nil { - t.Fatal(err) - } - from := crypto.PubkeyToAddress(key.PublicKey) - to := common.HexToAddress("0x2222222222222222222222222222222222222222") - tx, err := types.SignTx(types.NewTransaction(nonce, to, big.NewInt(1), 21000, big.NewInt(1), nil), types.NewEIP155Signer(big.NewInt(1)), key) - if err != nil { - t.Fatal(err) - } - return tx, from, to -} -func hardeningReceipt(b *ethmonitor.Block, tx *types.Transaction) *types.Receipt { - return &types.Receipt{TxHash: tx.Hash(), BlockHash: b.Hash(), BlockNumber: b.Number(), Status: 1} -} -func hardeningListener(t *testing.T, p *hardeningProvider, opts Options, blocks ...*ethmonitor.Block) *ReceiptsListener { - t.Helper() - log := slog.New(slog.NewTextHandler(io.Discard, nil)) - mo := ethmonitor.DefaultOptions - mo.Logger = log - mo.WithLogs = true - mo.BlockRetentionLimit = 100 - mo.Bootstrap = true - m, err := ethmonitor.NewMonitor(p, mo) - if err != nil { - t.Fatal(err) - } - if len(blocks) == 0 { - blocks = []*ethmonitor.Block{hardeningBlock(100)} - } - if err = m.Chain().BootstrapFromBlocks(blocks); err != nil { - t.Fatal(err) - } - l, err := NewReceiptsListener(log, p, m, opts) - if err != nil { - t.Fatal(err) - } - l.br = breaker.New(log, 0, 1, 0) - return l -} -func hardeningStart(t *testing.T, l *ReceiptsListener) { - t.Helper() - done := make(chan error, 1) - go func() { done <- l.Run(context.Background()) }() - deadline := time.After(time.Second) - for l.monitor.NumSubscribers() != 1 { - select { - case err := <-done: - t.Fatalf("Run exited before subscription: %v", err) - case <-deadline: - t.Fatal("Run did not subscribe to monitor") - default: - time.Sleep(time.Millisecond) - } - } - t.Cleanup(func() { - l.Stop() - select { - case err := <-done: - if err != nil { - t.Errorf("Run: %v", err) - } - case <-time.After(time.Second): - t.Error("Run did not stop") - } - }) -} -func hardeningRead(t *testing.T, s Subscription) Receipt { - t.Helper() - select { - case r := <-s.TransactionReceipt(): - return r - case <-time.After(time.Second): - t.Fatal("missing receipt") - return Receipt{} - } -} -func hardeningNoReceipt(t *testing.T, s Subscription) { - t.Helper() - select { - case r := <-s.TransactionReceipt(): - t.Errorf("unexpected receipt: txn=%s block=%s final=%v reorged=%v owner=%p", r.TransactionHash(), r.BlockHash(), r.Final, r.Reorged, r.Filter) - case <-time.After(30 * time.Millisecond): - } -} -func hardeningProcess(t *testing.T, l *ReceiptsListener, s *subscriber, blocks ...*ethmonitor.Block) { - t.Helper() - if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { - t.Fatal(err) - } -} -func hardeningOptions() Options { opts := DefaultOptions; opts.NumBlocksToFinality = 2; return opts } - -func TestHardeningRollbackRemining(t *testing.T) { - tx, _, _ := hardeningTxn(t, 1) - old, newBlock := hardeningBlock(100, tx), hardeningBlock(102, tx) - var current atomic.Pointer[types.Receipt] - current.Store(hardeningReceipt(old, tx)) - var calls atomic.Int32 - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { calls.Add(1); return current.Load(), nil }} - l := hardeningListener(t, p, hardeningOptions()) - q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - hardeningProcess(t, l, s, old) - first := hardeningRead(t, s) - if first.Final || first.Reorged || first.BlockHash() != old.Hash() { - t.Fatalf("initial receipt: %+v", first) - } - removed := *old - removed.Event = ethmonitor.Removed - hardeningProcess(t, l, s, &removed) - select { - case rollback := <-s.TransactionReceipt(): - if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.TransactionHash() != tx.Hash() { - t.Fatalf("rollback: %+v", rollback) - } - case <-time.After(100 * time.Millisecond): - t.Error("missing rollback receipt") - } - if calls.Load() != 1 { - t.Errorf("rollback fetched orphan receipt: %d RPC calls", calls.Load()) - } - if err := s.finalizeReceipts(big.NewInt(104)); err != nil { - t.Fatal(err) - } - hardeningNoReceipt(t, s) - current.Store(hardeningReceipt(newBlock, tx)) - hardeningProcess(t, l, s, newBlock) - mined := hardeningRead(t, s) - if mined.BlockHash() != newBlock.Hash() || mined.Reorged || mined.Final { - t.Fatalf("re-mined receipt owns wrong block: %+v", mined) - } - if err := s.finalizeReceipts(big.NewInt(105)); err != nil { - t.Fatal(err) - } - final := hardeningRead(t, s) - if !final.Final || final.Reorged || final.BlockHash() != newBlock.Hash() || final.Filter != q { - t.Fatalf("canonical final receipt: %+v", final) - } -} - -func TestHardeningSubscribeBeforeRunFinality(t *testing.T) { - tx, _, _ := hardeningTxn(t, 2) - b := hardeningBlock(100, tx) - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} - l := hardeningListener(t, p, DefaultOptions, b, hardeningBlock(101)) - s := l.Subscribe(FilterTxnHash(tx.Hash())).(*subscriber) - defer s.Unsubscribe() - hardeningStart(t, l) - if r := hardeningRead(t, s); r.Final { - t.Fatal("receipt final before network threshold") - } - processed := make(chan struct{}) - s.AddFilter(FilterLogs(func([]*types.Log) bool { return false }).SearchCache(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { close(processed); return nil, nil })) - select { - case <-processed: - case <-time.After(time.Second): - t.Fatal("registration did not complete") - } - hardeningNoReceipt(t, s) - if len(s.Filters()) != 2 { - t.Fatal("premature finality removed transaction filter") - } -} - -func TestHardeningAddressFilters(t *testing.T) { - tx, from, to := hardeningTxn(t, 3) - b := hardeningBlock(100, tx) - for _, tc := range []struct { - name string - q FilterQuery - }{{"from", FilterFrom(from)}, {"to", FilterTo(to)}} { - t.Run(tc.name, func(t *testing.T) { - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} - l := hardeningListener(t, p, hardeningOptions(), b) - s := l.Subscribe(tc.q.SearchCache(true)) - defer s.Unsubscribe() - hardeningStart(t, l) - select { - case r := <-s.TransactionReceipt(): - if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() { - t.Fatalf("wrong matching transaction: %+v", r) - } - case <-time.After(100 * time.Millisecond): - t.Error("signed address filter missed cached transaction") - } - }) - } -} - -func TestHardeningFinalizerOrderAndOwners(t *testing.T) { - tx1, _, _ := hardeningTxn(t, 4) - tx2, _, _ := hardeningTxn(t, 5) - newer, older := hardeningBlock(101, tx1), hardeningBlock(100, tx2) - p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { - if h == tx1.Hash() { - return hardeningReceipt(newer, tx1), nil - } - return hardeningReceipt(older, tx2), nil - }} - l := hardeningListener(t, p, hardeningOptions()) - q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) - q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) - s := l.Subscribe(q1, q2).(*subscriber) - defer s.Unsubscribe() - for _, r := range []*types.Receipt{hardeningReceipt(newer, tx1), hardeningReceipt(older, tx2)} { - if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: r}}); err != nil { - t.Fatal(err) - } - hardeningRead(t, s) - hardeningRead(t, s) - } - for _, tc := range []struct { - head int64 - tx *types.Transaction - b *ethmonitor.Block - }{{103, tx2, older}, {104, tx1, newer}} { - if err := s.finalizeReceipts(big.NewInt(tc.head)); err != nil { - t.Fatal(err) - } - owners := map[Filterer]bool{} - for i := 0; i < 2; i++ { - select { - case r := <-s.TransactionReceipt(): - if !r.Final || r.TransactionHash() != tc.tx.Hash() || r.BlockHash() != tc.b.Hash() { - t.Errorf("wrong final at %d: %+v", tc.head, r) - } - owners[r.Filter] = true - case <-time.After(100 * time.Millisecond): - t.Errorf("missing owner final at %d", tc.head) - } - } - if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { - t.Errorf("finality lost distinct owners with same public ID at %d", tc.head) - } - } - hardeningNoReceipt(t, s) -} - -func TestHardeningPendingOwners(t *testing.T) { - tx, _, _ := hardeningTxn(t, 6) - b := hardeningBlock(100, tx) - var failing atomic.Bool - failing.Store(true) - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { - if failing.Load() { - return nil, errors.New("temporary provider failure") - } - return hardeningReceipt(b, tx), nil - }} - l := hardeningListener(t, p, hardeningOptions()) - q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(0).Finalize(true) - q2 := FilterTxnHash(tx.Hash()).ID(0) - s := l.Subscribe(q1, q2).(*subscriber) - defer s.Unsubscribe() - _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, chainID: big.NewInt(1)}}) - if err == nil { - t.Fatal("expected transient fetch failure") - } - failing.Store(false) - s.retryMu.Lock() - for _, pending := range s.pendingReceipts { - pending.nextRetryAt = time.Time{} - } - s.retryMu.Unlock() - s.retryPendingReceipts(context.Background()) - owners := map[Filterer]bool{} - for i := 0; i < 2; i++ { - select { - case r := <-s.TransactionReceipt(): - if r.Final || r.BlockHash() != b.Hash() { - t.Errorf("wrong recovered receipt: %+v", r) - } - owners[r.Filter] = true - case <-time.After(100 * time.Millisecond): - t.Error("missing recovered owner") - } - } - if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { - t.Error("retry lost an owner") - } - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - for i := 0; i < 2; i++ { - select { - case r := <-s.TransactionReceipt(): - if !r.Final || r.BlockHash() != b.Hash() { - t.Errorf("wrong recovered final: %+v", r) - } - case <-time.After(100 * time.Millisecond): - t.Error("missing recovered final") - } - } -} - -func TestHardeningLimitOneSnapshots(t *testing.T) { - for _, sameBlock := range []bool{true, false} { - t.Run(map[bool]string{true: "same_block", false: "retained_blocks"}[sameBlock], func(t *testing.T) { - tx1, _, _ := hardeningTxn(t, 7) - tx2, _, _ := hardeningTxn(t, 8) - b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) - blocks := ethmonitor.Blocks{b1, b2} - if sameBlock { - b1 = hardeningBlock(100, tx1, tx2) - b2 = b1 - blocks = ethmonitor.Blocks{b1} - } - p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { - if h == tx1.Hash() { - return hardeningReceipt(b1, tx1), nil - } - return hardeningReceipt(b2, tx2), nil - }} - l := hardeningListener(t, p, hardeningOptions()) - s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true)).(*subscriber) - defer s.Unsubscribe() - fs := s.Filters() - if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{fs}); err != nil { - t.Fatal(err) - } - r := hardeningRead(t, s) - if r.TransactionHash() != tx1.Hash() { - t.Errorf("LimitOne did not select first match: %s", r.TransactionHash()) - } - hardeningNoReceipt(t, s) - }) - } -} - -func TestHardeningQueryOnChainUsesCompleteReceipt(t *testing.T) { - tx, _, _ := hardeningTxn(t, 9) - b := hardeningBlock(90, tx) - complete := hardeningReceipt(b, tx) - var calls atomic.Int32 - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { - calls.Add(1) - return nil, ethereum.NotFound - }} - l := hardeningListener(t, p, hardeningOptions()) - s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil })).(*subscriber) - defer s.Unsubscribe() - if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { - t.Fatal(err) - } - select { - case r := <-s.TransactionReceipt(): - if r.Receipt() != complete || r.BlockHash() != b.Hash() { - t.Fatal("callback receipt replaced") - } - case <-time.After(100 * time.Millisecond): - t.Error("complete callback receipt was not delivered") - } - if calls.Load() != 0 { - t.Errorf("complete callback caused %d redundant RPCs", calls.Load()) - } -} - -func TestHardeningFetchPreservesSharedQuery(t *testing.T) { - l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - q := FilterLogs(func([]*types.Log) bool { return true }) - s := l.Subscribe(q) - defer s.Unsubscribe() - ctx, cancel := context.WithCancel(context.Background()) - cancel() - var wg sync.WaitGroup - for i := 0; i < 16; i++ { - wg.Add(1) - go func() { defer wg.Done(); _, _, _ = l.FetchTransactionReceiptWithFilter(ctx, q) }() - } - wg.Wait() - opts := s.Filters()[0].Options() - if opts.LimitOne || opts.SearchCache { - t.Fatalf("Fetch rewrote active subscription: %+v", opts) - } -} - -func TestHardeningStopCancelsStartup(t *testing.T) { - entered := make(chan struct{}) - p := &hardeningProvider{chainID: func(ctx context.Context) (*big.Int, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() }} - l := hardeningListener(t, p, DefaultOptions) - parent, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan error, 1) - go func() { done <- l.Run(parent) }() - <-entered - l.Stop() - select { - case <-done: - case <-time.After(100 * time.Millisecond): - cancel() - <-done - t.Error("Stop did not cancel startup ChainID request") - } -} - -func TestHardeningStopCancelsStartupBackoff(t *testing.T) { - entered := make(chan struct{}) - var calls atomic.Int32 - p := &hardeningProvider{chainID: func(context.Context) (*big.Int, error) { - if calls.Add(1) == 1 { - close(entered) - return nil, errors.New("transient ChainID failure") - } - return nil, breaker.ErrFatal - }} - l := hardeningListener(t, p, DefaultOptions) - parent, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan error, 1) - go func() { done <- l.Run(parent) }() - <-entered - // Allow the failed attempt to enter its one-second backoff. - time.Sleep(100 * time.Millisecond) - l.Stop() - select { - case err := <-done: - if !errors.Is(err, context.Canceled) { - t.Errorf("startup cancellation error: %v", err) - } - case <-time.After(500 * time.Millisecond): - cancel() - <-done - t.Error("Stop waited for startup ChainID backoff") - } - if calls.Load() != 1 { - t.Errorf("ChainID retried after cancellation: %d calls", calls.Load()) - } -} - -func TestHardeningMonitorStopCancelsRegistration(t *testing.T) { - l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - entered := make(chan struct{}) - s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return false }).QueryOnChain(func(ctx context.Context) (*types.Receipt, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() })) - defer s.Unsubscribe() - done := make(chan error, 1) - go func() { done <- l.Run(context.Background()) }() - <-entered - l.monitor.UnsubscribeAll(errors.New("monitor closed")) - select { - case <-done: - case <-time.After(100 * time.Millisecond): - l.Stop() - <-done - t.Error("monitor closure did not cancel registration") - } -} -func TestHardeningFetchSemaphoreCancellation(t *testing.T) { - entered, release := make(chan struct{}), make(chan struct{}) - p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { - close(entered) - select { - case <-release: - return &types.Receipt{TxHash: h, BlockNumber: big.NewInt(100)}, nil - case <-ctx.Done(): - return nil, ctx.Err() - } - }} - opts := hardeningOptions() - opts.MaxConcurrentFetchReceiptWorkers = 1 - l := hardeningListener(t, p, opts) - first := make(chan error, 1) - go func() { _, err := l.fetchTransactionReceipt(context.Background(), common.Hash{1}, true); first <- err }() - <-entered - ctx, cancel := context.WithCancel(context.Background()) - cancel() - second := make(chan error, 1) - go func() { _, err := l.fetchTransactionReceipt(ctx, common.Hash{2}, true); second <- err }() - blocked := false - select { - case err := <-second: - if !errors.Is(err, context.Canceled) { - t.Errorf("cancellation error: %v", err) - } - case <-time.After(100 * time.Millisecond): - blocked = true - } - close(release) - <-first - if blocked { - <-second - t.Error("canceled fetch waited for unrelated fetch slot") - } -} -func TestHardeningFinalityWaitCancellation(t *testing.T) { - tx, _, _ := hardeningTxn(t, 10) - b := hardeningBlock(100, tx) - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} - l := hardeningListener(t, p, hardeningOptions()) - hardeningStart(t, l) - parent, cancel := context.WithCancel(context.Background()) - defer cancel() - mined, wait, err := l.FetchTransactionReceiptWithFinality(parent, tx.Hash()) - if err != nil { - t.Fatal(err) - } - if mined.Final { - t.Fatal("initial receipt final") - } - ctx, finalCancel := context.WithCancel(context.Background()) - finalCancel() - done := make(chan error, 1) - go func() { _, err := wait(ctx); done <- err }() - select { - case err := <-done: - if !errors.Is(err, context.Canceled) { - t.Errorf("wait error: %v", err) - } - case <-time.After(100 * time.Millisecond): - cancel() - <-done - t.Error("finality waiter ignored its own context") - } -} -func TestHardeningConcurrentRunStop(t *testing.T) { - for i := 0; i < 5; i++ { - l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - _, l.ctxStop = context.WithCancel(context.Background()) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - var wg sync.WaitGroup - start := make(chan struct{}) - for j := 0; j < 16; j++ { - wg.Add(1) - go func() { defer wg.Done(); <-start; _ = l.Run(ctx) }() - } - wg.Add(1) - go func() { - defer wg.Done() - <-start - for j := 0; j < 16; j++ { - l.Stop() - } - }() - close(start) - wg.Wait() - } -} - -func TestHardeningInflightRollbackRemining(t *testing.T) { - tx, _, _ := hardeningTxn(t, 11) - old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) - entered, release := make(chan struct{}), make(chan struct{}) - var calls atomic.Int32 - p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { - if calls.Add(1) == 1 { - close(entered) - select { - case <-release: - return hardeningReceipt(old, tx), nil - case <-ctx.Done(): - return nil, ctx.Err() - } - } - return hardeningReceipt(canonical, tx), nil - }} - l := hardeningListener(t, p, hardeningOptions()) - q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - snapshot := s.Filters() - oldDone := make(chan error, 1) - go func() { - _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}) - oldDone <- err - }() - <-entered - removed := *old - removed.Event = ethmonitor.Removed - hardeningProcess(t, l, s, &removed) - select { - case rollback := <-s.TransactionReceipt(): - if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.Filter != q { - t.Errorf("wrong in-flight rollback: %+v", rollback) - } - case <-time.After(100 * time.Millisecond): - t.Error("missing in-flight rollback") - } - if err := s.finalizeReceipts(big.NewInt(104)); err != nil { - t.Fatal(err) - } - hardeningNoReceipt(t, s) - hardeningProcess(t, l, s, canonical) - select { - case mined := <-s.TransactionReceipt(): - if mined.BlockHash() != canonical.Hash() || mined.Reorged || mined.Final || mined.Filter != q { - t.Errorf("wrong canonical mined receipt: %+v", mined) - } - case <-time.After(100 * time.Millisecond): - t.Error("missing canonical mined receipt") - } - close(release) - select { - case <-oldDone: - case <-time.After(time.Second): - t.Fatal("old fetch did not finish") - } - // A registration snapshot retained before rollback must also remain invalid. - if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}); err != nil { - t.Fatal(err) - } - hardeningNoReceipt(t, s) - if err := s.finalizeReceipts(big.NewInt(105)); err != nil { - t.Fatal(err) - } - select { - case final := <-s.TransactionReceipt(): - if !final.Final || final.Reorged || final.BlockHash() != canonical.Hash() || final.Filter != q { - t.Errorf("wrong canonical final: %+v", final) - } - case <-time.After(100 * time.Millisecond): - t.Error("missing canonical final") - } - cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) - if err != nil || cached.BlockHash != canonical.Hash() { - t.Errorf("late orphan RPC poisoned receipt cache: receipt=%+v error=%v", cached, err) - } - if calls.Load() != 2 { - t.Errorf("unexpected fetch count: %d", calls.Load()) - } -} - -func TestHardeningConcurrentLimitOne(t *testing.T) { - tx1, _, _ := hardeningTxn(t, 12) - tx2, _, _ := hardeningTxn(t, 13) - b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) - entered, release := make(chan struct{}), make(chan struct{}) - p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { - if h == tx1.Hash() { - close(entered) - select { - case <-release: - return hardeningReceipt(b1, tx1), nil - case <-ctx.Done(): - return nil, ctx.Err() - } - } - return hardeningReceipt(b2, tx2), nil - }} - l := hardeningListener(t, p, hardeningOptions()) - q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - fs := s.Filters() - done := make(chan error, 1) - go func() { - _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{fs}) - done <- err - }() - <-entered - if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b2}, []*subscriber{s}, [][]Filterer{fs}); err != nil { - t.Fatal(err) - } - close(release) - <-done - r := hardeningRead(t, s) - if r.TransactionHash() != tx1.Hash() || r.Filter != q { - t.Errorf("concurrent LimitOne selected different transaction: %+v", r) - } - hardeningNoReceipt(t, s) - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - final := hardeningRead(t, s) - if !final.Final || final.TransactionHash() != tx1.Hash() || final.Filter != q { - t.Errorf("wrong claimed final: %+v", final) - } - hardeningNoReceipt(t, s) -} - -// This matcher deliberately needs the fetched receipt, which removed block data -// cannot supply. Rollback must use its previously delivered owner and receipt. -type hardeningStatusFilter struct{ Filterer } - -func (f *hardeningStatusFilter) Match(_ context.Context, r Receipt) (bool, error) { - return r.Status() == 1, nil -} -func TestHardeningRollbackRetainsCustomOwner(t *testing.T) { - tx, _, _ := hardeningTxn(t, 14) - b := hardeningBlock(100, tx) - complete := hardeningReceipt(b, tx) - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return complete, nil }} - l := hardeningListener(t, p, hardeningOptions()) - q := &hardeningStatusFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil }).(Filterer)} - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { - t.Fatal(err) - } - first := hardeningRead(t, s) - if first.Filter != q || first.Receipt() != complete { - t.Fatal("wrong custom filter owner or receipt") - } - removed := *b - removed.Event = ethmonitor.Removed - hardeningProcess(t, l, s, &removed) - select { - case r := <-s.TransactionReceipt(): - if !r.Reorged || r.Final || r.Filter != q || r.Receipt() != complete { - t.Errorf("rollback lost retained data/owner: %+v", r) - } - case <-time.After(100 * time.Millisecond): - t.Error("custom complete-receipt matcher lost rollback") - } - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - hardeningNoReceipt(t, s) -} - -func TestHardeningRollbackInvalidatesPending(t *testing.T) { - tx, _, _ := hardeningTxn(t, 15) - old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) - var failing atomic.Bool - failing.Store(true) - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { - if failing.Load() { - return nil, errors.New("transient receipt error") - } - return hardeningReceipt(canonical, tx), nil - }} - l := hardeningListener(t, p, hardeningOptions()) - q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - hardeningProcess(t, l, s, old) - s.retryMu.Lock() - pendingBefore := len(s.pendingReceipts) - s.retryMu.Unlock() - if pendingBefore != 1 { - t.Fatalf("expected one pending owner, got %d", pendingBefore) - } - removed := *old - removed.Event = ethmonitor.Removed - hardeningProcess(t, l, s, &removed) - s.retryMu.Lock() - pendingAfter := len(s.pendingReceipts) - s.retryMu.Unlock() - if pendingAfter != 0 { - t.Errorf("rollback retained %d orphan pending deliveries", pendingAfter) - } - select { - case r := <-s.TransactionReceipt(): - if !r.Reorged || r.Filter != q || r.BlockHash() != old.Hash() { - t.Errorf("wrong pending rollback: %+v", r) - } - case <-time.After(100 * time.Millisecond): - t.Error("missing pending rollback") - } - failing.Store(false) - hardeningProcess(t, l, s, canonical) - mined := hardeningRead(t, s) - if mined.BlockHash() != canonical.Hash() || mined.Reorged { - t.Fatal("retry ownership interfered with re-mining") - } - if err := s.finalizeReceipts(big.NewInt(105)); err != nil { - t.Fatal(err) - } - final := hardeningRead(t, s) - if !final.Final || final.BlockHash() != canonical.Hash() || final.Filter != q { - t.Fatal("wrong re-mined final owner") - } -} - -func TestHardeningExplicitRemovalCancelsOnlyOwner(t *testing.T) { - for _, clearAll := range []bool{false, true} { - t.Run(map[bool]string{false: "remove_one", true: "clear_all"}[clearAll], func(t *testing.T) { - tx, _, _ := hardeningTxn(t, 16) - b := hardeningBlock(100, tx) - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} - l := hardeningListener(t, p, hardeningOptions()) - q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(1).Finalize(true) - q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(2).Finalize(true) - s := l.Subscribe(q1, q2).(*subscriber) - defer s.Unsubscribe() - hardeningProcess(t, l, s, b) - hardeningRead(t, s) - hardeningRead(t, s) - if clearAll { - s.ClearFilters() - } else { - s.RemoveFilter(q1.(Filterer)) - } - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - if !clearAll { - r := hardeningRead(t, s) - if !r.Final || r.Filter != q2 || r.BlockHash() != b.Hash() { - t.Errorf("removal lost surviving owner: %+v", r) - } - } - hardeningNoReceipt(t, s) - }) - } -} - -func TestHardeningExhaustedOwnerKeepsQueuedFinal(t *testing.T) { - tx, _, _ := hardeningTxn(t, 17) - b := hardeningBlock(100, tx) - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} - l := hardeningListener(t, p, hardeningOptions()) - q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).MaxWait(1) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - hardeningProcess(t, l, s, b) - hardeningRead(t, s) - // Exhaustion removes matching and closes its signal, but documented pending - // finality remains owned. Exercise that state without relying on a live chain. - s.mu.Lock() - s.filters = nil - s.mu.Unlock() - q.(*filter).closeExhausted() - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - final := hardeningRead(t, s) - if !final.Final || final.Filter != q || final.BlockHash() != b.Hash() { - t.Fatal("exhaustion canceled a queued final") - } - hardeningNoReceipt(t, s) -} - -func TestHardeningMonitorClosureCancelsHeadWait(t *testing.T) { - p := &hardeningProvider{} - log := slog.New(slog.NewTextHandler(io.Discard, nil)) - mo := ethmonitor.DefaultOptions - mo.Logger = log - mo.WithLogs = true - mo.BlockRetentionLimit = 100 - mo.Bootstrap = true - monitor, err := ethmonitor.NewMonitor(p, mo) - if err != nil { - t.Fatal(err) - } - l, err := NewReceiptsListener(log, p, monitor, hardeningOptions()) - if err != nil { - t.Fatal(err) - } - parent, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan error, 1) - go func() { done <- l.Run(parent) }() - deadline := time.After(time.Second) - for monitor.NumSubscribers() != 1 { - select { - case <-deadline: - t.Fatal("listener did not subscribe") - default: - time.Sleep(time.Millisecond) - } - } - monitor.UnsubscribeAll(errors.New("monitor closed before first block")) - select { - case <-done: - case <-time.After(100 * time.Millisecond): - cancel() - // Baseline LatestBlockNum has a fixed 30-second wait; let it finish - // without mutating the monitor's bootstrap state during concurrent reads. - select { - case <-done: - case <-time.After(32 * time.Second): - t.Fatal("Run did not clean up") - } - t.Error("monitor closure did not cancel waiting for first head") - } -} - -func TestHardeningClearCancelsExhaustedFinality(t *testing.T) { - tx, _, _ := hardeningTxn(t, 18) - b := hardeningBlock(100, tx) - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} - l := hardeningListener(t, p, hardeningOptions()) - q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - hardeningProcess(t, l, s, b) - hardeningRead(t, s) - s.mu.Lock() - s.filters = nil - s.mu.Unlock() - q.(*filter).closeExhausted() - s.ClearFilters() - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - hardeningNoReceipt(t, s) -} - -func TestHardeningSameHashReadoptionRejectsOldWork(t *testing.T) { - tx, _, _ := hardeningTxn(t, 19) - block := hardeningBlock(100, tx) - oldReceipt, canonicalReceipt := hardeningReceipt(block, tx), hardeningReceipt(block, tx) - entered, release := make(chan struct{}), make(chan struct{}) - var calls atomic.Int32 - p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { - if calls.Add(1) == 1 { - close(entered) - select { - case <-release: - return oldReceipt, nil - case <-ctx.Done(): - return nil, ctx.Err() - } - } - return canonicalReceipt, nil - }} - l := hardeningListener(t, p, hardeningOptions()) - q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - fs := s.Filters() - oldDone := make(chan error, 1) - go func() { - _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}) - oldDone <- err - }() - <-entered - removed := *block - removed.Event = ethmonitor.Removed - hardeningProcess(t, l, s, &removed) - rollback := hardeningRead(t, s) - if !rollback.Reorged || rollback.BlockHash() != block.Hash() || rollback.Filter != q { - t.Fatalf("wrong rollback: %+v", rollback) - } - // A cache snapshot taken before removal cannot authoritatively re-adopt it. - hardeningProcess(t, l, s, block) - hardeningNoReceipt(t, s) - if calls.Load() != 1 { - t.Errorf("stale snapshot cleared removal marker: %d fetches", calls.Load()) - } - // The live block-processing entry point receives the canonical Added event. - if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}); err != nil { - t.Fatal(err) - } - select { - case r := <-s.TransactionReceipt(): - if r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { - t.Errorf("wrong readopted receipt: %+v", r) - } - case <-time.After(100 * time.Millisecond): - t.Error("canonical same-hash re-adoption did not reopen delivery") - } - close(release) - <-oldDone - hardeningNoReceipt(t, s) - if calls.Load() != 2 { - t.Errorf("expected fresh readoption fetch, got %d", calls.Load()) - } - cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) - if err != nil || cached != canonicalReceipt { - t.Errorf("old in-flight work replaced canonical cache: receipt=%p error=%v", cached, err) - } - // A callback with missing block data must acquire the fresh block generation - // when completing its receipt from cache, rather than inheriting generation zero. - partial := &types.Receipt{TxHash: tx.Hash()} - completion := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return partial, nil })).(*subscriber) - defer completion.Unsubscribe() - if err := l.queryFilterOnChain(context.Background(), completion, completion.Filters()); err != nil { - t.Fatal(err) - } - select { - case r := <-completion.TransactionReceipt(): - if r.Receipt() != canonicalReceipt || r.BlockHash() != block.Hash() { - t.Error("incomplete callback lost readopted receipt") - } - case <-time.After(100 * time.Millisecond): - t.Error("missing-data callback lost canonical generation") - } - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - select { - case r := <-s.TransactionReceipt(): - if !r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { - t.Errorf("wrong readopted final: %+v", r) - } - case <-time.After(100 * time.Millisecond): - t.Error("missing readopted final") - } - hardeningNoReceipt(t, s) -} diff --git a/ethreceipts/helpers_test.go b/ethreceipts/helpers_test.go new file mode 100644 index 00000000..290bc539 --- /dev/null +++ b/ethreceipts/helpers_test.go @@ -0,0 +1,224 @@ +package ethreceipts + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "math/big" + "sync" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/ethrpc" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/0xsequence/ethkit/go-ethereum/crypto" + "github.com/goware/breaker" +) + +// Only the RPC methods used by the listener are provided. No external node is needed. +type hardeningProvider struct { + ethrpc.RawInterface + chainID func(context.Context) (*big.Int, error) + receipt func(context.Context, common.Hash) (*types.Receipt, error) +} + +func (p *hardeningProvider) ChainID(ctx context.Context) (*big.Int, error) { + if p.chainID != nil { + return p.chainID(ctx) + } + return big.NewInt(1), nil +} + +func (p *hardeningProvider) TransactionReceipt(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if p.receipt != nil { + return p.receipt(ctx, h) + } + return nil, ethereum.NotFound +} + +func hardeningBlock(num int64, txns ...*types.Transaction) *ethmonitor.Block { + b := types.NewBlockWithHeader(&types.Header{Number: big.NewInt(num), BlockHash: common.BigToHash(big.NewInt(num)), ParentHash: common.BigToHash(big.NewInt(num - 1)), GasLimit: 30_000_000, Time: uint64(num)}) + return ðmonitor.Block{Block: b.WithBody(types.Body{Transactions: txns}), Event: ethmonitor.Added, OK: true} +} + +func hardeningTxn(t *testing.T, nonce uint64) (*types.Transaction, common.Address, common.Address) { + t.Helper() + key, err := crypto.HexToECDSA("1111111111111111111111111111111111111111111111111111111111111111") + if err != nil { + t.Fatal(err) + } + from := crypto.PubkeyToAddress(key.PublicKey) + to := common.HexToAddress("0x2222222222222222222222222222222222222222") + tx, err := types.SignTx(types.NewTransaction(nonce, to, big.NewInt(1), 21000, big.NewInt(1), nil), types.NewEIP155Signer(big.NewInt(1)), key) + if err != nil { + t.Fatal(err) + } + return tx, from, to +} + +func hardeningReceipt(b *ethmonitor.Block, tx *types.Transaction) *types.Receipt { + return &types.Receipt{TxHash: tx.Hash(), BlockHash: b.Hash(), BlockNumber: b.Number(), Status: 1} +} + +func hardeningListener(t *testing.T, p *hardeningProvider, opts Options, blocks ...*ethmonitor.Block) *ReceiptsListener { + t.Helper() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger = log + mo.WithLogs = true + mo.BlockRetentionLimit = 100 + mo.Bootstrap = true + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if len(blocks) == 0 { + blocks = []*ethmonitor.Block{hardeningBlock(100)} + } + if err = m.Chain().BootstrapFromBlocks(blocks); err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + l.br = breaker.New(log, 0, 1, 0) + return l +} + +func hardeningStart(t *testing.T, l *ReceiptsListener) { + t.Helper() + done := make(chan error, 1) + go func() { done <- l.Run(context.Background()) }() + deadline := time.After(time.Second) + for l.monitor.NumSubscribers() != 1 { + select { + case err := <-done: + t.Fatalf("Run exited before subscription: %v", err) + case <-deadline: + t.Fatal("Run did not subscribe to monitor") + default: + time.Sleep(time.Millisecond) + } + } + t.Cleanup(func() { + l.Stop() + select { + case err := <-done: + if err != nil { + t.Errorf("Run: %v", err) + } + case <-time.After(time.Second): + t.Error("Run did not stop") + } + }) +} + +func hardeningRead(t *testing.T, s Subscription) Receipt { + t.Helper() + select { + case r := <-s.TransactionReceipt(): + return r + case <-time.After(time.Second): + t.Fatal("missing receipt") + return Receipt{} + } +} + +func hardeningNoReceipt(t *testing.T, s Subscription) { + t.Helper() + select { + case r := <-s.TransactionReceipt(): + t.Errorf("unexpected receipt: txn=%s block=%s final=%v reorged=%v owner=%p", r.TransactionHash(), r.BlockHash(), r.Final, r.Reorged, r.Filter) + case <-time.After(30 * time.Millisecond): + } +} + +func hardeningProcess(t *testing.T, l *ReceiptsListener, s *subscriber, blocks ...*ethmonitor.Block) { + t.Helper() + if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + t.Fatal(err) + } +} + +func hardeningOptions() Options { opts := DefaultOptions; opts.NumBlocksToFinality = 2; return opts } + +// The real monitor polls this provider and builds/broadcasts reorg events. Tests +// advance its canonical RPC responses without changing monitor production APIs. +type ownershipLiveProvider struct { + *hardeningProvider + mu sync.Mutex + canonical map[uint64]*types.Block + blocks map[common.Hash]*types.Block +} + +func (p *ownershipLiveProvider) advance(blocks ...*types.Block) { + p.mu.Lock() + defer p.mu.Unlock() + for _, b := range blocks { + p.canonical[b.NumberU64()] = b + p.blocks[b.Hash()] = b + } +} + +func ownershipBlockPayload(b *types.Block) (json.RawMessage, error) { + if b == nil { + return nil, ethereum.NotFound + } + header := b.Header() + header.Difficulty = big.NewInt(0) + payload, err := json.Marshal(header) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err = json.Unmarshal(payload, &fields); err != nil { + return nil, err + } + fields["transactions"], err = json.Marshal(b.Transactions()) + if err != nil { + return nil, err + } + fields["uncles"] = json.RawMessage(`[]`) + return json.Marshal(fields) +} + +func (p *ownershipLiveProvider) RawBlockByNumber(_ context.Context, num *big.Int) (json.RawMessage, error) { + p.mu.Lock() + b := p.canonical[num.Uint64()] + p.mu.Unlock() + return ownershipBlockPayload(b) +} + +func (p *ownershipLiveProvider) RawBlockByHash(_ context.Context, hash common.Hash) (json.RawMessage, error) { + p.mu.Lock() + b := p.blocks[hash] + p.mu.Unlock() + return ownershipBlockPayload(b) +} + +func (p *ownershipLiveProvider) RawFilterLogs(context.Context, ethereum.FilterQuery) (json.RawMessage, error) { + return json.RawMessage(`[]`), nil +} + +func receiptsFixCollect(t *testing.T, s Subscription) []Receipt { + t.Helper() + var receipts []Receipt + timer := time.NewTimer(40 * time.Millisecond) + defer timer.Stop() + for { + select { + case r, ok := <-s.TransactionReceipt(): + if !ok { + t.Fatal("receipt channel closed unexpectedly") + } + receipts = append(receipts, r) + case <-timer.C: + return receipts + } + } +} diff --git a/ethreceipts/lifecycle_test.go b/ethreceipts/lifecycle_test.go new file mode 100644 index 00000000..24ae4ee1 --- /dev/null +++ b/ethreceipts/lifecycle_test.go @@ -0,0 +1,228 @@ +package ethreceipts + +import ( + "context" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/goware/breaker" +) + +func TestHardeningStopCancelsStartup(t *testing.T) { + entered := make(chan struct{}) + p := &hardeningProvider{chainID: func(ctx context.Context) (*big.Int, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() }} + l := hardeningListener(t, p, DefaultOptions) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + <-entered + l.Stop() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + cancel() + <-done + t.Error("Stop did not cancel startup ChainID request") + } +} + +func TestHardeningStopCancelsStartupBackoff(t *testing.T) { + entered := make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{chainID: func(context.Context) (*big.Int, error) { + if calls.Add(1) == 1 { + close(entered) + return nil, errors.New("transient ChainID failure") + } + return nil, breaker.ErrFatal + }} + l := hardeningListener(t, p, DefaultOptions) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + <-entered + // Allow the failed attempt to enter its one-second backoff. + time.Sleep(100 * time.Millisecond) + l.Stop() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("startup cancellation error: %v", err) + } + case <-time.After(500 * time.Millisecond): + cancel() + <-done + t.Error("Stop waited for startup ChainID backoff") + } + if calls.Load() != 1 { + t.Errorf("ChainID retried after cancellation: %d calls", calls.Load()) + } +} + +func TestHardeningMonitorStopCancelsRegistration(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + entered := make(chan struct{}) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return false }).QueryOnChain(func(ctx context.Context) (*types.Receipt, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() })) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { done <- l.Run(context.Background()) }() + <-entered + l.monitor.UnsubscribeAll(errors.New("monitor closed")) + select { + case <-done: + case <-time.After(100 * time.Millisecond): + l.Stop() + <-done + t.Error("monitor closure did not cancel registration") + } +} + +func TestHardeningFetchSemaphoreCancellation(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + close(entered) + select { + case <-release: + return &types.Receipt{TxHash: h, BlockNumber: big.NewInt(100)}, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + }} + opts := hardeningOptions() + opts.MaxConcurrentFetchReceiptWorkers = 1 + l := hardeningListener(t, p, opts) + first := make(chan error, 1) + go func() { _, err := l.fetchTransactionReceipt(context.Background(), common.Hash{1}, true); first <- err }() + <-entered + ctx, cancel := context.WithCancel(context.Background()) + cancel() + second := make(chan error, 1) + go func() { _, err := l.fetchTransactionReceipt(ctx, common.Hash{2}, true); second <- err }() + blocked := false + select { + case err := <-second: + if !errors.Is(err, context.Canceled) { + t.Errorf("cancellation error: %v", err) + } + case <-time.After(100 * time.Millisecond): + blocked = true + } + close(release) + <-first + if blocked { + <-second + t.Error("canceled fetch waited for unrelated fetch slot") + } +} + +func TestHardeningFinalityWaitCancellation(t *testing.T) { + tx, _, _ := hardeningTxn(t, 10) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + hardeningStart(t, l) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + mined, wait, err := l.FetchTransactionReceiptWithFinality(parent, tx.Hash()) + if err != nil { + t.Fatal(err) + } + if mined.Final { + t.Fatal("initial receipt final") + } + ctx, finalCancel := context.WithCancel(context.Background()) + finalCancel() + done := make(chan error, 1) + go func() { _, err := wait(ctx); done <- err }() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("wait error: %v", err) + } + case <-time.After(100 * time.Millisecond): + cancel() + <-done + t.Error("finality waiter ignored its own context") + } +} + +func TestHardeningConcurrentRunStop(t *testing.T) { + for i := 0; i < 5; i++ { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + _, l.ctxStop = context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var wg sync.WaitGroup + start := make(chan struct{}) + for j := 0; j < 16; j++ { + wg.Add(1) + go func() { defer wg.Done(); <-start; _ = l.Run(ctx) }() + } + wg.Add(1) + go func() { + defer wg.Done() + <-start + for j := 0; j < 16; j++ { + l.Stop() + } + }() + close(start) + wg.Wait() + } +} + +func TestHardeningMonitorClosureCancelsHeadWait(t *testing.T) { + p := &hardeningProvider{} + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger = log + mo.WithLogs = true + mo.BlockRetentionLimit = 100 + mo.Bootstrap = true + monitor, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, monitor, hardeningOptions()) + if err != nil { + t.Fatal(err) + } + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + deadline := time.After(time.Second) + for monitor.NumSubscribers() != 1 { + select { + case <-deadline: + t.Fatal("listener did not subscribe") + default: + time.Sleep(time.Millisecond) + } + } + monitor.UnsubscribeAll(errors.New("monitor closed before first block")) + select { + case <-done: + case <-time.After(100 * time.Millisecond): + cancel() + // Baseline LatestBlockNum has a fixed 30-second wait; let it finish + // without mutating the monitor's bootstrap state during concurrent reads. + select { + case <-done: + case <-time.After(32 * time.Second): + t.Fatal("Run did not clean up") + } + t.Error("monitor closure did not cancel waiting for first head") + } +} diff --git a/ethreceipts/ownership_regression_test.go b/ethreceipts/ownership_regression_test.go index ecff1b87..b3d1cd5d 100644 --- a/ethreceipts/ownership_regression_test.go +++ b/ethreceipts/ownership_regression_test.go @@ -2,22 +2,158 @@ package ethreceipts import ( "context" - "encoding/json" "errors" - "io" - "log/slog" + "fmt" "math/big" - "sync" "sync/atomic" "testing" "time" "github.com/0xsequence/ethkit/ethmonitor" - "github.com/0xsequence/ethkit/go-ethereum" "github.com/0xsequence/ethkit/go-ethereum/common" "github.com/0xsequence/ethkit/go-ethereum/core/types" ) +func TestHardeningAddressFilters(t *testing.T) { + tx, from, to := hardeningTxn(t, 3) + b := hardeningBlock(100, tx) + for _, tc := range []struct { + name string + q FilterQuery + }{{"from", FilterFrom(from)}, {"to", FilterTo(to)}} { + t.Run(tc.name, func(t *testing.T) { + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions(), b) + s := l.Subscribe(tc.q.SearchCache(true)) + defer s.Unsubscribe() + hardeningStart(t, l) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() { + t.Fatalf("wrong matching transaction: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("signed address filter missed cached transaction") + } + }) + } +} + +func TestHardeningLimitOneSnapshots(t *testing.T) { + for _, sameBlock := range []bool{true, false} { + t.Run(map[bool]string{true: "same_block", false: "retained_blocks"}[sameBlock], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 7) + tx2, _, _ := hardeningTxn(t, 8) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + blocks := ethmonitor.Blocks{b1, b2} + if sameBlock { + b1 = hardeningBlock(100, tx1, tx2) + b2 = b1 + blocks = ethmonitor.Blocks{b1} + } + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + return hardeningReceipt(b1, tx1), nil + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true)).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + r := hardeningRead(t, s) + if r.TransactionHash() != tx1.Hash() { + t.Errorf("LimitOne did not select first match: %s", r.TransactionHash()) + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestHardeningConcurrentLimitOne(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 12) + tx2, _, _ := hardeningTxn(t, 13) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + close(entered) + select { + case <-release: + return hardeningReceipt(b1, tx1), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + done := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{fs}) + done <- err + }() + <-entered + if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b2}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + close(release) + <-done + r := hardeningRead(t, s) + if r.TransactionHash() != tx1.Hash() || r.Filter != q { + t.Errorf("concurrent LimitOne selected different transaction: %+v", r) + } + hardeningNoReceipt(t, s) + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.TransactionHash() != tx1.Hash() || final.Filter != q { + t.Errorf("wrong claimed final: %+v", final) + } + hardeningNoReceipt(t, s) +} + +func TestHardeningExplicitRemovalCancelsOnlyOwner(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove_one", true: "clear_all"}[clearAll], func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 16) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(1).Finalize(true) + q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(2).Finalize(true) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + hardeningRead(t, s) + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q1.(Filterer)) + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + if !clearAll { + r := hardeningRead(t, s) + if !r.Final || r.Filter != q2 || r.BlockHash() != b.Hash() { + t.Errorf("removal lost surviving owner: %+v", r) + } + } + hardeningNoReceipt(t, s) + }) + } +} + type ownershipValueFilter struct { Filterer labels []string @@ -27,6 +163,7 @@ type ownershipValueFilter struct { func (f ownershipValueFilter) Match(ctx context.Context, r Receipt) (bool, error) { return f.match(ctx, r) } + func TestOwnershipCustomValueFilter(t *testing.T) { tx, _, _ := hardeningTxn(t, 100) b := hardeningBlock(100, tx) @@ -68,6 +205,7 @@ func (f *ownershipCollectionFilter) Match(context.Context, Receipt) (bool, error } return false, errors.New("temporary match failure") } + func TestOwnershipCollectionAbort(t *testing.T) { for _, canceled := range []bool{false, true} { t.Run(map[bool]string{false: "matcher_error", true: "cancellation"}[canceled], func(t *testing.T) { @@ -160,6 +298,7 @@ func TestOwnershipQueryReuse(t *testing.T) { }) } } + func TestOwnershipReaddRejectsOldWorker(t *testing.T) { for _, clearAll := range []bool{false, true} { t.Run(map[bool]string{false: "remove", true: "clear"}[clearAll], func(t *testing.T) { @@ -213,6 +352,7 @@ func TestOwnershipReaddRejectsOldWorker(t *testing.T) { }) } } + func TestOwnershipCompletionReleasesState(t *testing.T) { l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) s := l.Subscribe().(*subscriber) @@ -246,376 +386,187 @@ func TestOwnershipCompletionReleasesState(t *testing.T) { } } -func TestOwnershipPendingDropReleasesClaim(t *testing.T) { - for _, notFound := range []bool{false, true} { - t.Run(map[bool]string{false: "retry_limit", true: "not_found"}[notFound], func(t *testing.T) { - tx1, _, _ := hardeningTxn(t, 108) - tx2, _, _ := hardeningTxn(t, 109) - b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) - p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { - if h == tx2.Hash() { - return hardeningReceipt(b2, tx2), nil - } - return nil, errors.New("temporary receipt failure") - }} - l := hardeningListener(t, p, hardeningOptions()) - q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx1, blockHash: b1.Hash(), blockNum: b1.Number()}}); err == nil { - t.Fatal("expected initial pending fetch failure") - } - if len(s.pendingReceipts) != 1 || len(s.claims) != 1 { - t.Fatal("pending fetch lost its selection") - } - for _, pending := range s.pendingReceipts { - pending.nextRetryAt = time.Time{} - pending.attempts = maxReceiptRetryAttempts - 1 - } - if notFound { - p.receipt = func(_ context.Context, h common.Hash) (*types.Receipt, error) { - if h == tx2.Hash() { - return hardeningReceipt(b2, tx2), nil +func TestReceiptsFixCustomValueRemoval(t *testing.T) { + for _, pointer := range []bool{false, true} { + for _, exhausted := range []bool{false, true} { + for _, source := range []string{"original", "Filters", "Receipt.Filter"} { + t.Run(fmt.Sprintf("pointer=%v/exhausted=%v/%s", pointer, exhausted, source), func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 801) + tx2, _, _ := hardeningTxn(t, 802) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + custom := func(label string) Filterer { + return ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).ID(7).(Filterer), []string{label}, func(context.Context, Receipt) (bool, error) { return label != "", nil }} } - return nil, ethereum.NotFound - } - } - s.retryPendingReceipts(context.Background()) - if len(s.pendingReceipts) != 0 || len(s.claims) != 0 { - t.Error("discarded pending work retained an undelivered claim") - } - hardeningProcess(t, l, s, b2) - select { - case r := <-s.TransactionReceipt(): - if r.TransactionHash() != tx2.Hash() || r.Filter != q { - t.Error("released pending claim selected wrong transaction") - } - case <-time.After(100 * time.Millisecond): - t.Error("discarded pending work blocked a new transaction") + value := custom("selected").(ownershipValueFilter) + var q Filterer = value + if pointer { + q = &value + } + other := custom("surviving") + s := l.Subscribe(q, other).(*subscriber) + defer s.Unsubscribe() + first, second := s.filterers()[0].(*filterOwner), s.filterers()[1].(*filterOwner) + fromFilters := s.Filters()[0] + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(b1, tx1)) + var mined Receipt + for i := 0; i < 2; i++ { + r := hardeningRead(t, s) + if r.owner == first { + mined = r + if pointer { + if r.Filter != q || fromFilters != q { + t.Fatal("public pointer identity changed") + } + } else if r.Filter.(ownershipValueFilter).Filterer != value.Filterer || fromFilters.(ownershipValueFilter).Filterer != value.Filterer { + t.Fatal("original public custom value changed") + } + } + } + if exhausted { + s.exhaustFilter(first) + value.Filterer.(*filter).closeExhausted() + } + candidate := q + switch source { + case "Filters": + candidate = fromFilters + case "Receipt.Filter": + candidate = mined.Filter + } + s.RemoveFilter(candidate) + if s.hasFilter(first) || s.finalizer.hasOwner(first) || !s.hasFilter(second) || !s.finalizer.hasOwner(second) { + t.Error("public removal did not isolate selected registration/finality") + } + process(hardeningReceipt(b2, tx2)) + late := receiptsFixCollect(t, s) + if len(late) != 1 || late[0].owner != second || late[0].TransactionHash() != tx2.Hash() || late[0].Final || late[0].Reorged { + t.Error("selected cancellation changed later surviving delivery") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(t, s) + seen := make(map[common.Hash]bool) + for _, r := range finals { + if r.owner != second || !r.Final || r.Reorged || seen[r.TransactionHash()] { + t.Error("removed custom value still finalized or surviving final duplicated") + } + seen[r.TransactionHash()] = true + } + if len(finals) != 2 || !seen[tx1.Hash()] || !seen[tx2.Hash()] { + t.Error("custom removal lost surviving finals") + } + }) } - }) - } -} - -type ownershipGateFilter struct { - Filterer - entered, release chan struct{} - calls atomic.Int32 -} - -func (f *ownershipGateFilter) Match(ctx context.Context, _ Receipt) (bool, error) { - if f.calls.Add(1) == 1 { - close(f.entered) - select { - case <-f.release: - case <-ctx.Done(): - return false, ctx.Err() } } - return false, nil -} -func TestOwnershipStaleFetchPreservesCache(t *testing.T) { - for _, sameHash := range []bool{false, true} { - t.Run(map[bool]string{false: "remined_block", true: "readopted_generation"}[sameHash], func(t *testing.T) { - tx, _, _ := hardeningTxn(t, 107) - old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) - if sameHash { - canonical = old - } - complete := hardeningReceipt(canonical, tx) - var calls atomic.Int32 - var unavailable atomic.Bool - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { - calls.Add(1) - if unavailable.Load() { - return nil, ethereum.NotFound - } - return complete, nil - }} - l := hardeningListener(t, p, hardeningOptions()) - q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) - gate := &ownershipGateFilter{Filterer: FilterLogs(func([]*types.Log) bool { return false }).(Filterer), entered: make(chan struct{}), release: make(chan struct{})} - s := l.Subscribe(q, gate).(*subscriber) - defer s.Unsubscribe() - done := make(chan error, 1) - go func() { - _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, blockHash: old.Hash(), blockNum: old.Number()}}) - done <- err - }() - <-gate.entered - removed := *old - removed.Event = ethmonitor.Removed - hardeningProcess(t, l, s, &removed) - hardeningRead(t, s) - if sameHash { - if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{canonical}, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { - t.Fatal(err) - } - } else { - hardeningProcess(t, l, s, canonical) - } - hardeningRead(t, s) - cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) - if !found || cached != complete { - t.Fatal("canonical receipt was not cached") - } - close(gate.release) - <-done - cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) - if !found || cached != complete { - t.Error("late collection destroyed canonical cache") - } - unavailable.Store(true) - if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{old.Hash(), 0}); !errors.Is(err, ethereum.NotFound) { - t.Errorf("stale request error: %v", err) - } - cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) - if !found || cached != complete { - t.Error("stale expected block/generation deleted current cache") - } - r, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) - if err != nil || r != complete { - t.Errorf("fresh cache query depended on unavailable origin: %v", err) - } - if calls.Load() != 1 { - t.Errorf("stale work refetched a current canonical result: calls=%d", calls.Load()) - } - }) - } } -// The real monitor polls this provider and builds/broadcasts reorg events. Tests -// advance its canonical RPC responses without changing monitor production APIs. -type ownershipLiveProvider struct { - *hardeningProvider - mu sync.Mutex - canonical map[uint64]*types.Block - blocks map[common.Hash]*types.Block -} - -func (p *ownershipLiveProvider) advance(blocks ...*types.Block) { - p.mu.Lock() - defer p.mu.Unlock() - for _, b := range blocks { - p.canonical[b.NumberU64()] = b - p.blocks[b.Hash()] = b +func TestReceiptsFixSharedBaseAliases(t *testing.T) { + tx, _, _ := hardeningTxn(t, 803) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + base := FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).(Filterer) + first := ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + second := ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + other := ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer), []string{"other"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + s := l.Subscribe(first, second, other).(*subscriber) + defer s.Unsubscribe() + owners := s.filterers() + if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) } -} - -func ownershipBlockPayload(b *types.Block) (json.RawMessage, error) { - if b == nil { - return nil, ethereum.NotFound + for i := 0; i < 3; i++ { + hardeningRead(t, s) } - header := b.Header() - header.Difficulty = big.NewInt(0) - payload, err := json.Marshal(header) - if err != nil { - return nil, err + // Shared base signals define aliases: old RemoveFilter removes the first + // matching registration. Independent callback identities use distinct bases. + s.RemoveFilter(second) + if s.hasFilter(owners[0].(*filterOwner)) || s.finalizer.hasOwner(owners[0].(*filterOwner)) || !s.hasFilter(owners[1].(*filterOwner)) || !s.hasFilter(owners[2].(*filterOwner)) { + t.Error("shared-base removal did not preserve first-match alias semantics") } - var fields map[string]json.RawMessage - if err = json.Unmarshal(payload, &fields); err != nil { - return nil, err + s.RemoveFilter(s.Filters()[0]) + if s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) || !s.finalizer.hasOwner(owners[2].(*filterOwner)) { + t.Error("removing second base alias affected independent owner") } - fields["transactions"], err = json.Marshal(b.Transactions()) - if err != nil { - return nil, err + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(t, s) + if len(finals) != 1 || finals[0].owner != owners[2] || !finals[0].Final || finals[0].TransactionHash() != tx.Hash() { + t.Error("shared aliases canceled an unrelated queued final") } - fields["uncles"] = json.RawMessage(`[]`) - return json.Marshal(fields) -} - -func (p *ownershipLiveProvider) RawBlockByNumber(_ context.Context, num *big.Int) (json.RawMessage, error) { - p.mu.Lock() - b := p.canonical[num.Uint64()] - p.mu.Unlock() - return ownershipBlockPayload(b) } -func (p *ownershipLiveProvider) RawBlockByHash(_ context.Context, hash common.Hash) (json.RawMessage, error) { - p.mu.Lock() - b := p.blocks[hash] - p.mu.Unlock() - return ownershipBlockPayload(b) +type receiptsFixComparableFilter struct { + Filterer + label string } -func (p *ownershipLiveProvider) RawFilterLogs(context.Context, ethereum.FilterQuery) (json.RawMessage, error) { - return json.RawMessage(`[]`), nil +func (f receiptsFixComparableFilter) Match(context.Context, Receipt) (bool, error) { + return f.label != "", nil } -func TestOwnershipLiveExhaustedRollbackReleasesOwner(t *testing.T) { - tx, _, _ := hardeningTxn(t, 110) - b := hardeningBlock(100, tx) - p := &ownershipLiveProvider{ - hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { - return hardeningReceipt(b, tx), nil - }}, - canonical: make(map[uint64]*types.Block), - blocks: make(map[common.Hash]*types.Block), - } - log := slog.New(slog.NewTextHandler(io.Discard, nil)) - mo := ethmonitor.DefaultOptions - mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true - mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond - mo.BlockRetentionLimit = 100 - m, err := ethmonitor.NewMonitor(p, mo) - if err != nil { - t.Fatal(err) - } - if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99)}); err != nil { - t.Fatal(err) - } - opts := hardeningOptions() - opts.NumBlocksToFinality = 10 - l, err := NewReceiptsListener(log, p, m, opts) - if err != nil { - t.Fatal(err) - } - q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true).MaxWait(1) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - hardeningStart(t, l) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- m.Run(ctx) }() - t.Cleanup(func() { - cancel() - select { - case err := <-done: - if err != nil { - t.Errorf("monitor Run: %v", err) +func TestReceiptsFixComparableSharedBaseIsolation(t *testing.T) { + for _, pointer := range []bool{false, true} { + t.Run(fmt.Sprintf("pointer=%v", pointer), func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 804) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + base := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer) + var first, second Filterer = receiptsFixComparableFilter{base, "first"}, receiptsFixComparableFilter{base, "second"} + if pointer { + first = &ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + second = &ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} } - case <-time.After(3 * time.Second): - t.Error("monitor Run did not stop") - } - }) - wait := func(what string, condition func() bool) { - t.Helper() - deadline := time.Now().Add(12 * time.Second) - for !condition() { - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for %s", what) + s := l.Subscribe(first, second).(*subscriber) + defer s.Unsubscribe() + owners := s.filterers() + if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) } - time.Sleep(time.Millisecond) - } - } - alternate := func(num, fork int64, parent common.Hash) *types.Block { - header := hardeningBlock(num).Header() - header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+fork)), parent - return types.NewBlockWithHeader(header) - } - p.advance(b.Block) - mined := hardeningRead(t, s) - if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Final || mined.Reorged { - t.Fatal("wrong initial mined owner") - } - wait("first live match", func() bool { return q.(Filterer).LastMatchBlockNum() == 100 }) - old101 := hardeningBlock(101) - p.advance(old101.Block) - wait("block 101", func() bool { return m.LatestBlockNum().Int64() == 101 }) - alt101 := alternate(101, 10000, b.Hash()) - alt102 := alternate(102, 10000, alt101.Hash()) - p.advance(alt101, alt102) - wait("live reorg counter reset", func() bool { - return q.(Filterer).StartBlockNum() == 102 && q.(Filterer).LastMatchBlockNum() == 0 - }) - alt103 := alternate(103, 10000, alt102.Hash()) - p.advance(alt103) - select { - case <-q.(Filterer).Exhausted(): - case <-time.After(time.Second): - t.Fatal("live MaxWait did not exhaust owner") - } - s.deliveryMu.Lock() - queued := len(s.finalizer.queue) == 1 && len(s.claims) == 1 - s.deliveryMu.Unlock() - if !queued || len(s.Filters()) != 0 { - t.Fatal("exhaustion lost queued finality ownership") - } - current := hardeningBlock(99).Block - for num := int64(100); num <= 104; num++ { - next := alternate(num, 20000, current.Hash()) - p.advance(next) - current = next - } - wait("rollback invalidation", func() bool { - s.deliveryMu.Lock() - defer s.deliveryMu.Unlock() - return m.LatestBlock().Hash() == current.Hash() && len(s.finalizer.queue) == 0 - }) - select { - case rollback := <-s.TransactionReceipt(): - if rollback.TransactionHash() != tx.Hash() || rollback.BlockHash() != b.Hash() || rollback.BlockNumber().Int64() != 100 || rollback.Filter != q || !rollback.Reorged || rollback.Final || rollback.Status() != 1 { - t.Error("wrong exhausted-owner rollback identity/state") - } - case <-time.After(100 * time.Millisecond): - t.Error("exhausted queued owner missed its delivered rollback") - } - checkRetired := func() { - t.Helper() - s.deliveryMu.Lock() - defer s.deliveryMu.Unlock() - if len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.inFlight) != 0 || len(s.pendingReceipts) != 0 || len(s.finalizer.queue) != 0 || len(s.finalizer.txns) != 0 { - t.Errorf("inactive rollback owner retained state: claims=%d deliveries=%d", len(s.claims), len(s.deliveries)) - } - } - checkRetired() - for num := int64(105); num <= 112; num++ { - next := alternate(num, 20000, current.Hash()) - p.advance(next) - current = next + hardeningRead(t, s) + hardeningRead(t, s) + s.RemoveFilter(second) + if !s.hasFilter(owners[0].(*filterOwner)) || !s.finalizer.hasOwner(owners[0].(*filterOwner)) || s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) { + t.Fatal("base fallback overrode definitive comparable identity") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(t, s) + if len(finals) != 1 || finals[0].owner != owners[0] || !finals[0].Final { + t.Fatal("comparable cancellation lost surviving owner") + } + }) } - wait("finality advancement", func() bool { return m.LatestBlock().Hash() == current.Hash() }) - s.RemoveFilter(q.(Filterer)) - checkRetired() - hardeningNoReceipt(t, s) } -func TestOwnershipExhaustedRollbackKeepsOtherCandidates(t *testing.T) { - tx1, _, _ := hardeningTxn(t, 111) - tx2, _, _ := hardeningTxn(t, 112) - b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) +type receiptsFixNilSignalFilter struct{ ownershipValueFilter } + +func (receiptsFixNilSignalFilter) Exhausted() <-chan struct{} { return nil } + +func TestReceiptsFixCustomValueWithoutIdentity(t *testing.T) { l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) - other := FilterTxnHash(tx2.Hash()).SearchCache(false).QueryOnChainTxnHash(false) - s := l.Subscribe(q, other).(*subscriber) + value := receiptsFixNilSignalFilter{ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).(Filterer), []string{"no identity"}, func(context.Context, Receipt) (bool, error) { return true, nil }}} + other := FilterLogs(func([]*types.Log) bool { return true }).(Filterer) + s := l.Subscribe(value, other).(*subscriber) defer s.Unsubscribe() - for _, b := range []*ethmonitor.Block{b1, b2} { - if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b, b.Transactions()[0])}}); err != nil { - t.Fatal(err) - } + s.RemoveFilter(s.Filters()[0]) + if len(s.Filters()) != 2 { + t.Fatal("unidentifiable callback value removed another registration") } - for i := 0; i < 3; i++ { - hardeningRead(t, s) - } - s.exhaustFilter(q.(Filterer)) - q.(*filter).closeExhausted() - removed := *b1 - removed.Event = ethmonitor.Removed - hardeningProcess(t, l, s, &removed) - select { - case r := <-s.TransactionReceipt(): - if r.TransactionHash() != tx1.Hash() || r.BlockHash() != b1.Hash() || r.Filter != q || !r.Reorged || r.Final { - t.Error("wrong exhausted retained rollback") - } - case <-time.After(100 * time.Millisecond): - t.Error("inactive owner missed rollback while another candidate remained") - } - if len(s.finalizer.queue) != 2 || len(s.Filters()) != 1 { - t.Fatal("rollback removed unrelated ownership") - } - if err := s.finalizeReceipts(big.NewInt(104)); err != nil { - t.Fatal(err) - } - seen := make(map[Filterer]bool) - for i := 0; i < 2; i++ { - r := hardeningRead(t, s) - if r.TransactionHash() != tx2.Hash() || r.BlockHash() != b2.Hash() || !r.Final || r.Reorged || seen[r.Filter] { - t.Error("rollback lost or duplicated another owned final") - } - seen[r.Filter] = true - } - if !seen[q.(Filterer)] || !seen[other.(Filterer)] || len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.Filters()) != 0 { - t.Error("finished owners were not retired") + s.ClearFilters() + s.AddFilter(&value, other) + s.RemoveFilter(s.Filters()[0]) + if len(s.Filters()) != 1 || s.Filters()[0] != other { + t.Fatal("pointer identity did not isolate custom nil-signal removal") } - hardeningNoReceipt(t, s) } diff --git a/ethreceipts/pending_receipts_test.go b/ethreceipts/pending_receipts_test.go new file mode 100644 index 00000000..54141e23 --- /dev/null +++ b/ethreceipts/pending_receipts_test.go @@ -0,0 +1,357 @@ +package ethreceipts + +import ( + "context" + "errors" + "math/big" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningPendingOwners(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6) + b := hardeningBlock(100, tx) + var failing atomic.Bool + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if failing.Load() { + return nil, errors.New("temporary provider failure") + } + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(0).Finalize(true) + q2 := FilterTxnHash(tx.Hash()).ID(0) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, chainID: big.NewInt(1)}}) + if err == nil { + t.Fatal("expected transient fetch failure") + } + failing.Store(false) + s.retryMu.Lock() + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + } + s.retryMu.Unlock() + s.retryPendingReceipts(context.Background()) + owners := map[Filterer]bool{} + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if r.Final || r.BlockHash() != b.Hash() { + t.Errorf("wrong recovered receipt: %+v", r) + } + owners[r.Filter] = true + case <-time.After(100 * time.Millisecond): + t.Error("missing recovered owner") + } + } + if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { + t.Error("retry lost an owner") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.BlockHash() != b.Hash() { + t.Errorf("wrong recovered final: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing recovered final") + } + } +} + +func TestOwnershipPendingDropReleasesClaim(t *testing.T) { + for _, notFound := range []bool{false, true} { + t.Run(map[bool]string{false: "retry_limit", true: "not_found"}[notFound], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 108) + tx2, _, _ := hardeningTxn(t, 109) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx2.Hash() { + return hardeningReceipt(b2, tx2), nil + } + return nil, errors.New("temporary receipt failure") + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx1, blockHash: b1.Hash(), blockNum: b1.Number()}}); err == nil { + t.Fatal("expected initial pending fetch failure") + } + if len(s.pendingReceipts) != 1 || len(s.claims) != 1 { + t.Fatal("pending fetch lost its selection") + } + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + pending.attempts = maxReceiptRetryAttempts - 1 + } + if notFound { + p.receipt = func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx2.Hash() { + return hardeningReceipt(b2, tx2), nil + } + return nil, ethereum.NotFound + } + } + s.retryPendingReceipts(context.Background()) + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 { + t.Error("discarded pending work retained an undelivered claim") + } + hardeningProcess(t, l, s, b2) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("released pending claim selected wrong transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("discarded pending work blocked a new transaction") + } + }) + } +} + +func TestReceiptsFixRetryDeadlineAfterFetch(t *testing.T) { + for _, terminal := range []string{"retry", "remove", "clear", "rollback"} { + t.Run(terminal, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 800) + b := hardeningBlock(100, tx) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), b) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + seed := Receipt{transaction: tx, blockHash: b.Hash(), blockNum: b.Number()} + if !s.reserve(seed, owner) { + t.Fatal("initial reservation failed") + } + s.addPendingReceipt(seed, owner) + s.releaseReservation(seed, owner) + key := receiptOwner(seed, owner) + s.pendingReceipts[key].nextRetryAt = time.Time{} + pending := s.pendingReceipts[key] + parent := context.Background() + attempt, cancel := context.WithTimeout(parent, 200*time.Millisecond) + defer cancel() + s.deliveryMu.Lock() + done := make(chan struct{}) + go func() { s.retryPendingReceipts(attempt); close(done) }() + deadline := time.Now().Add(time.Second) + for { + _, cached, _ := l.pastReceipts.Get(parent, tx.Hash().Hex()) + if cached && len(l.fetchSem) == 0 { + break + } + if time.Now().After(deadline) { + s.deliveryMu.Unlock() + t.Fatal("receipt fetch did not complete before publication") + } + time.Sleep(time.Millisecond) + } + <-attempt.Done() + terminalDone := make(chan struct{}) + go func() { + defer close(terminalDone) + switch terminal { + case "remove": + s.RemoveFilter(q.(Filterer)) + case "clear": + s.ClearFilters() + case "rollback": + ref := l.invalidateBlock(parent, b) + s.rollbackBlock(ref) + } + }() + s.deliveryMu.Unlock() + <-done + <-terminalDone + hardeningNoReceipt(t, s) + if parent.Err() != nil { + t.Fatal("retry child canceled its listener parent") + } + if terminal == "retry" { + if s.pendingReceipts[key] != pending || len(s.claims) != 1 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Errorf("expired successful retry lost exact ownership: pending=%d claims=%d", len(s.pendingReceipts), len(s.claims)) + } + if s.pendingReceipts[key] == nil { + return + } + if pending.attempts != 1 || pending.nextRetryAt.After(time.Now().Add(time.Second)) { + t.Error("successful expired attempt was not promptly released for retry") + } + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(parent) + r := hardeningRead(t, s) + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q || r.Final || r.Reorged { + t.Error("later active retry lost mined identity") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + r = hardeningRead(t, s) + if !r.Final || r.Reorged || r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q { + t.Error("later active retry lost finality identity") + } + if calls.Load() != 1 { + t.Error("later retry discarded its successful cached receipt") + } + } else { + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Error("expired retry resurrected terminal owner/block state") + } + s.retryPendingReceipts(parent) + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestReceiptsFixPartialPendingCandidate(t *testing.T) { + for _, terminal := range []string{"retry", "rollback", "readoption", "remove", "clear"} { + t.Run(terminal, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 808) + replacement, _, _ := hardeningTxn(t, 809) + old, next := hardeningBlock(100, tx), hardeningBlock(101, replacement) + var ready atomic.Bool + var calls atomic.Int32 + l := hardeningListener(t, &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if !ready.Load() { + return nil, errors.New("provider temporarily unavailable") + } + return hardeningReceipt(old, tx), nil + }}, hardeningOptions(), old) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true). + QueryOnChain(func(context.Context) (*types.Receipt, error) { return &types.Receipt{TxHash: tx.Hash()}, nil }) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + if err := l.queryFilterOnChain(context.Background(), s, s.filterers()); err != nil { + t.Fatal(err) + } + if len(s.pendingReceipts) != 1 || s.claims[owner] != tx.Hash() { + t.Fatal("partial public callback did not enter the pending completion path") + } + var key receiptKey + var pending *pendingReceipt + for k, p := range s.pendingReceipts { + key, pending = k, p + } + if key.blockHash != (common.Hash{}) { + t.Fatal("partial callback already knew its block") + } + pending.nextRetryAt = time.Time{} + ready.Store(true) + attempt, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + s.deliveryMu.Lock() + done := make(chan struct{}) + go func() { s.retryPendingReceipts(attempt); close(done) }() + deadline := time.Now().Add(time.Second) + for { + _, cached, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if cached && len(l.fetchSem) == 0 { + break + } + if time.Now().After(deadline) { + s.deliveryMu.Unlock() + t.Fatal("partial pending fetch did not complete") + } + time.Sleep(time.Millisecond) + } + <-attempt.Done() + s.deliveryMu.Unlock() + <-done + hardeningNoReceipt(t, s) + if s.pendingReceipts[key] != pending || pending.attempts != 1 || calls.Load() != 2 { + t.Fatal("successful canceled completion did not retain its exact pending owner") + } + if terminal == "retry" { + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(context.Background()) + mined := hardeningRead(t, s) + if mined.Filter != q || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != old.Hash() || mined.generation != 0 || mined.Final || mined.Reorged { + t.Fatal("partial pending live retry lost its learned candidate") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if final.Filter != q || final.TransactionHash() != tx.Hash() || final.BlockHash() != old.Hash() || !final.Final || final.Reorged || calls.Load() != 2 { + t.Fatal("partial retry lost cached completion or canonical finality") + } + hardeningNoReceipt(t, s) + return + } + switch terminal { + case "rollback", "readoption": + ref := l.invalidateBlock(context.Background(), old) + if terminal == "readoption" { + // Even if authoritative same-hash acceptance wins before subscriber + // cleanup, the learned generation-0 candidate must stay invalid. + l.acceptBlock(l.monitor.GetBlock(old.Hash())) + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(context.Background()) + hardeningNoReceipt(t, s) + } + s.rollbackBlock(ref) + case "remove": + s.RemoveFilter(q.(Filterer)) + case "clear": + s.ClearFilters() + } + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Error("terminal invalidation retained a learned pending candidate or selection") + } + s.retryPendingReceipts(context.Background()) + hardeningNoReceipt(t, s) + if terminal == "remove" || terminal == "clear" { + s.AddFilter(q) + if s.owner(q.(Filterer)) == owner { + t.Fatal("new registration reused the canceled owner") + } + } + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: hardeningReceipt(next, replacement)}}); err != nil { + t.Fatal(err) + } + select { + case mined := <-s.TransactionReceipt(): + if mined.Filter != q || mined.TransactionHash() != replacement.Hash() || mined.BlockHash() != next.Hash() || mined.Final || mined.Reorged { + t.Error("fresh replacement lost its exact mined identity") + } + case <-time.After(100 * time.Millisecond): + t.Error("orphan pending claim blocked a fresh canonical replacement") + return + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if final.Filter != q || final.TransactionHash() != replacement.Hash() || final.BlockHash() != next.Hash() || !final.Final || final.Reorged { + t.Error("terminal invalidation published an orphan final") + } + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.Filters()) != 0 || calls.Load() != 2 { + t.Error("terminal candidate or completed owner was retained/re-fetched") + } + hardeningNoReceipt(t, s) + }) + } +} diff --git a/ethreceipts/receipt_fetch_test.go b/ethreceipts/receipt_fetch_test.go new file mode 100644 index 00000000..4a4d63d0 --- /dev/null +++ b/ethreceipts/receipt_fetch_test.go @@ -0,0 +1,150 @@ +package ethreceipts + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningQueryOnChainUsesCompleteReceipt(t *testing.T) { + tx, _, _ := hardeningTxn(t, 9) + b := hardeningBlock(90, tx) + complete := hardeningReceipt(b, tx) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + return nil, ethereum.NotFound + }} + l := hardeningListener(t, p, hardeningOptions()) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil })).(*subscriber) + defer s.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.Receipt() != complete || r.BlockHash() != b.Hash() { + t.Fatal("callback receipt replaced") + } + case <-time.After(100 * time.Millisecond): + t.Error("complete callback receipt was not delivered") + } + if calls.Load() != 0 { + t.Errorf("complete callback caused %d redundant RPCs", calls.Load()) + } +} + +func TestHardeningFetchPreservesSharedQuery(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }) + s := l.Subscribe(q) + defer s.Unsubscribe() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { defer wg.Done(); _, _, _ = l.FetchTransactionReceiptWithFilter(ctx, q) }() + } + wg.Wait() + opts := s.Filters()[0].Options() + if opts.LimitOne || opts.SearchCache { + t.Fatalf("Fetch rewrote active subscription: %+v", opts) + } +} + +type ownershipGateFilter struct { + Filterer + entered, release chan struct{} + calls atomic.Int32 +} + +func (f *ownershipGateFilter) Match(ctx context.Context, _ Receipt) (bool, error) { + if f.calls.Add(1) == 1 { + close(f.entered) + select { + case <-f.release: + case <-ctx.Done(): + return false, ctx.Err() + } + } + return false, nil +} + +func TestOwnershipStaleFetchPreservesCache(t *testing.T) { + for _, sameHash := range []bool{false, true} { + t.Run(map[bool]string{false: "remined_block", true: "readopted_generation"}[sameHash], func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 107) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + if sameHash { + canonical = old + } + complete := hardeningReceipt(canonical, tx) + var calls atomic.Int32 + var unavailable atomic.Bool + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if unavailable.Load() { + return nil, ethereum.NotFound + } + return complete, nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + gate := &ownershipGateFilter{Filterer: FilterLogs(func([]*types.Log) bool { return false }).(Filterer), entered: make(chan struct{}), release: make(chan struct{})} + s := l.Subscribe(q, gate).(*subscriber) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { + _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, blockHash: old.Hash(), blockNum: old.Number()}}) + done <- err + }() + <-gate.entered + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + hardeningRead(t, s) + if sameHash { + if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{canonical}, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + t.Fatal(err) + } + } else { + hardeningProcess(t, l, s, canonical) + } + hardeningRead(t, s) + cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Fatal("canonical receipt was not cached") + } + close(gate.release) + <-done + cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Error("late collection destroyed canonical cache") + } + unavailable.Store(true) + if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{old.Hash(), 0}); !errors.Is(err, ethereum.NotFound) { + t.Errorf("stale request error: %v", err) + } + cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Error("stale expected block/generation deleted current cache") + } + r, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || r != complete { + t.Errorf("fresh cache query depended on unavailable origin: %v", err) + } + if calls.Load() != 1 { + t.Errorf("stale work refetched a current canonical result: calls=%d", calls.Load()) + } + }) + } +} diff --git a/ethreceipts/regression_test.go b/ethreceipts/regression_test.go deleted file mode 100644 index 024d32c5..00000000 --- a/ethreceipts/regression_test.go +++ /dev/null @@ -1,713 +0,0 @@ -package ethreceipts - -import ( - "context" - "errors" - "fmt" - "io" - "log/slog" - "math/big" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/0xsequence/ethkit/ethmonitor" - "github.com/0xsequence/ethkit/go-ethereum" - "github.com/0xsequence/ethkit/go-ethereum/common" - "github.com/0xsequence/ethkit/go-ethereum/core/types" -) - -func receiptsFixCollect(t *testing.T, s Subscription) []Receipt { - t.Helper() - var receipts []Receipt - timer := time.NewTimer(40 * time.Millisecond) - defer timer.Stop() - for { - select { - case r, ok := <-s.TransactionReceipt(): - if !ok { - t.Fatal("receipt channel closed unexpectedly") - } - receipts = append(receipts, r) - case <-timer.C: - return receipts - } - } -} - -func TestReceiptsFixRetryDeadlineAfterFetch(t *testing.T) { - for _, terminal := range []string{"retry", "remove", "clear", "rollback"} { - t.Run(terminal, func(t *testing.T) { - tx, _, _ := hardeningTxn(t, 800) - b := hardeningBlock(100, tx) - var calls atomic.Int32 - p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { - calls.Add(1) - return hardeningReceipt(b, tx), nil - }} - l := hardeningListener(t, p, hardeningOptions(), b) - q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - owner := s.owner(q.(Filterer)) - seed := Receipt{transaction: tx, blockHash: b.Hash(), blockNum: b.Number()} - if !s.reserve(seed, owner) { - t.Fatal("initial reservation failed") - } - s.addPendingReceipt(seed, owner) - s.releaseReservation(seed, owner) - key := receiptOwner(seed, owner) - s.pendingReceipts[key].nextRetryAt = time.Time{} - pending := s.pendingReceipts[key] - parent := context.Background() - attempt, cancel := context.WithTimeout(parent, 200*time.Millisecond) - defer cancel() - s.deliveryMu.Lock() - done := make(chan struct{}) - go func() { s.retryPendingReceipts(attempt); close(done) }() - deadline := time.Now().Add(time.Second) - for { - _, cached, _ := l.pastReceipts.Get(parent, tx.Hash().Hex()) - if cached && len(l.fetchSem) == 0 { - break - } - if time.Now().After(deadline) { - s.deliveryMu.Unlock() - t.Fatal("receipt fetch did not complete before publication") - } - time.Sleep(time.Millisecond) - } - <-attempt.Done() - terminalDone := make(chan struct{}) - go func() { - defer close(terminalDone) - switch terminal { - case "remove": - s.RemoveFilter(q.(Filterer)) - case "clear": - s.ClearFilters() - case "rollback": - ref := l.invalidateBlock(parent, b) - s.rollbackBlock(ref) - } - }() - s.deliveryMu.Unlock() - <-done - <-terminalDone - hardeningNoReceipt(t, s) - if parent.Err() != nil { - t.Fatal("retry child canceled its listener parent") - } - if terminal == "retry" { - if s.pendingReceipts[key] != pending || len(s.claims) != 1 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { - t.Errorf("expired successful retry lost exact ownership: pending=%d claims=%d", len(s.pendingReceipts), len(s.claims)) - } - if s.pendingReceipts[key] == nil { - return - } - if pending.attempts != 1 || pending.nextRetryAt.After(time.Now().Add(time.Second)) { - t.Error("successful expired attempt was not promptly released for retry") - } - pending.nextRetryAt = time.Time{} - s.retryPendingReceipts(parent) - r := hardeningRead(t, s) - if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q || r.Final || r.Reorged { - t.Error("later active retry lost mined identity") - } - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - r = hardeningRead(t, s) - if !r.Final || r.Reorged || r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q { - t.Error("later active retry lost finality identity") - } - if calls.Load() != 1 { - t.Error("later retry discarded its successful cached receipt") - } - } else { - if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { - t.Error("expired retry resurrected terminal owner/block state") - } - s.retryPendingReceipts(parent) - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - } - hardeningNoReceipt(t, s) - }) - } -} - -func TestReceiptsFixPartialPendingCandidate(t *testing.T) { - for _, terminal := range []string{"retry", "rollback", "readoption", "remove", "clear"} { - t.Run(terminal, func(t *testing.T) { - tx, _, _ := hardeningTxn(t, 808) - replacement, _, _ := hardeningTxn(t, 809) - old, next := hardeningBlock(100, tx), hardeningBlock(101, replacement) - var ready atomic.Bool - var calls atomic.Int32 - l := hardeningListener(t, &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { - calls.Add(1) - if !ready.Load() { - return nil, errors.New("provider temporarily unavailable") - } - return hardeningReceipt(old, tx), nil - }}, hardeningOptions(), old) - q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true). - QueryOnChain(func(context.Context) (*types.Receipt, error) { return &types.Receipt{TxHash: tx.Hash()}, nil }) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - owner := s.owner(q.(Filterer)) - if err := l.queryFilterOnChain(context.Background(), s, s.filterers()); err != nil { - t.Fatal(err) - } - if len(s.pendingReceipts) != 1 || s.claims[owner] != tx.Hash() { - t.Fatal("partial public callback did not enter the pending completion path") - } - var key receiptKey - var pending *pendingReceipt - for k, p := range s.pendingReceipts { - key, pending = k, p - } - if key.blockHash != (common.Hash{}) { - t.Fatal("partial callback already knew its block") - } - pending.nextRetryAt = time.Time{} - ready.Store(true) - attempt, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - s.deliveryMu.Lock() - done := make(chan struct{}) - go func() { s.retryPendingReceipts(attempt); close(done) }() - deadline := time.Now().Add(time.Second) - for { - _, cached, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) - if cached && len(l.fetchSem) == 0 { - break - } - if time.Now().After(deadline) { - s.deliveryMu.Unlock() - t.Fatal("partial pending fetch did not complete") - } - time.Sleep(time.Millisecond) - } - <-attempt.Done() - s.deliveryMu.Unlock() - <-done - hardeningNoReceipt(t, s) - if s.pendingReceipts[key] != pending || pending.attempts != 1 || calls.Load() != 2 { - t.Fatal("successful canceled completion did not retain its exact pending owner") - } - if terminal == "retry" { - pending.nextRetryAt = time.Time{} - s.retryPendingReceipts(context.Background()) - mined := hardeningRead(t, s) - if mined.Filter != q || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != old.Hash() || mined.generation != 0 || mined.Final || mined.Reorged { - t.Fatal("partial pending live retry lost its learned candidate") - } - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - final := hardeningRead(t, s) - if final.Filter != q || final.TransactionHash() != tx.Hash() || final.BlockHash() != old.Hash() || !final.Final || final.Reorged || calls.Load() != 2 { - t.Fatal("partial retry lost cached completion or canonical finality") - } - hardeningNoReceipt(t, s) - return - } - switch terminal { - case "rollback", "readoption": - ref := l.invalidateBlock(context.Background(), old) - if terminal == "readoption" { - // Even if authoritative same-hash acceptance wins before subscriber - // cleanup, the learned generation-0 candidate must stay invalid. - l.acceptBlock(l.monitor.GetBlock(old.Hash())) - pending.nextRetryAt = time.Time{} - s.retryPendingReceipts(context.Background()) - hardeningNoReceipt(t, s) - } - s.rollbackBlock(ref) - case "remove": - s.RemoveFilter(q.(Filterer)) - case "clear": - s.ClearFilters() - } - if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { - t.Error("terminal invalidation retained a learned pending candidate or selection") - } - s.retryPendingReceipts(context.Background()) - hardeningNoReceipt(t, s) - if terminal == "remove" || terminal == "clear" { - s.AddFilter(q) - if s.owner(q.(Filterer)) == owner { - t.Fatal("new registration reused the canceled owner") - } - } - if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: hardeningReceipt(next, replacement)}}); err != nil { - t.Fatal(err) - } - select { - case mined := <-s.TransactionReceipt(): - if mined.Filter != q || mined.TransactionHash() != replacement.Hash() || mined.BlockHash() != next.Hash() || mined.Final || mined.Reorged { - t.Error("fresh replacement lost its exact mined identity") - } - case <-time.After(100 * time.Millisecond): - t.Error("orphan pending claim blocked a fresh canonical replacement") - return - } - if err := s.finalizeReceipts(big.NewInt(104)); err != nil { - t.Fatal(err) - } - final := hardeningRead(t, s) - if final.Filter != q || final.TransactionHash() != replacement.Hash() || final.BlockHash() != next.Hash() || !final.Final || final.Reorged { - t.Error("terminal invalidation published an orphan final") - } - if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.Filters()) != 0 || calls.Load() != 2 { - t.Error("terminal candidate or completed owner was retained/re-fetched") - } - hardeningNoReceipt(t, s) - }) - } -} - -func TestReceiptsFixCustomValueRemoval(t *testing.T) { - for _, pointer := range []bool{false, true} { - for _, exhausted := range []bool{false, true} { - for _, source := range []string{"original", "Filters", "Receipt.Filter"} { - t.Run(fmt.Sprintf("pointer=%v/exhausted=%v/%s", pointer, exhausted, source), func(t *testing.T) { - tx1, _, _ := hardeningTxn(t, 801) - tx2, _, _ := hardeningTxn(t, 802) - b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) - l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - custom := func(label string) Filterer { - return ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).ID(7).(Filterer), []string{label}, func(context.Context, Receipt) (bool, error) { return label != "", nil }} - } - value := custom("selected").(ownershipValueFilter) - var q Filterer = value - if pointer { - q = &value - } - other := custom("surviving") - s := l.Subscribe(q, other).(*subscriber) - defer s.Unsubscribe() - first, second := s.filterers()[0].(*filterOwner), s.filterers()[1].(*filterOwner) - fromFilters := s.Filters()[0] - process := func(b *types.Receipt) { - t.Helper() - if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { - t.Fatal(err) - } - } - process(hardeningReceipt(b1, tx1)) - var mined Receipt - for i := 0; i < 2; i++ { - r := hardeningRead(t, s) - if r.owner == first { - mined = r - if pointer { - if r.Filter != q || fromFilters != q { - t.Fatal("public pointer identity changed") - } - } else if r.Filter.(ownershipValueFilter).Filterer != value.Filterer || fromFilters.(ownershipValueFilter).Filterer != value.Filterer { - t.Fatal("original public custom value changed") - } - } - } - if exhausted { - s.exhaustFilter(first) - value.Filterer.(*filter).closeExhausted() - } - candidate := q - switch source { - case "Filters": - candidate = fromFilters - case "Receipt.Filter": - candidate = mined.Filter - } - s.RemoveFilter(candidate) - if s.hasFilter(first) || s.finalizer.hasOwner(first) || !s.hasFilter(second) || !s.finalizer.hasOwner(second) { - t.Error("public removal did not isolate selected registration/finality") - } - process(hardeningReceipt(b2, tx2)) - late := receiptsFixCollect(t, s) - if len(late) != 1 || late[0].owner != second || late[0].TransactionHash() != tx2.Hash() || late[0].Final || late[0].Reorged { - t.Error("selected cancellation changed later surviving delivery") - } - if err := s.finalizeReceipts(big.NewInt(105)); err != nil { - t.Fatal(err) - } - finals := receiptsFixCollect(t, s) - seen := make(map[common.Hash]bool) - for _, r := range finals { - if r.owner != second || !r.Final || r.Reorged || seen[r.TransactionHash()] { - t.Error("removed custom value still finalized or surviving final duplicated") - } - seen[r.TransactionHash()] = true - } - if len(finals) != 2 || !seen[tx1.Hash()] || !seen[tx2.Hash()] { - t.Error("custom removal lost surviving finals") - } - }) - } - } - } -} - -func TestReceiptsFixSharedBaseAliases(t *testing.T) { - tx, _, _ := hardeningTxn(t, 803) - b := hardeningBlock(100, tx) - l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - base := FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).(Filterer) - first := ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} - second := ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} - other := ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer), []string{"other"}, func(context.Context, Receipt) (bool, error) { return true, nil }} - s := l.Subscribe(first, second, other).(*subscriber) - defer s.Unsubscribe() - owners := s.filterers() - if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { - t.Fatal(err) - } - for i := 0; i < 3; i++ { - hardeningRead(t, s) - } - // Shared base signals define aliases: old RemoveFilter removes the first - // matching registration. Independent callback identities use distinct bases. - s.RemoveFilter(second) - if s.hasFilter(owners[0].(*filterOwner)) || s.finalizer.hasOwner(owners[0].(*filterOwner)) || !s.hasFilter(owners[1].(*filterOwner)) || !s.hasFilter(owners[2].(*filterOwner)) { - t.Error("shared-base removal did not preserve first-match alias semantics") - } - s.RemoveFilter(s.Filters()[0]) - if s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) || !s.finalizer.hasOwner(owners[2].(*filterOwner)) { - t.Error("removing second base alias affected independent owner") - } - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - finals := receiptsFixCollect(t, s) - if len(finals) != 1 || finals[0].owner != owners[2] || !finals[0].Final || finals[0].TransactionHash() != tx.Hash() { - t.Error("shared aliases canceled an unrelated queued final") - } -} - -type receiptsFixComparableFilter struct { - Filterer - label string -} - -func (f receiptsFixComparableFilter) Match(context.Context, Receipt) (bool, error) { - return f.label != "", nil -} - -func TestReceiptsFixComparableSharedBaseIsolation(t *testing.T) { - for _, pointer := range []bool{false, true} { - t.Run(fmt.Sprintf("pointer=%v", pointer), func(t *testing.T) { - tx, _, _ := hardeningTxn(t, 804) - b := hardeningBlock(100, tx) - l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - base := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer) - var first, second Filterer = receiptsFixComparableFilter{base, "first"}, receiptsFixComparableFilter{base, "second"} - if pointer { - first = &ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} - second = &ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} - } - s := l.Subscribe(first, second).(*subscriber) - defer s.Unsubscribe() - owners := s.filterers() - if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { - t.Fatal(err) - } - hardeningRead(t, s) - hardeningRead(t, s) - s.RemoveFilter(second) - if !s.hasFilter(owners[0].(*filterOwner)) || !s.finalizer.hasOwner(owners[0].(*filterOwner)) || s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) { - t.Fatal("base fallback overrode definitive comparable identity") - } - if err := s.finalizeReceipts(big.NewInt(103)); err != nil { - t.Fatal(err) - } - finals := receiptsFixCollect(t, s) - if len(finals) != 1 || finals[0].owner != owners[0] || !finals[0].Final { - t.Fatal("comparable cancellation lost surviving owner") - } - }) - } -} - -type receiptsFixNilSignalFilter struct{ ownershipValueFilter } - -func (receiptsFixNilSignalFilter) Exhausted() <-chan struct{} { return nil } - -func TestReceiptsFixCustomValueWithoutIdentity(t *testing.T) { - l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - value := receiptsFixNilSignalFilter{ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).(Filterer), []string{"no identity"}, func(context.Context, Receipt) (bool, error) { return true, nil }}} - other := FilterLogs(func([]*types.Log) bool { return true }).(Filterer) - s := l.Subscribe(value, other).(*subscriber) - defer s.Unsubscribe() - s.RemoveFilter(s.Filters()[0]) - if len(s.Filters()) != 2 { - t.Fatal("unidentifiable callback value removed another registration") - } - s.ClearFilters() - s.AddFilter(&value, other) - s.RemoveFilter(s.Filters()[0]) - if len(s.Filters()) != 1 || s.Filters()[0] != other { - t.Fatal("pointer identity did not isolate custom nil-signal removal") - } -} - -func TestReceiptsFixRollbackPreservesMinedSelection(t *testing.T) { - tx, _, _ := hardeningTxn(t, 805) - other, _, _ := hardeningTxn(t, 806) - old, replacement, canonical := hardeningBlock(100, tx), hardeningBlock(101, other), hardeningBlock(102, tx) - l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) - q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - owner := s.owner(q.(Filterer)) - process := func(b *types.Receipt) { - t.Helper() - if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { - t.Fatal(err) - } - } - process(hardeningReceipt(old, tx)) - hardeningRead(t, s) - s.rollbackBlock(l.invalidateBlock(context.Background(), old)) - rollback := hardeningRead(t, s) - if s.claims[owner] != tx.Hash() || !rollback.Reorged || rollback.Filter != q || rollback.BlockHash() != old.Hash() { - t.Fatal("rollback released an already-mined selection") - } - process(hardeningReceipt(replacement, other)) - hardeningNoReceipt(t, s) - process(hardeningReceipt(canonical, tx)) - mined := hardeningRead(t, s) - if mined.Reorged || mined.Final || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != canonical.Hash() || mined.Filter != q { - t.Fatal("retained selection did not re-mine correctly") - } - if err := s.finalizeReceipts(big.NewInt(105)); err != nil { - t.Fatal(err) - } - final := hardeningRead(t, s) - if !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || final.BlockHash() != canonical.Hash() || final.Filter != q { - t.Fatal("retained mined selection finalized an orphan") - } - hardeningNoReceipt(t, s) -} - -func TestReceiptsFixReadoptionAfterRetention(t *testing.T) { - for _, scenario := range []string{"canonical_retention", "obsolete_incarnation", "later_removal", "alternate_retention"} { - t.Run(scenario, func(t *testing.T) { - tx, _, _ := hardeningTxn(t, 807) - b := hardeningBlock(100, tx) - old, canonical := hardeningReceipt(b, tx), hardeningReceipt(b, tx) - entered, release := make(chan struct{}), make(chan struct{}) - var calls atomic.Int32 - var releaseOnce sync.Once - p := &ownershipLiveProvider{ - hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { - if calls.Add(1) == 1 { - close(entered) - // Deliberately finish an old RPC after its request timeout, - // to verify invalidation of late provider completions. - <-release - return old, nil - } - return canonical, nil - }}, - canonical: make(map[uint64]*types.Block), blocks: make(map[common.Hash]*types.Block), - } - log := slog.New(slog.NewTextHandler(io.Discard, nil)) - mo := ethmonitor.DefaultOptions - mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true - mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond - mo.BlockRetentionLimit = 50 - m, err := ethmonitor.NewMonitor(p, mo) - if err != nil { - t.Fatal(err) - } - if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99), b}); err != nil { - t.Fatal(err) - } - initialSnapshot := *m.LatestBlock() - opts := hardeningOptions() - opts.NumBlocksToFinality = 200 - l, err := NewReceiptsListener(log, p, m, opts) - if err != nil { - t.Fatal(err) - } - q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) - s := l.Subscribe(q).(*subscriber) - defer s.Unsubscribe() - stream := m.Subscribe("delayed receipt events") - defer stream.Unsubscribe() - oldDone := make(chan error, 1) - go func() { - _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{b.Hash(), 0}) - oldDone <- err - }() - <-entered - releaseOld := func() { releaseOnce.Do(func() { close(release) }) } - t.Cleanup(releaseOld) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- m.Run(ctx) }() - t.Cleanup(func() { - cancel() - select { - case err := <-done: - if err != nil { - t.Errorf("monitor Run: %v", err) - } - case <-time.After(3 * time.Second): - t.Error("monitor did not stop") - } - }) - consume := func() ethmonitor.Blocks { - t.Helper() - select { - case batch := <-stream.Blocks(): - return batch - case <-time.After(10 * time.Second): - t.Fatal("missing canonical monitor event") - return nil - } - } - process := func(batch ethmonitor.Blocks, cached bool) { - t.Helper() - var err error - if cached { - _, err = l.processCachedBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) - } else { - _, err = l.processBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) - } - if err != nil { - t.Fatal(err) - } - } - fork := func(from, through, offset int64, parent common.Hash) { - for num := from; num <= through; num++ { - header := hardeningBlock(num).Header() - header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+offset)), parent - next := types.NewBlockWithHeader(header) - p.advance(next) - parent = next.Hash() - } - } - waitHeight := func(height int64) { - t.Helper() - deadline := time.Now().Add(10 * time.Second) - for m.LatestBlockNum().Int64() != height { - if time.Now().After(deadline) { - t.Fatal("monitor did not advance canonical height") - } - time.Sleep(time.Millisecond) - } - } - // The first real reorg removes H, invalidating the held generation-0 RPC. - fork(100, 101, 10000, hardeningBlock(99).Hash()) - process(consume(), false) - rollback := hardeningRead(t, s) - if !rollback.Reorged || rollback.Final || rollback.BlockHash() != b.Hash() || rollback.TransactionHash() != tx.Hash() || rollback.Filter != q { - t.Fatal("wrong original removal identity") - } - process(ethmonitor.Blocks{&initialSnapshot}, true) - hardeningNoReceipt(t, s) - // Rebuild the same hash H in a fresh, owned monitor incarnation. - p.advance(b.Block, hardeningBlock(101).Block, hardeningBlock(102).Block) - readoption := consume() - var added *ethmonitor.Block - for _, event := range readoption { - if event.Event == ethmonitor.Added && event.Hash() == b.Hash() { - added = event - } - } - if added == nil { - t.Fatal("monitor omitted readopted incarnation") - } - incarnation, valid := added.CanonicalState() - if incarnation == 0 || !valid { - t.Fatal("Added input was not the owned canonical monitor event") - } - process(ethmonitor.Blocks{added}, true) - hardeningNoReceipt(t, s) - negative := scenario == "later_removal" || scenario == "alternate_retention" - var removed *ethmonitor.Block - if negative { - fork(100, 103, 20000, hardeningBlock(99).Hash()) - for _, event := range consume() { - if event.Event == ethmonitor.Removed && event.Hash() == b.Hash() { - removed = event - } - } - if removed == nil { - t.Fatal("monitor omitted later removal") - } - } - if scenario == "canonical_retention" { - fork(103, 155, 0, hardeningBlock(102).Hash()) - waitHeight(155) - } else if scenario == "alternate_retention" { - fork(104, 155, 20000, common.BigToHash(big.NewInt(20103))) - waitHeight(155) - } - if scenario == "canonical_retention" || scenario == "alternate_retention" { - if m.GetBlock(b.Hash()) != nil || m.OldestBlockNum().Int64() != 106 { - t.Fatal("H was not evicted more than one block behind retention") - } - } - if scenario == "obsolete_incarnation" { - // A removed tracked snapshot cannot fall back to a newer retained - // incarnation of the same hash, even during authoritative processing. - if m.GetBlock(b.Hash()) == nil { - t.Fatal("newer same-hash incarnation is missing") - } - process(ethmonitor.Blocks{&initialSnapshot}, false) - hardeningNoReceipt(t, s) - if !l.blockStates[b.Hash()].removed { - t.Error("obsolete tracked incarnation used same-hash fallback") - } - } - process(readoption, false) - if negative { - hardeningNoReceipt(t, s) - if !l.blockStates[b.Hash()].removed { - t.Error("later-removed Added reopened canonical marker") - } - process(ethmonitor.Blocks{added, removed}, false) - for _, r := range receiptsFixCollect(t, s) { - if !r.Reorged || r.Final { - t.Error("same-batch removal published stale mined/final receipt") - } - } - if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false); !errors.Is(err, ethereum.NotFound) { - t.Errorf("later removal reopened hash lookup: %v", err) - } - } else { - select { - case mined := <-s.TransactionReceipt(): - if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Reorged || mined.Final || mined.generation != 1 { - t.Error("readopted incarnation lost exact mined ownership") - } - case <-time.After(100 * time.Millisecond): - t.Error("canonical readoption after retention was rejected") - } - result, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) - if err != nil || result != canonical { - t.Errorf("canonical readoption hash lookup failed: %v", err) - } - } - releaseOld() - if err := <-oldDone; !errors.Is(err, ethereum.NotFound) { - t.Errorf("old generation RPC escaped invalidation: %v", err) - } - if !negative { - cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) - if !found || cached != canonical { - t.Error("old RPC replaced fresh canonical cache") - } - } - hardeningNoReceipt(t, s) - }) - } -} diff --git a/ethreceipts/reorg_test.go b/ethreceipts/reorg_test.go new file mode 100644 index 00000000..55183c97 --- /dev/null +++ b/ethreceipts/reorg_test.go @@ -0,0 +1,773 @@ +package ethreceipts + +import ( + "context" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningRollbackRemining(t *testing.T) { + tx, _, _ := hardeningTxn(t, 1) + old, newBlock := hardeningBlock(100, tx), hardeningBlock(102, tx) + var current atomic.Pointer[types.Receipt] + current.Store(hardeningReceipt(old, tx)) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { calls.Add(1); return current.Load(), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, old) + first := hardeningRead(t, s) + if first.Final || first.Reorged || first.BlockHash() != old.Hash() { + t.Fatalf("initial receipt: %+v", first) + } + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case rollback := <-s.TransactionReceipt(): + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.TransactionHash() != tx.Hash() { + t.Fatalf("rollback: %+v", rollback) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing rollback receipt") + } + if calls.Load() != 1 { + t.Errorf("rollback fetched orphan receipt: %d RPC calls", calls.Load()) + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + current.Store(hardeningReceipt(newBlock, tx)) + hardeningProcess(t, l, s, newBlock) + mined := hardeningRead(t, s) + if mined.BlockHash() != newBlock.Hash() || mined.Reorged || mined.Final { + t.Fatalf("re-mined receipt owns wrong block: %+v", mined) + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Reorged || final.BlockHash() != newBlock.Hash() || final.Filter != q { + t.Fatalf("canonical final receipt: %+v", final) + } +} + +func TestHardeningInflightRollbackRemining(t *testing.T) { + tx, _, _ := hardeningTxn(t, 11) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + select { + case <-release: + return hardeningReceipt(old, tx), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(canonical, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + snapshot := s.Filters() + oldDone := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}) + oldDone <- err + }() + <-entered + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case rollback := <-s.TransactionReceipt(): + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.Filter != q { + t.Errorf("wrong in-flight rollback: %+v", rollback) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing in-flight rollback") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + hardeningProcess(t, l, s, canonical) + select { + case mined := <-s.TransactionReceipt(): + if mined.BlockHash() != canonical.Hash() || mined.Reorged || mined.Final || mined.Filter != q { + t.Errorf("wrong canonical mined receipt: %+v", mined) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing canonical mined receipt") + } + close(release) + select { + case <-oldDone: + case <-time.After(time.Second): + t.Fatal("old fetch did not finish") + } + // A registration snapshot retained before rollback must also remain invalid. + if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + select { + case final := <-s.TransactionReceipt(): + if !final.Final || final.Reorged || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Errorf("wrong canonical final: %+v", final) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing canonical final") + } + cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || cached.BlockHash != canonical.Hash() { + t.Errorf("late orphan RPC poisoned receipt cache: receipt=%+v error=%v", cached, err) + } + if calls.Load() != 2 { + t.Errorf("unexpected fetch count: %d", calls.Load()) + } +} + +// This matcher deliberately needs the fetched receipt, which removed block data +// cannot supply. Rollback must use its previously delivered owner and receipt. +type hardeningStatusFilter struct{ Filterer } + +func (f *hardeningStatusFilter) Match(_ context.Context, r Receipt) (bool, error) { + return r.Status() == 1, nil +} + +func TestHardeningRollbackRetainsCustomOwner(t *testing.T) { + tx, _, _ := hardeningTxn(t, 14) + b := hardeningBlock(100, tx) + complete := hardeningReceipt(b, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return complete, nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := &hardeningStatusFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil }).(Filterer)} + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { + t.Fatal(err) + } + first := hardeningRead(t, s) + if first.Filter != q || first.Receipt() != complete { + t.Fatal("wrong custom filter owner or receipt") + } + removed := *b + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case r := <-s.TransactionReceipt(): + if !r.Reorged || r.Final || r.Filter != q || r.Receipt() != complete { + t.Errorf("rollback lost retained data/owner: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("custom complete-receipt matcher lost rollback") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} + +func TestHardeningRollbackInvalidatesPending(t *testing.T) { + tx, _, _ := hardeningTxn(t, 15) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + var failing atomic.Bool + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if failing.Load() { + return nil, errors.New("transient receipt error") + } + return hardeningReceipt(canonical, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, old) + s.retryMu.Lock() + pendingBefore := len(s.pendingReceipts) + s.retryMu.Unlock() + if pendingBefore != 1 { + t.Fatalf("expected one pending owner, got %d", pendingBefore) + } + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + s.retryMu.Lock() + pendingAfter := len(s.pendingReceipts) + s.retryMu.Unlock() + if pendingAfter != 0 { + t.Errorf("rollback retained %d orphan pending deliveries", pendingAfter) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Reorged || r.Filter != q || r.BlockHash() != old.Hash() { + t.Errorf("wrong pending rollback: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing pending rollback") + } + failing.Store(false) + hardeningProcess(t, l, s, canonical) + mined := hardeningRead(t, s) + if mined.BlockHash() != canonical.Hash() || mined.Reorged { + t.Fatal("retry ownership interfered with re-mining") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Fatal("wrong re-mined final owner") + } +} + +func TestHardeningSameHashReadoptionRejectsOldWork(t *testing.T) { + tx, _, _ := hardeningTxn(t, 19) + block := hardeningBlock(100, tx) + oldReceipt, canonicalReceipt := hardeningReceipt(block, tx), hardeningReceipt(block, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + select { + case <-release: + return oldReceipt, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return canonicalReceipt, nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + oldDone := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}) + oldDone <- err + }() + <-entered + removed := *block + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.BlockHash() != block.Hash() || rollback.Filter != q { + t.Fatalf("wrong rollback: %+v", rollback) + } + // A cache snapshot taken before removal cannot authoritatively re-adopt it. + hardeningProcess(t, l, s, block) + hardeningNoReceipt(t, s) + if calls.Load() != 1 { + t.Errorf("stale snapshot cleared removal marker: %d fetches", calls.Load()) + } + // The live block-processing entry point receives the canonical Added event. + if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { + t.Errorf("wrong readopted receipt: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("canonical same-hash re-adoption did not reopen delivery") + } + close(release) + <-oldDone + hardeningNoReceipt(t, s) + if calls.Load() != 2 { + t.Errorf("expected fresh readoption fetch, got %d", calls.Load()) + } + cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || cached != canonicalReceipt { + t.Errorf("old in-flight work replaced canonical cache: receipt=%p error=%v", cached, err) + } + // A callback with missing block data must acquire the fresh block generation + // when completing its receipt from cache, rather than inheriting generation zero. + partial := &types.Receipt{TxHash: tx.Hash()} + completion := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return partial, nil })).(*subscriber) + defer completion.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), completion, completion.Filters()); err != nil { + t.Fatal(err) + } + select { + case r := <-completion.TransactionReceipt(): + if r.Receipt() != canonicalReceipt || r.BlockHash() != block.Hash() { + t.Error("incomplete callback lost readopted receipt") + } + case <-time.After(100 * time.Millisecond): + t.Error("missing-data callback lost canonical generation") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { + t.Errorf("wrong readopted final: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing readopted final") + } + hardeningNoReceipt(t, s) +} + +func TestOwnershipLiveExhaustedRollbackReleasesOwner(t *testing.T) { + tx, _, _ := hardeningTxn(t, 110) + b := hardeningBlock(100, tx) + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }}, + canonical: make(map[uint64]*types.Block), + blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 100 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99)}); err != nil { + t.Fatal(err) + } + opts := hardeningOptions() + opts.NumBlocksToFinality = 10 + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true).MaxWait(1) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningStart(t, l) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor Run did not stop") + } + }) + wait := func(what string, condition func() bool) { + t.Helper() + deadline := time.Now().Add(12 * time.Second) + for !condition() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } + } + alternate := func(num, fork int64, parent common.Hash) *types.Block { + header := hardeningBlock(num).Header() + header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+fork)), parent + return types.NewBlockWithHeader(header) + } + p.advance(b.Block) + mined := hardeningRead(t, s) + if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Final || mined.Reorged { + t.Fatal("wrong initial mined owner") + } + wait("first live match", func() bool { return q.(Filterer).LastMatchBlockNum() == 100 }) + old101 := hardeningBlock(101) + p.advance(old101.Block) + wait("block 101", func() bool { return m.LatestBlockNum().Int64() == 101 }) + alt101 := alternate(101, 10000, b.Hash()) + alt102 := alternate(102, 10000, alt101.Hash()) + p.advance(alt101, alt102) + wait("live reorg counter reset", func() bool { + return q.(Filterer).StartBlockNum() == 102 && q.(Filterer).LastMatchBlockNum() == 0 + }) + alt103 := alternate(103, 10000, alt102.Hash()) + p.advance(alt103) + select { + case <-q.(Filterer).Exhausted(): + case <-time.After(time.Second): + t.Fatal("live MaxWait did not exhaust owner") + } + s.deliveryMu.Lock() + queued := len(s.finalizer.queue) == 1 && len(s.claims) == 1 + s.deliveryMu.Unlock() + if !queued || len(s.Filters()) != 0 { + t.Fatal("exhaustion lost queued finality ownership") + } + current := hardeningBlock(99).Block + for num := int64(100); num <= 104; num++ { + next := alternate(num, 20000, current.Hash()) + p.advance(next) + current = next + } + wait("rollback invalidation", func() bool { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + return m.LatestBlock().Hash() == current.Hash() && len(s.finalizer.queue) == 0 + }) + select { + case rollback := <-s.TransactionReceipt(): + if rollback.TransactionHash() != tx.Hash() || rollback.BlockHash() != b.Hash() || rollback.BlockNumber().Int64() != 100 || rollback.Filter != q || !rollback.Reorged || rollback.Final || rollback.Status() != 1 { + t.Error("wrong exhausted-owner rollback identity/state") + } + case <-time.After(100 * time.Millisecond): + t.Error("exhausted queued owner missed its delivered rollback") + } + checkRetired := func() { + t.Helper() + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.inFlight) != 0 || len(s.pendingReceipts) != 0 || len(s.finalizer.queue) != 0 || len(s.finalizer.txns) != 0 { + t.Errorf("inactive rollback owner retained state: claims=%d deliveries=%d", len(s.claims), len(s.deliveries)) + } + } + checkRetired() + for num := int64(105); num <= 112; num++ { + next := alternate(num, 20000, current.Hash()) + p.advance(next) + current = next + } + wait("finality advancement", func() bool { return m.LatestBlock().Hash() == current.Hash() }) + s.RemoveFilter(q.(Filterer)) + checkRetired() + hardeningNoReceipt(t, s) +} + +func TestOwnershipExhaustedRollbackKeepsOtherCandidates(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 111) + tx2, _, _ := hardeningTxn(t, 112) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + other := FilterTxnHash(tx2.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q, other).(*subscriber) + defer s.Unsubscribe() + for _, b := range []*ethmonitor.Block{b1, b2} { + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b, b.Transactions()[0])}}); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 3; i++ { + hardeningRead(t, s) + } + s.exhaustFilter(q.(Filterer)) + q.(*filter).closeExhausted() + removed := *b1 + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx1.Hash() || r.BlockHash() != b1.Hash() || r.Filter != q || !r.Reorged || r.Final { + t.Error("wrong exhausted retained rollback") + } + case <-time.After(100 * time.Millisecond): + t.Error("inactive owner missed rollback while another candidate remained") + } + if len(s.finalizer.queue) != 2 || len(s.Filters()) != 1 { + t.Fatal("rollback removed unrelated ownership") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + seen := make(map[Filterer]bool) + for i := 0; i < 2; i++ { + r := hardeningRead(t, s) + if r.TransactionHash() != tx2.Hash() || r.BlockHash() != b2.Hash() || !r.Final || r.Reorged || seen[r.Filter] { + t.Error("rollback lost or duplicated another owned final") + } + seen[r.Filter] = true + } + if !seen[q.(Filterer)] || !seen[other.(Filterer)] || len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.Filters()) != 0 { + t.Error("finished owners were not retired") + } + hardeningNoReceipt(t, s) +} + +func TestReceiptsFixRollbackPreservesMinedSelection(t *testing.T) { + tx, _, _ := hardeningTxn(t, 805) + other, _, _ := hardeningTxn(t, 806) + old, replacement, canonical := hardeningBlock(100, tx), hardeningBlock(101, other), hardeningBlock(102, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(old, tx)) + hardeningRead(t, s) + s.rollbackBlock(l.invalidateBlock(context.Background(), old)) + rollback := hardeningRead(t, s) + if s.claims[owner] != tx.Hash() || !rollback.Reorged || rollback.Filter != q || rollback.BlockHash() != old.Hash() { + t.Fatal("rollback released an already-mined selection") + } + process(hardeningReceipt(replacement, other)) + hardeningNoReceipt(t, s) + process(hardeningReceipt(canonical, tx)) + mined := hardeningRead(t, s) + if mined.Reorged || mined.Final || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != canonical.Hash() || mined.Filter != q { + t.Fatal("retained selection did not re-mine correctly") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Fatal("retained mined selection finalized an orphan") + } + hardeningNoReceipt(t, s) +} + +func TestReceiptsFixReadoptionAfterRetention(t *testing.T) { + for _, scenario := range []string{"canonical_retention", "obsolete_incarnation", "later_removal", "alternate_retention"} { + t.Run(scenario, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 807) + b := hardeningBlock(100, tx) + old, canonical := hardeningReceipt(b, tx), hardeningReceipt(b, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + var releaseOnce sync.Once + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + // Deliberately finish an old RPC after its request timeout, + // to verify invalidation of late provider completions. + <-release + return old, nil + } + return canonical, nil + }}, + canonical: make(map[uint64]*types.Block), blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 50 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99), b}); err != nil { + t.Fatal(err) + } + initialSnapshot := *m.LatestBlock() + opts := hardeningOptions() + opts.NumBlocksToFinality = 200 + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + stream := m.Subscribe("delayed receipt events") + defer stream.Unsubscribe() + oldDone := make(chan error, 1) + go func() { + _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{b.Hash(), 0}) + oldDone <- err + }() + <-entered + releaseOld := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(releaseOld) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor did not stop") + } + }) + consume := func() ethmonitor.Blocks { + t.Helper() + select { + case batch := <-stream.Blocks(): + return batch + case <-time.After(10 * time.Second): + t.Fatal("missing canonical monitor event") + return nil + } + } + process := func(batch ethmonitor.Blocks, cached bool) { + t.Helper() + var err error + if cached { + _, err = l.processCachedBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + } else { + _, err = l.processBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + } + if err != nil { + t.Fatal(err) + } + } + fork := func(from, through, offset int64, parent common.Hash) { + for num := from; num <= through; num++ { + header := hardeningBlock(num).Header() + header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+offset)), parent + next := types.NewBlockWithHeader(header) + p.advance(next) + parent = next.Hash() + } + } + waitHeight := func(height int64) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for m.LatestBlockNum().Int64() != height { + if time.Now().After(deadline) { + t.Fatal("monitor did not advance canonical height") + } + time.Sleep(time.Millisecond) + } + } + // The first real reorg removes H, invalidating the held generation-0 RPC. + fork(100, 101, 10000, hardeningBlock(99).Hash()) + process(consume(), false) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != b.Hash() || rollback.TransactionHash() != tx.Hash() || rollback.Filter != q { + t.Fatal("wrong original removal identity") + } + process(ethmonitor.Blocks{&initialSnapshot}, true) + hardeningNoReceipt(t, s) + // Rebuild the same hash H in a fresh, owned monitor incarnation. + p.advance(b.Block, hardeningBlock(101).Block, hardeningBlock(102).Block) + readoption := consume() + var added *ethmonitor.Block + for _, event := range readoption { + if event.Event == ethmonitor.Added && event.Hash() == b.Hash() { + added = event + } + } + if added == nil { + t.Fatal("monitor omitted readopted incarnation") + } + incarnation, valid := added.CanonicalState() + if incarnation == 0 || !valid { + t.Fatal("Added input was not the owned canonical monitor event") + } + process(ethmonitor.Blocks{added}, true) + hardeningNoReceipt(t, s) + negative := scenario == "later_removal" || scenario == "alternate_retention" + var removed *ethmonitor.Block + if negative { + fork(100, 103, 20000, hardeningBlock(99).Hash()) + for _, event := range consume() { + if event.Event == ethmonitor.Removed && event.Hash() == b.Hash() { + removed = event + } + } + if removed == nil { + t.Fatal("monitor omitted later removal") + } + } + if scenario == "canonical_retention" { + fork(103, 155, 0, hardeningBlock(102).Hash()) + waitHeight(155) + } else if scenario == "alternate_retention" { + fork(104, 155, 20000, common.BigToHash(big.NewInt(20103))) + waitHeight(155) + } + if scenario == "canonical_retention" || scenario == "alternate_retention" { + if m.GetBlock(b.Hash()) != nil || m.OldestBlockNum().Int64() != 106 { + t.Fatal("H was not evicted more than one block behind retention") + } + } + if scenario == "obsolete_incarnation" { + // A removed tracked snapshot cannot fall back to a newer retained + // incarnation of the same hash, even during authoritative processing. + if m.GetBlock(b.Hash()) == nil { + t.Fatal("newer same-hash incarnation is missing") + } + process(ethmonitor.Blocks{&initialSnapshot}, false) + hardeningNoReceipt(t, s) + if !l.blockStates[b.Hash()].removed { + t.Error("obsolete tracked incarnation used same-hash fallback") + } + } + process(readoption, false) + if negative { + hardeningNoReceipt(t, s) + if !l.blockStates[b.Hash()].removed { + t.Error("later-removed Added reopened canonical marker") + } + process(ethmonitor.Blocks{added, removed}, false) + for _, r := range receiptsFixCollect(t, s) { + if !r.Reorged || r.Final { + t.Error("same-batch removal published stale mined/final receipt") + } + } + if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false); !errors.Is(err, ethereum.NotFound) { + t.Errorf("later removal reopened hash lookup: %v", err) + } + } else { + select { + case mined := <-s.TransactionReceipt(): + if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Reorged || mined.Final || mined.generation != 1 { + t.Error("readopted incarnation lost exact mined ownership") + } + case <-time.After(100 * time.Millisecond): + t.Error("canonical readoption after retention was rejected") + } + result, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || result != canonical { + t.Errorf("canonical readoption hash lookup failed: %v", err) + } + } + releaseOld() + if err := <-oldDone; !errors.Is(err, ethereum.NotFound) { + t.Errorf("old generation RPC escaped invalidation: %v", err) + } + if !negative { + cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != canonical { + t.Error("old RPC replaced fresh canonical cache") + } + } + hardeningNoReceipt(t, s) + }) + } +} From d711f5c0d0fba00c53f137177fb021f7298de1e0 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 10:40:40 -0400 Subject: [PATCH 08/11] refactor(ethreceipts): clarify pending receipt retries --- ethreceipts/subscription.go | 213 +++++++++++++++++++----------------- 1 file changed, 112 insertions(+), 101 deletions(-) diff --git a/ethreceipts/subscription.go b/ethreceipts/subscription.go index 33249a12..f4d554b4 100644 --- a/ethreceipts/subscription.go +++ b/ethreceipts/subscription.go @@ -55,22 +55,27 @@ type subscriber struct { ch channel.Channel[Receipt] done chan struct{} unsubscribe func() - filters []*filterOwner + filters []*filterOwner // Protected by mu. finalizer *finalizer mu sync.Mutex + // retryMu protects pendingReceipts and the mutable fields of each pendingReceipt. pendingReceipts map[receiptKey]*pendingReceipt retryMu sync.Mutex - deliveryMu sync.Mutex - deliveries map[receiptKey]Receipt - inFlight map[receiptKey]struct{} - claims map[*filterOwner]common.Hash + + // deliveryMu serializes publication, rollback, finality and owner retirement, + // and protects deliveries, inFlight and claims. When both locks are needed, + // acquire deliveryMu before retryMu. RPC calls run outside these locks. + deliveryMu sync.Mutex + deliveries map[receiptKey]Receipt + inFlight map[receiptKey]struct{} + claims map[*filterOwner]common.Hash } type pendingReceipt struct { receipt Receipt completed *Receipt // Learned candidate; receipt retains the original queue key. - filterer *filterOwner + owner *filterOwner attempts int nextRetryAt time.Time } @@ -583,17 +588,17 @@ func (s *subscriber) finalizeReceipts(blockNum *big.Int) error { return nil } -func (s *subscriber) addPendingReceipt(receipt Receipt, filterer *filterOwner) { +func (s *subscriber) addPendingReceipt(receipt Receipt, owner *filterOwner) { s.deliveryMu.Lock() defer s.deliveryMu.Unlock() - if !s.hasFilter(filterer) { + if !s.hasFilter(owner) { return } s.retryMu.Lock() defer s.retryMu.Unlock() txnHash := receipt.TransactionHash() - key := receiptOwner(receipt, filterer) + key := receiptOwner(receipt, owner) if !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { return } @@ -619,7 +624,7 @@ func (s *subscriber) addPendingReceipt(receipt Receipt, filterer *filterOwner) { s.pendingReceipts[key] = &pendingReceipt{ receipt: receipt, - filterer: filterer, + owner: owner, attempts: 1, nextRetryAt: time.Now().Add(1 * time.Second), // first retry after 1s } @@ -661,7 +666,7 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { wg.Add(1) go func(p *pendingReceipt) { defer wg.Done() - defer s.releaseReservation(p.receipt, p.filterer) + defer s.releaseReservation(p.receipt, p.owner) select { case sem <- struct{}{}: @@ -669,110 +674,116 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { case <-ctx.Done(): // If context is cancelled, release the claim so the item can be retried later. s.retryMu.Lock() - if current, ok := s.pendingReceipts[receiptOwner(p.receipt, p.filterer)]; ok && current == p { + if current, ok := s.pendingReceipts[receiptOwner(p.receipt, p.owner)]; ok && current == p { current.nextRetryAt = time.Now().Add(100 * time.Millisecond) // small delay to avoid immediate retry } s.retryMu.Unlock() return } - // Attempt to fetch the receipt - txnHash := p.receipt.TransactionHash() - key := receiptOwner(p.receipt, p.filterer) - s.retryMu.Lock() - candidate := p.receipt - if p.completed != nil { - candidate = *p.completed - } - s.retryMu.Unlock() - receipt, err := s.fetchReceipt(ctx, candidate) - - s.retryMu.Lock() - - // Check if the item still exists and is the same one we claimed. - currentPending, exists := s.pendingReceipts[key] - if !exists || currentPending != p { - s.retryMu.Unlock() - s.listener.log.Debug("Pending receipt is stale or already processed, skipping retry", "txnHash", txnHash.String()) - return - } + s.retryPendingReceipt(ctx, p) + }(pending) + } - if err != nil { - defer s.retryMu.Unlock() - if errors.Is(err, ethereum.NotFound) { - // Transaction genuinely doesn't exist - remove from queue - delete(s.pendingReceipts, key) - s.listener.log.Debug("Receipt not found after retry, removing from queue", "txnHash", txnHash.String()) - return - } + wg.Wait() +} - // Provider error - update retry state directly on the pointer. - currentPending.attempts++ - if currentPending.attempts >= maxReceiptRetryAttempts { - delete(s.pendingReceipts, key) - s.listener.log.Error( - "Failed to fetch receipt after max retries", - "txnHash", txnHash.String(), - "attempts", currentPending.attempts, - "error", err, - ) - // TODO: perhaps we should close the subscription here as we failed - // to deliver a receipt after many attempts? - return - } +// retryPendingReceipt fetches and commits one claimed pending receipt. The caller +// holds its semaphore slot and handles reservation cleanup. +func (s *subscriber) retryPendingReceipt(ctx context.Context, p *pendingReceipt) { + // Attempt to fetch the receipt + txnHash := p.receipt.TransactionHash() + key := receiptOwner(p.receipt, p.owner) + s.retryMu.Lock() + candidate := p.receipt + if p.completed != nil { + candidate = *p.completed + } + s.retryMu.Unlock() + receipt, err := s.fetchReceipt(ctx, candidate) - // Exponential backoff for next retry - backoff := time.Duration(1< maxWaitBetweenRetries { - backoff = maxWaitBetweenRetries - } - currentPending.nextRetryAt = time.Now().Add(backoff) - - s.listener.log.Debug( - "Receipt fetch failed, will retry", - "txnHash", txnHash.String(), - "attempt", currentPending.attempts, - "nextRetryIn", backoff, - ) - return - } + s.retryMu.Lock() - attempts := currentPending.attempts - s.retryMu.Unlock() + // Check if the item still exists and is the same one we claimed. + currentPending, exists := s.pendingReceipts[key] + if !exists || currentPending != p { + s.retryMu.Unlock() + s.listener.log.Debug("Pending receipt is stale or already processed, skipping retry", "txnHash", txnHash.String()) + return + } - s.deliveryMu.Lock() - s.retryMu.Lock() - currentPending, exists = s.pendingReceipts[key] - s.retryMu.Unlock() - if !exists || currentPending != p { - s.deliveryMu.Unlock() - return - } - published := s.publishLocked(ctx, receipt, p.filterer) - s.retryMu.Lock() - if current, exists := s.pendingReceipts[key]; exists && current == p { - if published || !s.hasFilter(p.filterer) || !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { - delete(s.pendingReceipts, key) - } else { - // A successful fetch can outlive its child deadline while waiting - // for delivery. Keep this exact valid owner available to a live retry. - current.completed = &receipt - current.nextRetryAt = time.Now().Add(100 * time.Millisecond) - } - } - s.retryMu.Unlock() - s.deliveryMu.Unlock() - if !published { - return - } + if err != nil { + defer s.retryMu.Unlock() + if errors.Is(err, ethereum.NotFound) { + // NotFound is terminal for missing receipts and obsolete block generations. + delete(s.pendingReceipts, key) + s.listener.log.Debug("Receipt not found after retry, removing from queue", "txnHash", txnHash.String()) + return + } - s.listener.log.Info( - "Successfully fetched receipt after retry", + // Provider error - update retry state directly on the pointer. + currentPending.attempts++ + if currentPending.attempts >= maxReceiptRetryAttempts { + delete(s.pendingReceipts, key) + s.listener.log.Error( + "Failed to fetch receipt after max retries", "txnHash", txnHash.String(), - "attempts", attempts, + "attempts", currentPending.attempts, + "error", err, ) - }(pending) + // TODO: perhaps we should close the subscription here as we failed + // to deliver a receipt after many attempts? + return + } + + // Exponential backoff for next retry + backoff := time.Duration(1< maxWaitBetweenRetries { + backoff = maxWaitBetweenRetries + } + currentPending.nextRetryAt = time.Now().Add(backoff) + + s.listener.log.Debug( + "Receipt fetch failed, will retry", + "txnHash", txnHash.String(), + "attempt", currentPending.attempts, + "nextRetryIn", backoff, + ) + return } - wg.Wait() + attempts := currentPending.attempts + s.retryMu.Unlock() + + s.deliveryMu.Lock() + s.retryMu.Lock() + currentPending, exists = s.pendingReceipts[key] + s.retryMu.Unlock() + if !exists || currentPending != p { + s.deliveryMu.Unlock() + return + } + published := s.publishLocked(ctx, receipt, p.owner) + s.retryMu.Lock() + if current, exists := s.pendingReceipts[key]; exists && current == p { + if published || !s.hasFilter(p.owner) || !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + delete(s.pendingReceipts, key) + } else { + // A successful fetch can outlive its child deadline while waiting + // for delivery. Keep this exact valid owner available to a live retry. + current.completed = &receipt + current.nextRetryAt = time.Now().Add(100 * time.Millisecond) + } + } + s.retryMu.Unlock() + s.deliveryMu.Unlock() + if !published { + return + } + + s.listener.log.Info( + "Successfully fetched receipt after retry", + "txnHash", txnHash.String(), + "attempts", attempts, + ) } From bea071b9e2c1283eda96501a145d1d27731fdee1 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 11:11:42 -0400 Subject: [PATCH 09/11] fix(ethreceipts): harden subscription cleanup and finality --- ethreceipts/ethreceipts.go | 121 +++++++++------------------------- ethreceipts/filterer.go | 24 ++++--- ethreceipts/finalizer.go | 2 +- ethreceipts/finalizer_test.go | 85 +++++++++++++++++++++++- ethreceipts/lifecycle_test.go | 54 +++++++++++++++ ethreceipts/receipt.go | 12 +--- ethreceipts/subscription.go | 3 +- 7 files changed, 182 insertions(+), 119 deletions(-) diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index 41ed5775..3f2d9862 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -40,36 +40,27 @@ const ( ) type Options struct { - // .. + // MaxConcurrentFetchReceiptWorkers limits concurrent receipt RPC requests. MaxConcurrentFetchReceiptWorkers int - // .. + // MaxConcurrentFilterWorkers limits concurrent block and filter processing. MaxConcurrentFilterWorkers int - // MaxConcurrentSearchOnChainWorkers is the maximum amount of concurrent - // on-chain searches (this is per subscriber) + // MaxConcurrentSearchOnChainWorkers limits on-chain searches per subscriber. MaxConcurrentSearchOnChainWorkers int - // .. + // PastReceiptsCacheSize limits the number of cached receipts. PastReceiptsCacheSize int - // .. + // NumBlocksToFinality is the number of blocks after mining required for finality. + // Values <= 0 select the network's finality policy when Run starts. NumBlocksToFinality int - // FilterMaxWaitNumBlocks is the maximum amount of blocks a filter will wait between getting - // a receipt filter match, before the filter will unsubscribe itself and stop listening. - // This value may be overriden by setting FilterCond#MaxListenNumBlocks on per-filter basis. - // - // NOTE: - // * value of -1 will use NumBlocksToFinality*2 - // * value of 0 will set no limit, so filter will always listen [default] - // * value of N will set the N number of blocks without results before unsubscribing between iterations + // FilterMaxWaitNumBlocks sets the default block wait between filter matches. + // Zero disables the limit. Individual filters can override it with MaxWait. FilterMaxWaitNumBlocks int - // Cache backend ... - // CacheBackend cachestore.Backend - - // Alerter config via github.com/goware/alerter + // Alerter receives listener and subscriber alerts. Alerter util.Alerter } @@ -82,7 +73,7 @@ type ReceiptsListener struct { chainID *big.Int br *breaker.Breaker - // fetchSem is used to limit amount of concurrenct fetch requests + // fetchSem limits concurrent receipt RPC requests. fetchSem chan struct{} // pastReceipts is a cache of past requested receipts @@ -93,7 +84,6 @@ type ReceiptsListener struct { // for us if they end up turning up. notFoundTxnHashes cachestore.Store[uint64] - // ... subscribers []*subscriber registerFiltersCh chan registerFilters filterSem chan struct{} @@ -142,12 +132,12 @@ func NewReceiptsListener(log *slog.Logger, provider ethrpc.Interface, monitor *e return nil, err } - notFoundTxnHashes, err := memcache.NewCacheWithSize[uint64](uint32(5000)) //, cachestore.WithDefaultKeyExpiry(2*time.Minute)) + notFoundTxnHashes, err := memcache.NewCacheWithSize[uint64](uint32(5000)) if err != nil { return nil, err } - // max ~12s total wait time before giving up + // Retry transient RPC failures with exponential backoff. br := breaker.New(log, 200*time.Millisecond, 1.2, 20) return &ReceiptsListener{ @@ -256,7 +246,7 @@ func (l *ReceiptsListener) Subscribe(filterQueries ...FilterQuery) Subscription }, } - subscriber.unsubscribe = func() { + subscriber.unsubscribe = sync.OnceFunc(func() { close(subscriber.done) subscriber.ch.Close() subscriber.ch.Flush() @@ -270,7 +260,7 @@ func (l *ReceiptsListener) Subscribe(filterQueries ...FilterQuery) Subscription return } } - } + }) l.subscribers = append(l.subscribers, subscriber) @@ -343,18 +333,13 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context sub := l.Subscribe(query) - // Use a WaitGroup to ensure the goroutine cleans up before the function returns - var wg sync.WaitGroup - workerDone := make(chan struct{}) exhausted := make(chan struct{}) mined := make(chan Receipt, 2) finalized := make(chan Receipt, 1) - found := uint32(0) finalityFunc := func(ctx context.Context) (*Receipt, error) { - // Wait for the goroutine to finish its cleanup before proceeding in finalityFunc, - // ensuring Unsubscribe has been called if the goroutine exited. + // Wait for the worker's cleanup, including Unsubscribe, before returning finality. select { case <-workerDone: case <-ctx.Done(): @@ -379,13 +364,8 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context } } - // TODO/NOTE: perhaps in an extended node failure. could there be a scenario - // where filterer.Exhausted is never hit? and this subscription never unsubscribes..? - // don't think so, but we can double check. - wg.Add(1) go func() { defer close(workerDone) - defer wg.Done() defer sub.Unsubscribe() defer close(mined) defer close(finalized) @@ -397,6 +377,8 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context } }() + found := false + filterExhausted := filterer.Exhausted() for { select { case <-ctx.Done(): @@ -406,19 +388,14 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // Subscription closed externally (less likely here, but good practice) return - case <-filterer.Exhausted(): - // Exhausted, check if we ever found a match. - if atomic.LoadUint32(&found) == 0 { - // Never found a match, signal exhaustion and exit. - close(exhausted) + case <-filterExhausted: + // Handle the closed exhaustion channel once. Already mined receipts + // keep waiting for finality until the subscription or context closes. + filterExhausted = nil + close(exhausted) + if !found { return } - // Found a match previously, but now exhausted. - // Allow loop to continue briefly to let finalizer potentially finish, - // but the finalized channel will eventually be closed if no final receipt comes. - // The finalityFunc will handle the exhausted state if needed. - // We signal exhaustion mainly for the initial return value check. - close(exhausted) case receipt, ok := <-sub.TransactionReceipt(): if !ok { @@ -426,7 +403,7 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context return } - atomic.StoreUint32(&found, 1) + found = true if receipt.Final { // Send to mined (in case caller only waits for mined) @@ -463,19 +440,17 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // Wait for the first mined receipt or an exit signal select { case <-ctx.Done(): - wg.Wait() // Ensure cleanup + <-workerDone return nil, nil, ctx.Err() case <-sub.Done(): - wg.Wait() // Ensure cleanup + <-workerDone return nil, nil, ErrSubscriptionClosed case <-exhausted: - // Exhausted before finding *any* receipt. - // finalityFunc will handle waiting and returning the exhaustion error. return nil, finalityFunc, superr.Wrap(ErrFilterExhausted, fmt.Errorf("txnHash=%s maxWait=%d", condTxnHash, condMaxWait)) case receipt, ok := <-mined: if !ok { // Mined channel closed without sending, implies goroutine exited early. - wg.Wait() // Ensure cleanup + <-workerDone // Check if exhaustion occurred select { case <-exhausted: @@ -484,8 +459,6 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context return nil, nil, ErrSubscriptionClosed } } - // Got the first mined receipt. Return it and the finality func. - // The finalityFunc will use wg.Wait() internally. return &receipt, finalityFunc, nil } } @@ -794,9 +767,7 @@ func (l *ReceiptsListener) listener(runCtx context.Context) error { } } } else { - // NOTE: even if a filter is exhausted, the finalizer will still run - // for those transactions which were previously mined and marked by the finalizer. - // Therefore, the code below will not impact the functionality of the finalizer. + // Exhaustion stops matching but preserves queued finality. maxWait := l.getMaxWaitBlocks(filterer.Options().MaxWait) blockNum := max(filterer.StartBlockNum(), filterer.LastMatchBlockNum()) @@ -824,13 +795,7 @@ func (l *ReceiptsListener) listener(runCtx context.Context) error { } }) - // TODO/NOTE: perhaps in an extended node failure. could there be a scenario - // where filterer.Exhausted is never hit? and this subscription never unsubscribes..? - // TODO: we ultimately need to check the monitor and if we get no new blocks for a period - // of time, then we can assume node problems.. even more helpful woudl be if the monitor - // gave us an error count of node failures, and we'd listen on that, and if we hit a threshold - // and our block number doesn't change after a period of time, then we return an error - // that we're exhausted due to a node failure. + // MaxWait counts blocks; callers should use a context deadline to bound stalls. return g.Wait() } @@ -888,11 +853,10 @@ func (l *ReceiptsListener) processBlockEvents(ctx context.Context, blocks ethmon generation = removed[block.Hash()].generation } - // TODOXXX: feels wasteful to build all receipts for all subscribers every time, but its okay for now receipts := make([]Receipt, len(block.Transactions())) logs := groupLogsByTransaction(block.Logs) - // build unfiltered complete receipts for each txn which include the transaction and the logs + // Build transaction candidates with the logs from the monitored block. for i, txn := range block.Transactions() { txnLog, ok := logs[txn.Hash().Hex()] if !ok { @@ -1044,7 +1008,6 @@ func (l *ReceiptsListener) queryFilterOnChain(ctx context.Context, subscriber *s receipt: r, generation: generation, // NOTE: we do not include the transaction at this point, as we don't have it. - // transaction: txn, Final: l.isBlockFinal(r.BlockNumber), } @@ -1157,20 +1120,6 @@ func collectOk[T any](in []T, oks []bool, okCond bool) []T { return out } -// func txnLogs(blockLogs []types.Log, txnHash ethkit.Hash) []*types.Log { -// txnLogs := []*types.Log{} -// for i, log := range blockLogs { -// if log.TxHash == txnHash { -// log := log // copy -// txnLogs = append(txnLogs, &log) -// if i+1 >= len(blockLogs) || blockLogs[i+1].TxHash != txnHash { -// break -// } -// } -// } -// return txnLogs -// } - func groupLogsByTransaction(logs []types.Log) map[string][]*types.Log { var out = make(map[string][]*types.Log) for _, log := range logs { @@ -1188,16 +1137,6 @@ func groupLogsByTransaction(logs []types.Log) map[string][]*types.Log { return out } -// func blockLogsCount(numTxns int, logs []types.Log) uint { -// var max uint = uint(numTxns) -// for _, log := range logs { -// if log.TxIndex+1 > max { -// max = log.TxIndex + 1 -// } -// } -// return max -// } - // A removed hash can later be canonical again. Generations invalidate work // started before rollback even when its transaction and block hash are unchanged. type blockRef struct { diff --git a/ethreceipts/filterer.go b/ethreceipts/filterer.go index 2a40a804..28046694 100644 --- a/ethreceipts/filterer.go +++ b/ethreceipts/filterer.go @@ -120,22 +120,21 @@ type FilterQuery interface { QueryOnChain(func(context.Context) (*types.Receipt, error)) FilterQuery MaxWait(int) FilterQuery - // DEPRECATED: please use QueryOnChainTxnHash instead, which is the same thing, renamed to be more clear - // in addition, see new QueryChain(fn) as additional feature. + // Deprecated: use QueryOnChainTxnHash instead. SearchOnChain(bool) FilterQuery } type FilterOptions struct { - // .. + // ID is a caller-supplied label, not a unique registration identity. ID uint64 - // .. + // Finalize requests a final receipt after the mined receipt. Finalize bool - // . + // LimitOne stops matching after one mined receipt and preserves queued finality. LimitOne bool - // .. + // SearchCache searches the monitor's retained blocks when the filter is registered. SearchCache bool // QueryOnChainTxnHash will query the chain for the txn hash at the start @@ -144,17 +143,16 @@ type FilterOptions struct { // mined before the filter was created. QueryOnChainTxnHash bool - // .. + // QueryOnChain optionally fetches a receipt when the filter is registered. QueryOnChain func(context.Context) (*types.Receipt, error) - // MaxWait filter option waits some number of blocks without a filter match after - // which point will auto-unsubscribe the filter. This is useful to help automatically - // remove filters which likely won't come up. + // MaxWait limits blocks without a match before matching is exhausted. + // Receipts already queued for finality are still finalized. // // nil : use the ReceiptsListener option FilterMaxWaitNumBlocks value as the default - // -1 : set value to ReceiptsListener option NumFinality * 3 + // <0 : use ReceiptsListener option NumBlocksToFinality * 2 // 0 : option is disabled, and has no limit on wait. filters need to be manually unsubscribed - // N : a specified number of blocks without a match before unsusbcribe + // N : a specified number of blocks without a match before exhaustion MaxWait *int } @@ -207,7 +205,7 @@ func (f *filter) SearchCache(searchCache bool) FilterQuery { return f } -// DEPRECATED: please use QueryChainForTxnHash instead, which is the same thing, renamed to be more clear +// Deprecated: use QueryOnChainTxnHash instead. func (f *filter) SearchOnChain(searchOnChain bool) FilterQuery { f.options.QueryOnChainTxnHash = searchOnChain return f diff --git a/ethreceipts/finalizer.go b/ethreceipts/finalizer.go index 40ee9f12..85ed7dcf 100644 --- a/ethreceipts/finalizer.go +++ b/ethreceipts/finalizer.go @@ -120,7 +120,7 @@ func (f *finalizer) dequeue(currentBlockNum *big.Int) []finalTxn { var finalized []finalTxn retained := f.queue[:0] for _, txn := range f.queue { - if currentBlockNum.Cmp(new(big.Int).Add(txn.blockNum, f.numBlocksToFinality)) > 0 { + if currentBlockNum.Cmp(new(big.Int).Add(txn.blockNum, f.numBlocksToFinality)) >= 0 { finalized = append(finalized, txn) delete(f.txns, receiptOwner(txn.receipt, txn.receipt.owner)) } else { diff --git a/ethreceipts/finalizer_test.go b/ethreceipts/finalizer_test.go index 407abd67..70eb3c00 100644 --- a/ethreceipts/finalizer_test.go +++ b/ethreceipts/finalizer_test.go @@ -11,6 +11,89 @@ import ( "github.com/0xsequence/ethkit/go-ethereum/core/types" ) +func TestFinalizerFinalityBoundary(t *testing.T) { + tx, _, _ := hardeningTxn(t, 3000) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions(), b, hardeningBlock(101), hardeningBlock(102)) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + // Queue the receipt as it would have been at block 100, before head 102. + s.finalizer.enqueue(owner, Receipt{receipt: hardeningReceipt(b, tx), owner: owner, Filter: owner.Filterer}, b.Number()) + if err := s.finalizeReceipts(big.NewInt(101)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + if !l.isBlockFinal(b.Number()) { + t.Fatal("block is not final at the two-block threshold") + } + if err := s.finalizeReceipts(big.NewInt(102)); err != nil { + t.Fatal(err) + } + r := hardeningRead(t, s) + if !r.Final || r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q { + t.Fatal("wrong final receipt at the two-block threshold") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} + +func TestFetchFinalityAfterExhaustion(t *testing.T) { + tx, _, _ := hardeningTxn(t, 3001) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + hardeningStart(t, l) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + mined, waitFinal, err := l.FetchTransactionReceiptWithFinality(ctx, tx.Hash(), 1) + if err != nil { + t.Fatal(err) + } + if mined == nil || mined.Final || mined.Reorged || mined.BlockHash() != b.Hash() { + t.Fatalf("wrong mined receipt: %+v", mined) + } + l.mu.Lock() + s := l.subscribers[0] + l.mu.Unlock() + f := builtinFilter(s.Filters()[0]) + // A shallow reorg can exhaust matching while the mined block's finality + // remains queued. Use the same exhaustion steps as the listener. + s.exhaustFilter(f) + f.closeExhausted() + done := make(chan struct{}) + var final *Receipt + var finalErr error + go func() { + defer close(done) + final, finalErr = waitFinal(ctx) + }() + select { + case <-done: + t.Fatalf("finality stopped before its threshold: %v", finalErr) + case <-time.After(100 * time.Millisecond): + } + if err := s.finalizeReceipts(big.NewInt(102)); err != nil { + t.Fatal(err) + } + select { + case <-done: + if finalErr != nil || final == nil || !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || final.BlockHash() != b.Hash() { + t.Fatalf("wrong final receipt after exhaustion: %+v, error: %v", final, finalErr) + } + case <-ctx.Done(): + t.Fatal("finality did not complete after exhaustion") + } + if l.NumSubscribers() != 0 { + t.Error("fetch helper did not unsubscribe after finality") + } +} + func TestHardeningSubscribeBeforeRunFinality(t *testing.T) { tx, _, _ := hardeningTxn(t, 2) b := hardeningBlock(100, tx) @@ -61,7 +144,7 @@ func TestHardeningFinalizerOrderAndOwners(t *testing.T) { head int64 tx *types.Transaction b *ethmonitor.Block - }{{103, tx2, older}, {104, tx1, newer}} { + }{{102, tx2, older}, {103, tx1, newer}} { if err := s.finalizeReceipts(big.NewInt(tc.head)); err != nil { t.Fatal(err) } diff --git a/ethreceipts/lifecycle_test.go b/ethreceipts/lifecycle_test.go index 24ae4ee1..63907f7f 100644 --- a/ethreceipts/lifecycle_test.go +++ b/ethreceipts/lifecycle_test.go @@ -17,6 +17,60 @@ import ( "github.com/goware/breaker" ) +func TestSubscriptionUnsubscribe(t *testing.T) { + for _, workers := range []int{1, 8} { + name := "repeated" + if workers > 1 { + name = "concurrent" + } + t.Run(name, func(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + s := l.Subscribe() + other := l.Subscribe() + defer other.Unsubscribe() + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + defer func() { + if p := recover(); p != nil { + t.Errorf("Unsubscribe panicked: %v", p) + } + }() + <-start + s.Unsubscribe() + s.Unsubscribe() + }() + } + close(start) + wg.Wait() + if l.NumSubscribers() != 1 { + t.Fatal("Unsubscribe did not remove exactly one subscription") + } + select { + case <-s.Done(): + default: + t.Error("unsubscribed Done channel remains open") + } + select { + case _, ok := <-s.TransactionReceipt(): + if ok { + t.Error("unsubscribed receipt channel remains open") + } + case <-time.After(time.Second): + t.Error("unsubscribed receipt channel did not close") + } + select { + case <-other.Done(): + t.Error("Unsubscribe closed an unrelated subscription") + default: + } + }) + } +} + func TestHardeningStopCancelsStartup(t *testing.T) { entered := make(chan struct{}) p := &hardeningProvider{chainID: func(ctx context.Context) (*big.Int, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() }} diff --git a/ethreceipts/receipt.go b/ethreceipts/receipt.go index 9cfb918b..94af72df 100644 --- a/ethreceipts/receipt.go +++ b/ethreceipts/receipt.go @@ -26,8 +26,7 @@ type Receipt struct { generation uint64 owner *filterOwner - // TODOXXX: this intermediate type is lame.. with new ethrpc we can remove - // NOTE: we only use this for From/To address resolution currently + // Cache transaction message decoding for From/To address resolution. message atomic.Value } @@ -193,17 +192,8 @@ func (r *Receipt) AsMessage() (*core.Message, error) { return nil, fmt.Errorf("ethreceipts: transaction and chainID required to resolve message") } - // TODOXXX: avoid using AsMessage as its fairly expensive operation, especially - // to do it for every txn for every filter. - // TODO: in order to do this, we'll have to update ethrpc with a different - // implementation to just use raw types, aka, ethrpc/types.go with Block/Transaction/Receipt/Log .. txnMsg, err := ethtxn.AsMessage(r.transaction, r.chainID) if err != nil { - // NOTE: this should never happen, but lets log in case it does. In the - // future, we should just not use go-ethereum for these types. - // l.log.Warn(fmt.Sprintf("unexpected failure of txn (%s index %d) on block %d (total txns=%d) AsMessage(..): %s", - // txn.Hash(), i, block.NumberU64(), len(block.Transactions()), err, - // )) return nil, err } r.message.Store(txnMsg) diff --git a/ethreceipts/subscription.go b/ethreceipts/subscription.go index f4d554b4..ed55b382 100644 --- a/ethreceipts/subscription.go +++ b/ethreceipts/subscription.go @@ -34,6 +34,7 @@ var ( type Subscription interface { TransactionReceipt() <-chan Receipt Done() <-chan struct{} + // Unsubscribe closes the subscription. Repeated and concurrent calls are safe. Unsubscribe() Filters() []Filterer @@ -731,8 +732,6 @@ func (s *subscriber) retryPendingReceipt(ctx context.Context, p *pendingReceipt) "attempts", currentPending.attempts, "error", err, ) - // TODO: perhaps we should close the subscription here as we failed - // to deliver a receipt after many attempts? return } From e007e0f82e04f0beb1b2e1f922cba8eb67aa884f Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 12:00:52 -0400 Subject: [PATCH 10/11] fix(ethreceipts): preserve fetch filter compatibility --- ethmonitor/prefetch_internal_test.go | 9 ++- ethreceipts/ethreceipts.go | 26 ++++--- ethreceipts/helpers_test.go | 37 +++++---- ethreceipts/receipt_fetch_test.go | 108 +++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 26 deletions(-) diff --git a/ethmonitor/prefetch_internal_test.go b/ethmonitor/prefetch_internal_test.go index 0bdb593a..65bb4966 100644 --- a/ethmonitor/prefetch_internal_test.go +++ b/ethmonitor/prefetch_internal_test.go @@ -187,7 +187,14 @@ func TestPrefetchPanicStopsWorkers(t *testing.T) { defer close(done) monitor.prefetch.run(ctx) }() - defer func() { cancel(); <-done }() + defer func() { + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Error("prefetch did not stop after cancellation") + } + }() select { case <-done: case <-time.After(time.Second): diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index 3f2d9862..7c411654 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -309,16 +309,20 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // open to listen to many similar receipts, use .Subscribe(filter) directly instead. source, ok := filter.(Filterer) if !ok { - return nil, nil, fmt.Errorf("ethreceipts: unable to cast Filterer from FilterQuery") + // Builders may expose a Filterer only after their normal option chaining. + filter = filter.LimitOne(true).SearchCache(true) + if len(optFilterFinalize) > 0 && optFilterFinalize[0] { + filter = filter.Finalize(true) + } + source, ok = filter.(Filterer) + if !ok { + return nil, nil, fmt.Errorf("ethreceipts: unable to cast Filterer from FilterQuery") + } } - query := snapshotFilter(source).LimitOne(true).SearchCache(true) + filterer := snapshotFilter(source) + filterer.LimitOne(true).SearchCache(true) if len(optFilterFinalize) > 0 && optFilterFinalize[0] { - query = query.Finalize(true) - } - - filterer, ok := query.(Filterer) - if !ok { - return nil, nil, fmt.Errorf("ethreceipts: unable to cast Filterer from FilterQuery") + filterer.Finalize(true) } condMaxWait := 0 @@ -328,10 +332,10 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context condTxnHash := "" if filterer.Cond().TxnHash != nil { condTxnHash = (*filterer.Cond().TxnHash).String() - query = query.QueryOnChainTxnHash(true) + filterer.QueryOnChainTxnHash(true) } - sub := l.Subscribe(query) + sub := l.Subscribe(filterer) workerDone := make(chan struct{}) exhausted := make(chan struct{}) @@ -404,6 +408,8 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context } found = true + // Helper state stays private; public receipts retain the resolved source filter. + receipt.Filter = source if receipt.Final { // Send to mined (in case caller only waits for mined) diff --git a/ethreceipts/helpers_test.go b/ethreceipts/helpers_test.go index 290bc539..e01ab671 100644 --- a/ethreceipts/helpers_test.go +++ b/ethreceipts/helpers_test.go @@ -3,6 +3,7 @@ package ethreceipts import ( "context" "encoding/json" + "errors" "io" "log/slog" "math/big" @@ -92,30 +93,36 @@ func hardeningListener(t *testing.T, p *hardeningProvider, opts Options, blocks func hardeningStart(t *testing.T, l *ReceiptsListener) { t.Helper() - done := make(chan error, 1) - go func() { done <- l.Run(context.Background()) }() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + var runErr error + t.Cleanup(func() { + cancel() + l.Stop() + select { + case <-done: + if runErr != nil && !errors.Is(runErr, context.Canceled) { + t.Errorf("Run: %v", runErr) + } + case <-time.After(time.Second): + t.Error("Run did not stop") + } + }) + go func() { + defer close(done) + runErr = l.Run(ctx) + }() deadline := time.After(time.Second) for l.monitor.NumSubscribers() != 1 { select { - case err := <-done: - t.Fatalf("Run exited before subscription: %v", err) + case <-done: + t.Fatalf("Run exited before subscription: %v", runErr) case <-deadline: t.Fatal("Run did not subscribe to monitor") default: time.Sleep(time.Millisecond) } } - t.Cleanup(func() { - l.Stop() - select { - case err := <-done: - if err != nil { - t.Errorf("Run: %v", err) - } - case <-time.After(time.Second): - t.Error("Run did not stop") - } - }) } func hardeningRead(t *testing.T, s Subscription) Receipt { diff --git a/ethreceipts/receipt_fetch_test.go b/ethreceipts/receipt_fetch_test.go index 4a4d63d0..59c9b634 100644 --- a/ethreceipts/receipt_fetch_test.go +++ b/ethreceipts/receipt_fetch_test.go @@ -3,6 +3,7 @@ package ethreceipts import ( "context" "errors" + "math/big" "sync" "sync/atomic" "testing" @@ -61,6 +62,113 @@ func TestHardeningFetchPreservesSharedQuery(t *testing.T) { } } +// This builder becomes a Filterer only after the optional Finalize step. +type fetchFinalizeBuilder struct{ FilterQuery } + +func (b *fetchFinalizeBuilder) LimitOne(v bool) FilterQuery { + b.FilterQuery = b.FilterQuery.LimitOne(v) + return b +} + +func (b *fetchFinalizeBuilder) SearchCache(v bool) FilterQuery { + b.FilterQuery = b.FilterQuery.SearchCache(v) + return b +} + +func TestFetchFilterCompatibility(t *testing.T) { + for _, name := range []string{"builtin", "custom_pointer", "custom_value", "builder", "finalize_builder"} { + t.Run(name, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 3002) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), b) + var calls atomic.Int32 + match := func(context.Context, Receipt) (bool, error) { calls.Add(1); return true, nil } + base := FilterLogs(func([]*types.Log) bool { calls.Add(1); return true }).MaxWait(4).(Filterer) + var query FilterQuery = base + var public Filterer = base + builder := false + switch name { + case "custom_pointer", "custom_value": + // Only the custom Match accepts receipts, proving it survives the snapshot. + base = FilterLogs(func([]*types.Log) bool { return false }).MaxWait(4).(Filterer) + custom := ownershipValueFilter{base, []string{"fetch"}, match} + if name == "custom_pointer" { + public = &custom + } else { + public = custom + } + query = public + case "builder": + query = struct{ FilterQuery }{base} + builder = true + case "finalize_builder": + query = &fetchFinalizeBuilder{base} + builder = true + } + var other Subscription + if !builder { + other = l.Subscribe(public) + defer other.Unsubscribe() + } + hardeningStart(t, l) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + mined, waitFinal, err := l.FetchTransactionReceiptWithFilter(ctx, query, true) + if err != nil { + t.Fatal(err) + } + if mined == nil || mined.Final || mined.Reorged || mined.TransactionHash() != tx.Hash() || !sameFilter(mined.Filter, public) { + t.Fatalf("wrong mined receipt or public filter: %+v", mined) + } + if calls.Load() == 0 { + t.Fatal("source matching behavior was not used") + } + if !builder { + opts := public.Options() + if opts.LimitOne || opts.SearchCache || opts.Finalize || opts.MaxWait == nil || *opts.MaxWait != 4 { + t.Fatalf("fetch mutated shared filter options: %+v", opts) + } + } + if public.StartBlockNum() != 0 || public.LastMatchBlockNum() != 0 { + t.Fatal("fetch mutated public filter counters") + } + select { + case <-public.Exhausted(): + t.Fatal("fetch exhausted the public filter") + default: + } + l.mu.Lock() + var helper *subscriber + for _, s := range l.subscribers { + if s != other { + helper = s + } + } + l.mu.Unlock() + if helper == nil { + t.Fatal("fetch helper unsubscribed before finality") + } + if err := helper.finalizeReceipts(big.NewInt(102)); err != nil { + t.Fatal(err) + } + final, err := waitFinal(ctx) + if err != nil || final == nil || !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || !sameFilter(final.Filter, public) { + t.Fatalf("wrong final receipt or public filter: %+v, error: %v", final, err) + } + wantSubscribers := 0 + if other != nil { + wantSubscribers = 1 + } + if l.NumSubscribers() != wantSubscribers { + t.Error("fetch completion removed the wrong subscription") + } + }) + } +} + type ownershipGateFilter struct { Filterer entered, release chan struct{} From 584f585fae476980227926864174dbd8538d2845 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 12:41:22 -0400 Subject: [PATCH 11/11] fix(ethmonitor): reject invalid empty log responses --- ethmonitor/ethmonitor.go | 21 +++++++++++---------- ethmonitor/prefetch_internal_test.go | 8 ++++---- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/ethmonitor/ethmonitor.go b/ethmonitor/ethmonitor.go index d59c4898..37acf283 100644 --- a/ethmonitor/ethmonitor.go +++ b/ethmonitor/ethmonitor.go @@ -869,16 +869,9 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics if err != nil { return nil, err } - if blockBloom != (types.Bloom{}) && (len(logsPayload) == 0 || (len(logsPayload) == 2 && logsPayload[0] == '[' && logsPayload[1] == ']')) { - // If we have no logs and the block bloom is set, then we need to return an error - // as the node is incorrectly telling us the block-logs response is '[]' but in fact - // the block log bloom filter tells us we should be expecting logs. We do this to - // ensure we do not incorrectly cache an empty block-logs response as valid. - return nil, fmt.Errorf("ethmonitor: filterLogs detected empty block-logs response but block bloom is set, ignoring node response") - } // Validate before caching so a malformed response cannot block log // backfilling until cache expiry. - fetchedLogs, err = m.unmarshalLogs(logsPayload) + fetchedLogs, err = m.unmarshalLogs(logsPayload, blockBloom) if err != nil { return nil, err } @@ -901,7 +894,7 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics if fetchedLogs != nil { return fetchedLogs, resp, nil } - logs, err := m.unmarshalLogs(resp) + logs, err := m.unmarshalLogs(resp, blockBloom) if err != nil { // Recover entries cached by peers or older monitors that did not // validate logs before writing them. @@ -1424,11 +1417,19 @@ func (m *Monitor) unmarshalBlock(blockPayload []byte) (*types.Block, error) { return block, nil } -func (m *Monitor) unmarshalLogs(logsPayload []byte) ([]types.Log, error) { +func (m *Monitor) unmarshalLogs(logsPayload []byte, blockBloom types.Bloom) ([]types.Log, error) { var logs []types.Log err := json.Unmarshal(logsPayload, &logs) if err != nil { return nil, err } + // JSON null decodes without error, but a logs response must be an array. + if logs == nil { + return nil, fmt.Errorf("ethmonitor: filterLogs expected a JSON array of block logs") + } + // Apply the bloom guard to decoded responses, including cache hits. + if len(logs) == 0 && blockBloom != (types.Bloom{}) { + return nil, fmt.Errorf("ethmonitor: filterLogs detected empty block-logs response but block bloom is set, ignoring node response") + } return logs, nil } diff --git a/ethmonitor/prefetch_internal_test.go b/ethmonitor/prefetch_internal_test.go index 65bb4966..736f3e6b 100644 --- a/ethmonitor/prefetch_internal_test.go +++ b/ethmonitor/prefetch_internal_test.go @@ -40,7 +40,7 @@ func TestPrefetchRejectsInvalidBlockBeforeCaching(t *testing.T) { } func TestMonitorRecoversInvalidPrefetchedLogs(t *testing.T) { - for _, payload := range []string{`{}`, `[{}]`} { + for _, payload := range []string{`{}`, `[{}]`, `null`, `[]`, `[ ]`} { for _, cached := range []bool{false, true} { t.Run(fmt.Sprintf("payload=%s/cached=%v", payload, cached), func(t *testing.T) { const first, target, last = uint64(1000), uint64(1002), uint64(1004) @@ -114,12 +114,12 @@ func TestMonitorRecoversInvalidPrefetchedLogs(t *testing.T) { require.NoError(t, monitor.cache.SetEx(context.Background(), key, []byte(payload), time.Hour)) } else { // Complete the speculative fetch before starting the serial loop, - // so the worker deterministically receives the malformed response. + // so the worker deterministically receives the invalid response. monitor.prefetch.fetch(context.Background(), prefetchJob{num: target}) require.Equal(t, int64(1), originCalls.Load()) _, found, err := monitor.cache.Get(context.Background(), key) require.NoError(t, err) - require.False(t, found, "failed prefetch must not cache malformed logs") + require.False(t, found, "failed prefetch must not cache invalid logs") } sub := monitor.Subscribe("TestMonitorRecoversInvalidPrefetchedLogs") @@ -157,7 +157,7 @@ func TestMonitorRecoversInvalidPrefetchedLogs(t *testing.T) { } } if !cached { - require.GreaterOrEqual(t, originCalls.Load(), int64(2), "malformed prefetch must allow an origin retry") + require.GreaterOrEqual(t, originCalls.Load(), int64(2), "invalid prefetch must allow an origin retry") } else { require.Positive(t, originCalls.Load(), "invalid cache entry must allow an origin retry") }