From cf8c4c191a3e6acafd92d7ac1d3dd80bb8071303 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Wed, 30 Sep 2026 13:23:47 -0400 Subject: [PATCH 01/11] ethmonitor: parallel block prefetch for fast chains The monitor fetches each block and its logs serially, which caps ingestion at roughly 1 / (2 x node latency) blocks per second. On chains producing blocks faster than that (robinhood mainnet at ~10 blocks/s) the monitor falls behind and never recovers: node-gateway on dev was serving a head ~14 minutes stale. Add an opt-in prefetcher (Options.PrefetchConcurrency, default 0 = off, and Options.PrefetchWindow, default 4x concurrency). While the monitor trails the head, N workers fetch the blocks past its next block, and their logs, into the cache under the existing cache keys. The serial run loop is unchanged and still the only place the canonical chain is built; it just reads prefetched payloads as cache hits. At the head the prefetcher is idle, and in polling mode it only polls the head while the run loop is catching up, so slow chains see no extra node calls. Reorg safety: - with prefetch on, a block that does not extend the head is confirmed with a direct, uncached node fetch before it is treated as a reorg - a reorg resets the prefetcher and drops its cached blocks above the reverted block; a generation counter discards in-flight stale results Also, affecting all monitors: - block payloads are decoded before caching, and undecodable cache entries are dropped, so one bad node response can't poison a block number until cache expiry - Run owns a child context and joins its publisher, head listeners and prefetcher before returning, so goroutines don't outlive a fatal exit and a later Run - fetchNextBlock reads nextBlockNumber under its lock Co-Authored-By: Claude Opus 5.5 (1M context) --- ethmonitor/ethmonitor.go | 244 ++++++-- ethmonitor/prefetch.go | 286 ++++++++++ ethmonitor/prefetch_internal_test.go | 61 ++ ethmonitor/prefetch_test.go | 812 +++++++++++++++++++++++++++ 4 files changed, 1360 insertions(+), 43 deletions(-) create mode 100644 ethmonitor/prefetch.go create mode 100644 ethmonitor/prefetch_internal_test.go create mode 100644 ethmonitor/prefetch_test.go diff --git a/ethmonitor/ethmonitor.go b/ethmonitor/ethmonitor.go index 8f7bd916..c1c70b84 100644 --- a/ethmonitor/ethmonitor.go +++ b/ethmonitor/ethmonitor.go @@ -19,6 +19,7 @@ import ( "github.com/0xsequence/ethkit/go-ethereum/core/types" "github.com/0xsequence/ethkit/util" "github.com/goware/breaker" + memcache "github.com/goware/cachestore-mem" cachestore "github.com/goware/cachestore2" "github.com/goware/channel" "github.com/goware/superr" @@ -41,6 +42,8 @@ var DefaultOptions = Options{ LogTopics: []common.Hash{}, // all logs DebugLogging: false, CacheExpiry: 600 * time.Second, + PrefetchConcurrency: 0, // Prefetching is disabled by default + PrefetchWindow: 0, // 4x PrefetchConcurrency set on init Alerter: util.NoopAlerter(), } @@ -104,6 +107,18 @@ type Options struct { // CacheExpiry is how long to keep each record in cache CacheExpiry time.Duration + // PrefetchConcurrency is the number of workers fetching blocks (and their + // logs, when WithLogs is set) ahead of the monitor while it trails the chain + // head. Prefetched payloads land in the cache, so the monitor's serial loop + // reads them as cache hits instead of paying node round-trips per block. + // Useful on chains whose block rate outpaces a serial fetch. If no + // CacheBackend is set, an in-memory cache is used. 0 disables prefetching. + PrefetchConcurrency int + + // PrefetchWindow is how many blocks past the monitor's next block the + // prefetcher may fetch. Defaults to 4x PrefetchConcurrency. + PrefetchWindow int + // Alerter config via github.com/goware/alerter Alerter util.Alerter @@ -135,7 +150,16 @@ type Monitor struct { pollInterval atomic.Int64 isStreamingMode atomic.Bool - cache cachestore.Store[[]byte] + // hitStreak counts the run loop's consecutive fetches which found the + // next block without a miss, capped to avoid overflow. + hitStreak atomic.Int32 + + // latestHead is the most recent chain head number seen, from the + // newHeads stream or, in polling mode, the prefetcher's head poll. + latestHead atomic.Uint64 + + cache cachestore.Store[[]byte] + prefetch *prefetcher publishCh chan Blocks publishQueue *queue @@ -172,6 +196,25 @@ func NewMonitor(provider ethrpc.RawInterface, options ...Options) (*Monitor, err // with slog, we can't modify the log level after the logger is created. } + if opts.PrefetchConcurrency < 0 { + opts.PrefetchConcurrency = 0 + } + if opts.PrefetchConcurrency > 0 && opts.PrefetchWindow <= 0 { + opts.PrefetchWindow = 4 * opts.PrefetchConcurrency + } + + // prefetching hands payloads to the monitor loop through the cache, + // so it needs one even when the caller didn't configure a backend. + if opts.PrefetchConcurrency > 0 && opts.CacheBackend == nil { + // room for a block-by-number and a logs entry per block in the + // window, plus headroom for by-hash lookups during reorgs. + backend, err := memcache.NewBackend(uint32(4*opts.PrefetchWindow + 256)) + if err != nil { + return nil, fmt.Errorf("ethmonitor: creating prefetch cache: %w", err) + } + opts.CacheBackend = backend + } + var cache cachestore.Store[[]byte] if opts.CacheBackend != nil { if opts.CacheExpiry == 0 { @@ -180,7 +223,7 @@ func NewMonitor(provider ethrpc.RawInterface, options ...Options) (*Monitor, err cache = cachestore.OpenStore[[]byte](opts.CacheBackend, cachestore.WithDefaultKeyExpiry(opts.CacheExpiry)) } - return &Monitor{ + m := &Monitor{ options: opts, log: opts.Logger, alert: opts.Alerter, @@ -191,7 +234,11 @@ func NewMonitor(provider ethrpc.RawInterface, options ...Options) (*Monitor, err publishCh: make(chan Blocks), publishQueue: newQueue(opts.BlockRetentionLimit * 2), subscribers: make([]*subscriber, 0), - }, nil + } + if opts.PrefetchConcurrency > 0 { + m.prefetch = newPrefetcher(m, opts.PrefetchConcurrency, opts.PrefetchWindow) + } + return m, nil } func (m *Monitor) lazyInit(ctx context.Context) error { @@ -221,10 +268,19 @@ func (m *Monitor) Run(ctx context.Context) error { return fmt.Errorf("ethmonitor: already running") } - m.ctx, m.ctxStop = context.WithCancel(ctx) + ctx, cancel := context.WithCancel(ctx) + m.ctx, m.ctxStop = ctx, cancel atomic.StoreInt32(&m.running, 1) defer atomic.StoreInt32(&m.running, 0) + var wg sync.WaitGroup + defer func() { + cancel() + wg.Wait() + }() + m.hitStreak.Store(0) + m.latestHead.Store(0) + m.isStreamingMode.Store(false) if err := m.lazyInit(ctx); err != nil { return err @@ -265,7 +321,9 @@ func (m *Monitor) Run(ctx context.Context) error { } // Broadcast published events to all subscribers + wg.Add(1) go func() { + defer wg.Done() defer func() { if r := recover(); r != nil { m.log.Error(fmt.Sprintf("ethmonitor: panic in publish loop: %v - stack: %s", r, string(debug.Stack()))) @@ -289,7 +347,7 @@ func (m *Monitor) Run(ctx context.Context) error { }() // Monitor the chain for canonical representation - err := m.monitor() + err := m.monitor(ctx, &wg) if m.options.UnsubscribeOnStop { m.UnsubscribeAll(err) } @@ -325,13 +383,33 @@ func (m *Monitor) IsStreamingEnabled() bool { func (m *Monitor) IsStreamingMode() bool { return m.isStreamingMode.Load() } -func (m *Monitor) listenNewHead() <-chan uint64 { + +// nextBlockNum returns the number of the next block the run loop will fetch, +// or false while the monitor has not settled on one yet. +func (m *Monitor) nextBlockNum() (uint64, bool) { + m.nextBlockNumberMu.Lock() + defer m.nextBlockNumberMu.Unlock() + if m.nextBlockNumber == nil || !m.nextBlockNumber.IsUint64() { + return 0, false + } + return m.nextBlockNumber.Uint64(), true +} + +// isCatchingUp reports whether the run loop found its next block on at least +// its last two fetches in a row, ie. the chain has been ahead of it. +func (m *Monitor) isCatchingUp() bool { + return m.hitStreak.Load() >= 2 +} + +func (m *Monitor) listenNewHead(ctx context.Context, wg *sync.WaitGroup) <-chan uint64 { ch := make(chan uint64) var latestHeadBlock atomic.Uint64 nextBlock := make(chan uint64) + wg.Add(2) go func() { + defer wg.Done() defer func() { if r := recover(); r != nil { m.log.Error(fmt.Sprintf("ethmonitor: panic in new head loop: %v - stack: %s", r, string(debug.Stack()))) @@ -376,18 +454,23 @@ func (m *Monitor) listenNewHead() <-chan uint64 { m.isStreamingMode.Store(true) newHeads := make(chan *types.Header) - sub, err := m.provider.SubscribeNewHeads(m.ctx, newHeads) + sub, err := m.provider.SubscribeNewHeads(ctx, newHeads) if err != nil { m.log.Warn(fmt.Sprintf("ethmonitor (chain %s): websocket connect failed: %v", m.chainID.String(), err)) m.alert.Alert(context.Background(), "ethmonitor (chain %s): websocket connect failed: %v", m.chainID.String(), err) - time.Sleep(2000 * time.Millisecond) + select { + case <-ctx.Done(): + close(nextBlock) + return + case <-time.After(2 * time.Second): + } streamingErrLastTime = time.Now() goto reconnect } for { select { - case <-m.ctx.Done(): + case <-ctx.Done(): // if we're done, we'll unsubscribe and close the nextBlock channel sub.Unsubscribe() close(nextBlock) @@ -404,6 +487,10 @@ func (m *Monitor) listenNewHead() <-chan uint64 { case newHead := <-newHeads: latestHeadBlock.Store(newHead.Number.Uint64()) + m.latestHead.Store(newHead.Number.Uint64()) + if m.prefetch != nil { + m.prefetch.notify() + } select { case nextBlock <- newHead.Number.Uint64(): default: @@ -434,7 +521,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { // Polling mode, where we poll for the latest block number select { - case <-m.ctx.Done(): + case <-ctx.Done(): // if we're done, we'll close the nextBlock channel close(nextBlock) retryStreamingTimer.Stop() @@ -443,7 +530,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { case <-time.After(time.Duration(m.pollInterval.Load())): select { case nextBlock <- 0: - case <-m.ctx.Done(): + case <-ctx.Done(): } } } @@ -452,6 +539,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { // The main loop which notifies the monitor to continue to the next block go func() { + defer wg.Done() defer func() { if r := recover(); r != nil { m.log.Error(fmt.Sprintf("ethmonitor: panic in next block loop: %v - stack: %s", r, string(debug.Stack()))) @@ -461,7 +549,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { for { select { - case <-m.ctx.Done(): + case <-ctx.Done(): return default: } @@ -479,20 +567,23 @@ func (m *Monitor) listenNewHead() <-chan uint64 { // waiting on the nextBlock channel select { case ch <- nextBlockNumber: - case <-m.ctx.Done(): + case <-ctx.Done(): return } continue } else { // wait for the next block select { - case <-nextBlock: - case <-m.ctx.Done(): + case _, ok := <-nextBlock: + if !ok { + return + } + case <-ctx.Done(): return } select { case ch <- latestBlockNum: - case <-m.ctx.Done(): + case <-ctx.Done(): return } } @@ -502,8 +593,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { return ch } -func (m *Monitor) monitor() error { - ctx := m.ctx +func (m *Monitor) monitor(ctx context.Context, wg *sync.WaitGroup) error { events := Blocks{} // minLoopInterval is time we monitor between cycles. It's a fast @@ -512,13 +602,22 @@ func (m *Monitor) monitor() error { minLoopInterval := 5 * time.Millisecond // listen for new heads either via streaming or polling - listenNewHead := m.listenNewHead() + listenNewHead := m.listenNewHead(ctx, wg) + + // fetch blocks ahead of the run loop into the cache while we trail the head + if m.prefetch != nil { + wg.Add(1) + go func() { + defer wg.Done() + m.prefetch.run(ctx) + }() + } // monitor run loop for { select { - case <-m.ctx.Done(): + case <-ctx.Done(): return nil case newHeadNum := <-listenNewHead: @@ -539,8 +638,17 @@ func (m *Monitor) monitor() error { } // fetch the next block, either via the stream or via a poll - nextBlock, nextBlockPayload, miss, err := m.fetchNextBlock(ctx) + nextBlock, nextBlockPayload, miss, err := m.fetchNextBlock(ctx, false) + + // with prefetching, a cached block may be a stale entry from an abandoned + // fork, written by a prefetch worker of this or a peer monitor. confirm a + // parent mismatch with the node before treating it as a reorg. + if err == nil && m.prefetch != nil && headBlock != nil && nextBlock.ParentHash() != headBlock.Hash() { + nextBlock, nextBlockPayload, miss, err = m.refetchNextBlock(ctx, nextBlock) + } + if err != nil { + m.hitStreak.Store(0) if errors.Is(err, context.DeadlineExceeded) { m.log.Info(fmt.Sprintf("ethmonitor: fetchNextBlock timed out: '%v', for blockNum:%v, retrying..", err, m.nextBlockNumber)) } else { @@ -556,8 +664,12 @@ func (m *Monitor) monitor() error { // we speed up the polling interval if miss { m.pollInterval.Store(int64(m.options.PollingInterval)) + m.hitStreak.Store(0) } else { m.pollInterval.Store(int64(clampDuration(minLoopInterval, time.Duration(m.pollInterval.Load())/4))) + if m.hitStreak.Load() < 1<<20 { + m.hitStreak.Add(1) + } } // build deterministic set of add/remove events which construct the canonical chain @@ -593,6 +705,11 @@ func (m *Monitor) monitor() error { // clear events sink events = Blocks{} + + // the monitor advanced, which opens up room in the prefetch window + if m.prefetch != nil { + m.prefetch.notify() + } } } } @@ -633,6 +750,11 @@ func (m *Monitor) buildCanonicalChain(ctx context.Context, nextBlock *types.Bloc } } + // blocks prefetched above the popped block may be from the abandoned fork + if m.prefetch != nil { + m.prefetch.reset(ctx, poppedBlock.NumberU64()) + } + if m.options.DebugLogging { m.log.Debug(fmt.Sprintf("ethmonitor: block reorg, reverting block #%d hash:%s prevHash:%s", poppedBlock.NumberU64(), poppedBlock.Hash().Hex(), poppedBlock.ParentHash().Hex())) } @@ -691,12 +813,7 @@ func (m *Monitor) addLogs(ctx context.Context, blocks Blocks) { blockHash := block.Hash() - topics := [][]common.Hash{} - if len(m.options.LogTopics) > 0 { - topics = append(topics, m.options.LogTopics) - } - - logs, _, err := m.filterLogs(tctx, blockHash, topics, block.Bloom()) + logs, _, err := m.filterLogs(tctx, blockHash, m.logTopics(), block.Bloom()) if err == nil { // check the logsBloom from the block to check if we should be expecting logs. logsBloom @@ -723,6 +840,16 @@ func (m *Monitor) addLogs(ctx context.Context, blocks Blocks) { } } +// logTopics returns the topic filter for block log queries, which is also +// part of the logs cache key. +func (m *Monitor) logTopics() [][]common.Hash { + topics := [][]common.Hash{} + if len(m.options.LogTopics) > 0 { + topics = append(topics, m.options.LogTopics) + } + return topics +} + func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics [][]common.Hash, blockBloom types.Bloom) ([]types.Log, []byte, error) { getter := func(ctx context.Context, _ string) ([]byte, error) { if m.options.DebugLogging { @@ -806,12 +933,20 @@ func (m *Monitor) backfillChainLogs(ctx context.Context, newBlocks Blocks) { } } -func (m *Monitor) fetchNextBlock(ctx context.Context) (*types.Block, []byte, bool, error) { +func (m *Monitor) fetchNextBlock(ctx context.Context, bypassCache bool) (*types.Block, []byte, bool, error) { miss := false + var fetchedBlock *types.Block + + var nextBlockNumber *big.Int + m.nextBlockNumberMu.Lock() + if m.nextBlockNumber != nil { + nextBlockNumber = big.NewInt(0).Set(m.nextBlockNumber) + } + m.nextBlockNumberMu.Unlock() getter := func(ctx context.Context, _ string) ([]byte, error) { if m.options.DebugLogging { - m.log.Debug(fmt.Sprintf("ethmonitor: fetchNextBlock is calling origin for number %s", m.nextBlockNumber)) + m.log.Debug(fmt.Sprintf("ethmonitor: fetchNextBlock is calling origin for number %s", nextBlockNumber)) } for { select { @@ -820,10 +955,11 @@ func (m *Monitor) fetchNextBlock(ctx context.Context) (*types.Block, []byte, boo default: } - nextBlockPayload, err := m.fetchRawBlockByNumber(ctx, m.nextBlockNumber) + nextBlockPayload, err := m.fetchRawBlockByNumber(ctx, nextBlockNumber) if err != nil { - m.log.Debug(fmt.Sprintf("ethmonitor: [retrying] failed to fetch next block # %d, due to: %v", m.nextBlockNumber, err)) + m.log.Debug(fmt.Sprintf("ethmonitor: [retrying] failed to fetch next block # %d, due to: %v", nextBlockNumber, err)) miss = true + m.hitStreak.Store(0) if m.IsStreamingMode() { // in streaming mode, we'll use a shorter time to pause before we refetch time.Sleep(200 * time.Millisecond) @@ -833,25 +969,23 @@ func (m *Monitor) fetchNextBlock(ctx context.Context) (*types.Block, []byte, boo continue } + // Validate before caching so a bad node response cannot poison this + // block number until cache expiry. Let the run loop retry decode errors. + fetchedBlock, err = m.unmarshalBlock(nextBlockPayload) + if err != nil { + return nil, err + } return nextBlockPayload, nil } } - var nextBlockNumber *big.Int - m.nextBlockNumberMu.Lock() - if m.nextBlockNumber != nil { - nextBlockNumber = big.NewInt(0).Set(m.nextBlockNumber) - } - m.nextBlockNumberMu.Unlock() - - // skip cache if isn't provided, or in case when nextBlockNumber is nil (latest) - if m.cache == nil || nextBlockNumber == nil { + // Reorg confirmation must bypass cache reads and any in-flight cache getter. + if bypassCache || m.cache == nil || nextBlockNumber == nil { resp, err := getter(ctx, "") if err != nil { return nil, resp, miss, err } - block, err := m.unmarshalBlock(resp) - return block, resp, miss, err + return fetchedBlock, resp, miss, nil } // fetch with distributed mutex @@ -860,10 +994,32 @@ func (m *Monitor) fetchNextBlock(ctx context.Context) (*types.Block, []byte, boo if err != nil { return nil, resp, miss, err } + if fetchedBlock != nil { + return fetchedBlock, resp, miss, nil + } block, err := m.unmarshalBlock(resp) + if err != nil { + // A peer or an older monitor may have cached an invalid payload. + if deleteErr := m.cache.Delete(ctx, key); deleteErr != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting invalid block cache for block num %s due to: '%v'", nextBlockNumber, deleteErr)) + } + } return block, resp, miss, err } +// refetchNextBlock drops the cached by-number entry for block and confirms the +// next block directly with the node, even if a peer repopulates the cache. +func (m *Monitor) refetchNextBlock(ctx context.Context, block *types.Block) (*types.Block, []byte, bool, error) { + key := CacheKeyBlockByNumber(m.chainID, block.Number()) + if err := m.cache.Delete(ctx, key); err != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting block cache for block num %d due to: '%v'", block.NumberU64(), err)) + } + if m.options.DebugLogging { + m.log.Debug(fmt.Sprintf("ethmonitor: block #%d hash:%s does not extend head, refetching from origin", block.NumberU64(), block.Hash().Hex())) + } + return m.fetchNextBlock(ctx, true) +} + func CacheKeyBlockByNumber(chainID *big.Int, num *big.Int) string { return fmt.Sprintf("ethmonitor:%s:BlockNum:%s", chainID.String(), num.String()) } @@ -913,6 +1069,8 @@ func (m *Monitor) fetchRawBlockByNumber(ctx context.Context, num *big.Int) ([]by cancel() if err != nil { + // Clear the catch-up signal before retries, which may wait at the head. + m.hitStreak.Store(0) if errors.Is(err, ethereum.NotFound) { return nil, ethereum.NotFound } else { @@ -1025,7 +1183,7 @@ func (m *Monitor) publish(ctx context.Context, events Blocks) error { if ok { select { case m.publishCh <- pubEvents: - case <-m.ctx.Done(): + case <-ctx.Done(): } } diff --git a/ethmonitor/prefetch.go b/ethmonitor/prefetch.go new file mode 100644 index 00000000..9b6c0565 --- /dev/null +++ b/ethmonitor/prefetch.go @@ -0,0 +1,286 @@ +package ethmonitor + +import ( + "context" + "fmt" + "math/big" + "runtime/debug" + "sync" + "sync/atomic" + "time" + + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +// prefetchTick is how often the prefetcher reconsiders its window when +// nothing else wakes it up. +const prefetchTick = 25 * time.Millisecond + +// prefetcher fetches blocks, and their logs when the monitor runs WithLogs, +// ahead of the monitor's run loop while the monitor trails the chain head. +// +// It only fills the cache, using the same keys the run loop reads, so the run +// loop stays the single place the canonical chain is built and validated; +// prefetching only changes where a payload comes from. When the monitor is at +// the head there is nothing past its next block to fetch, and the prefetcher +// is idle. +type prefetcher struct { + m *Monitor + concurrency int + window uint64 + + jobs chan prefetchJob + wake chan struct{} + + // gen is bumped on every reset, so a job scheduled before a reorg can + // tell its payload may belong to the abandoned fork. + gen atomic.Uint64 + + mu sync.Mutex + cursor uint64 // next block number to schedule + highWater uint64 // highest block number scheduled since the last reset +} + +type prefetchJob struct { + num uint64 + gen uint64 +} + +func newPrefetcher(m *Monitor, concurrency, window int) *prefetcher { + return &prefetcher{ + m: m, + concurrency: concurrency, + window: uint64(window), + jobs: make(chan prefetchJob, concurrency), + wake: make(chan struct{}, 1), + } +} + +// notify asks the prefetcher to reconsider its window, without blocking. +func (p *prefetcher) notify() { + select { + case p.wake <- struct{}{}: + default: + } +} + +// run schedules prefetch jobs until ctx is done, and returns once all of its +// workers have exited. +func (p *prefetcher) run(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + p.m.log.Error(fmt.Sprintf("ethmonitor: panic in prefetch loop: %v - stack: %s", r, string(debug.Stack()))) + p.m.alert.Alert(context.Background(), "ethmonitor: panic in prefetch loop: %v", r) + } + }() + + // start from a clean slate, as Run may be called again after a failure + p.mu.Lock() + p.gen.Add(1) + p.cursor, p.highWater = 0, 0 + p.mu.Unlock() + + ctx, cancel := context.WithCancel(ctx) + var wg sync.WaitGroup + defer func() { + cancel() + wg.Wait() + }() + for i := 0; i < p.concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + p.worker(ctx) + }() + } + + ticker := time.NewTicker(prefetchTick) + defer ticker.Stop() + + var lastHeadPoll time.Time + + for { + select { + case <-ctx.Done(): + return + case <-p.wake: + case <-ticker.C: + } + + // in streaming mode the newHeads stream keeps latestHead current, + // in polling mode nothing does, so we ask the node for it. but only + // while the monitor is catching up: at the head there is nothing to + // prefetch, and on a slow chain we'd just double the polling load. + if !p.m.IsStreamingMode() && p.m.isCatchingUp() && time.Since(lastHeadPoll) >= p.m.options.PollingInterval { + lastHeadPoll = time.Now() + p.pollHead(ctx) + } + + p.schedule() + } +} + +func (p *prefetcher) pollHead(ctx context.Context) { + tctx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + + head, err := p.m.provider.BlockNumber(tctx) + if err != nil { + p.m.log.Debug(fmt.Sprintf("ethmonitor: prefetch failed to poll head block number: %v", err)) + return + } + p.m.latestHead.Store(head) +} + +// schedule hands the workers the block numbers in the window past the +// monitor's next block, as far as they have room for. +func (p *prefetcher) schedule() { + next, ok := p.m.nextBlockNum() + if !ok { + return + } + + // The run loop fetches `next` itself, so we only look past it. We also + // leave the newest head block to the run loop: it is the one most likely + // not yet served by the node, and the run loop already retries it. + head := p.m.latestHead.Load() + if head < next+2 { + return + } + hi := min(head-1, next+p.window) + + p.mu.Lock() + defer p.mu.Unlock() + + gen := p.gen.Load() + for n := max(p.cursor, next+1); n <= hi; n++ { + select { + case p.jobs <- prefetchJob{num: n, gen: gen}: + p.cursor = n + 1 + p.highWater = max(p.highWater, n) + default: + // workers are busy, carry on from the cursor next time + return + } + } +} + +// reset is called when the run loop pops the block fromNum off the canonical +// chain during a reorg. Any block prefetched above it may belong to the +// abandoned fork, so their by-number cache entries are dropped and scheduling +// resumes from fromNum. +// +// Entries a worker writes after the reset are dropped by the worker itself, +// and any stale entry that slips through is caught by the run loop, which +// confirms a block that does not extend its head with the node before +// treating it as a reorg. +func (p *prefetcher) reset(ctx context.Context, fromNum uint64) { + p.mu.Lock() + p.gen.Add(1) + hi := p.highWater + p.cursor = fromNum + p.highWater = fromNum + p.mu.Unlock() + + for n := fromNum + 1; n <= hi; n++ { + key := CacheKeyBlockByNumber(p.m.chainID, new(big.Int).SetUint64(n)) + if err := p.m.cache.Delete(ctx, key); err != nil { + p.m.log.Warn(fmt.Sprintf("ethmonitor: error deleting prefetched block cache for block num %d due to: '%v'", n, err)) + } + } +} + +func (p *prefetcher) worker(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case job := <-p.jobs: + p.fetch(ctx, job) + p.notify() + } + } +} + +// fetch loads one block, and its logs, into the cache. It makes a single +// attempt: on any failure the run loop fetches the block itself when it gets +// there. +func (p *prefetcher) fetch(ctx context.Context, job prefetchJob) { + defer func() { + if r := recover(); r != nil { + p.m.log.Error(fmt.Sprintf("ethmonitor: panic in prefetch worker: %v - stack: %s", r, string(debug.Stack()))) + p.m.alert.Alert(context.Background(), "ethmonitor: panic in prefetch worker: %v", r) + } + }() + + m := p.m + if job.gen != p.gen.Load() { + return + } + + num := new(big.Int).SetUint64(job.num) + key := CacheKeyBlockByNumber(m.chainID, num) + var block *types.Block + + // NOTE: the getter must not retry, as it runs while holding the lock for the key + getter := func(ctx context.Context, _ string) ([]byte, error) { + if m.options.DebugLogging { + m.log.Debug(fmt.Sprintf("ethmonitor: prefetch is calling origin for number %d", job.num)) + } + tctx, cancel := context.WithTimeout(ctx, m.options.Timeout) + defer cancel() + + payload, err := m.provider.RawBlockByNumber(tctx, num) + if err != nil { + return nil, err + } + if len(payload) == 0 || string(payload) == "null" { + return nil, ethereum.NotFound + } + // Validate even without logs: the serial loop must never inherit an + // undecodable response from a successful cache write. + block, err = m.unmarshalBlock(payload) + if err != nil { + return nil, err + } + return payload, nil + } + + payload, err := m.cache.GetOrSetWithLockEx(ctx, key, getter, m.options.CacheExpiry) + if err != nil { + m.log.Debug(fmt.Sprintf("ethmonitor: prefetch of block %d failed: %v", job.num, err)) + return + } + + if job.gen != p.gen.Load() { + // a reorg reset ran while we were fetching, so this payload may be + // from the abandoned fork + if err := m.cache.Delete(ctx, key); err != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting prefetched block cache for block num %d due to: '%v'", job.num, err)) + } + return + } + + if block == nil { + block, err = m.unmarshalBlock(payload) + if err != nil { + m.log.Debug(fmt.Sprintf("ethmonitor: prefetch failed to decode block %d: %v", job.num, err)) + if deleteErr := m.cache.Delete(ctx, key); deleteErr != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting invalid block cache for block num %d due to: '%v'", job.num, deleteErr)) + } + return + } + } + + if !m.options.WithLogs { + return + } + + // logs are keyed by block hash, so they stay valid across reorgs + tctx, cancel := context.WithTimeout(ctx, m.options.Timeout) + defer cancel() + if _, _, err := m.filterLogs(tctx, block.Hash(), m.logTopics(), block.Bloom()); err != nil { + m.log.Debug(fmt.Sprintf("ethmonitor: prefetch of logs for block %d failed: %v", job.num, err)) + } +} diff --git a/ethmonitor/prefetch_internal_test.go b/ethmonitor/prefetch_internal_test.go new file mode 100644 index 00000000..259ae05e --- /dev/null +++ b/ethmonitor/prefetch_internal_test.go @@ -0,0 +1,61 @@ +package ethmonitor + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestPrefetchRejectsInvalidBlockBeforeCaching(t *testing.T) { + for _, withLogs := range []bool{false, true} { + t.Run(fmt.Sprintf("withLogs=%v", withLogs), func(t *testing.T) { + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + provider.EXPECT().RawBlockByNumber(gomock.Any(), big.NewInt(1000)). + Return(json.RawMessage(`{"number":"0x3e8"}`), nil) + opts := DefaultOptions + opts.PrefetchConcurrency = 1 + opts.WithLogs = withLogs + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID = big.NewInt(1) + + monitor.prefetch.fetch(context.Background(), prefetchJob{num: 1000}) + key := CacheKeyBlockByNumber(monitor.chainID, big.NewInt(1000)) + _, found, err := monitor.cache.Get(context.Background(), key) + require.NoError(t, err) + require.False(t, found, "worker cached an undecodable block") + }) + } +} + +func TestPrefetchPanicStopsWorkers(t *testing.T) { + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + provider.EXPECT().BlockNumber(gomock.Any()).DoAndReturn(func(context.Context) (uint64, error) { + panic("simulated head poll panic") + }) + opts := DefaultOptions + opts.PrefetchConcurrency = 4 + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.hitStreak.Store(2) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + monitor.prefetch.run(ctx) + }() + defer func() { cancel(); <-done }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("prefetch panic did not cancel and join its workers") + } +} diff --git a/ethmonitor/prefetch_test.go b/ethmonitor/prefetch_test.go new file mode 100644 index 00000000..51d4216d --- /dev/null +++ b/ethmonitor/prefetch_test.go @@ -0,0 +1,812 @@ +package ethmonitor_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/big" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" + "github.com/0xsequence/ethkit/ethrpc" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/0xsequence/ethkit/go-ethereum/crypto" + memcache "github.com/goware/cachestore-mem" + cachestore "github.com/goware/cachestore2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// TestMonitorPrefetchKeepsUp reproduces a chain producing blocks faster than +// the monitor can fetch them one at a time, and checks prefetching keeps the +// monitor at the head, in both polling and streaming mode. +func TestMonitorPrefetchKeepsUp(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + // 100 blocks/s, with 15ms per node call: a serial fetch of a block and + // its logs manages ~30 blocks/s at best. + const blockInterval = 10 * time.Millisecond + const latency = 15 * time.Millisecond + const runFor = 2 * time.Second + + for _, streaming := range []bool{false, true} { + for _, concurrency := range []int{0, 4, 8} { + t.Run(fmt.Sprintf("streaming=%v/prefetch=%d", streaming, concurrency), func(t *testing.T) { + chain := newFakeChain(1000, 1, latency) + monitor := newTestMonitor(t, chain, streaming, concurrency, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go chain.produce(ctx, blockInterval) + go monitor.Run(ctx) + defer monitor.Stop() + + time.Sleep(runFor) + lag := chain.head() - monitor.LatestBlockNum().Uint64() + t.Logf("head:%d monitor:%d lag:%d blocks", chain.head(), monitor.LatestBlockNum().Uint64(), lag) + + if concurrency == 0 { + // the bug: without prefetching, the monitor falls behind + assert.Greater(t, lag, uint64(60)) + } else { + assert.Less(t, lag, uint64(15)) + } + }) + } + } +} + +// TestMonitorPrefetchReorg reorgs the chain while the prefetcher is ahead of +// the monitor, and checks the monitor ends on the new fork with a consistent +// stream of events. +func TestMonitorPrefetchReorg(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + // prefetched: the reorg starts at a block the prefetcher already cached + // from the abandoned fork, ahead of the monitor head. otherwise it starts + // below the monitor head, which has already published blocks from it. + // + // the window is kept small, as the monitor pauses 2s for every block it + // reverts, and stale prefetched blocks can deepen a reorg up to the window. + for _, concurrency := range []int{2, 4} { + for _, prefetched := range []bool{true, false} { + t.Run(fmt.Sprintf("prefetch=%d/prefetched=%v", concurrency, prefetched), func(t *testing.T) { + testMonitorPrefetchReorg(t, concurrency, prefetched) + }) + } + } +} + +func testMonitorPrefetchReorg(t *testing.T, concurrency int, prefetched bool) { + chain := newFakeChain(1000, 200, 15*time.Millisecond) + backend, err := memcache.NewBackend(1024) + require.NoError(t, err) + cache := cachestore.OpenStore[[]byte](backend) + monitor := newTestMonitor(t, chain, false, concurrency, 4, backend) + + sub := monitor.Subscribe("TestMonitorPrefetchReorg") + defer sub.Unsubscribe() + events := newEventLog() + go func() { + for blocks := range sub.Blocks() { + events.apply(t, blocks) + } + }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go monitor.Run(ctx) + defer monitor.Stop() + + require.Eventually(t, func() bool { + return monitor.LatestBlockNum().Uint64() >= 1050 + }, 10*time.Second, time.Millisecond) + + var reorgAt uint64 + if prefetched { + // the monitor takes ~15ms a block here, so a block cached two or + // more past its head is still ahead of it when we reorg + require.Eventually(t, func() bool { + head := monitor.LatestBlockNum().Uint64() + for n := head + 4; n >= head+2; n-- { + key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), new(big.Int).SetUint64(n)) + if _, ok, _ := cache.Get(context.Background(), key); ok { + reorgAt = n + return true + } + } + return false + }, 5*time.Second, time.Millisecond) + } else { + reorgAt = monitor.LatestBlockNum().Uint64() - 2 + } + chain.reorgFrom(reorgAt) + t.Logf("reorged chain from block %d, monitor head %d", reorgAt, monitor.LatestBlockNum().Uint64()) + + require.Eventually(t, func() bool { + head := monitor.LatestReadyBlock() + return head != nil && head.Hash() == chain.hashAt(chain.head()) + }, 60*time.Second, 10*time.Millisecond) + + // the retained chain must be the new canonical chain + for _, b := range monitor.Chain().Blocks() { + assert.Equal(t, chain.hashAt(b.NumberU64()), b.Hash(), "block %d", b.NumberU64()) + } + + // and so must the chain subscribers built from the events + require.Eventually(t, func() bool { + return events.head() == chain.hashAt(chain.head()) + }, 5*time.Second, 10*time.Millisecond) + for num, hash := range events.blocks() { + assert.Equal(t, chain.hashAt(num), hash, "subscriber block %d", num) + } +} + +// TestMonitorPrefetchHeadAhead has the node announce heads it cannot serve +// yet, as when a websocket is ahead of the http node. The prefetcher's misses +// must not wedge the monitor. +func TestMonitorPrefetchHeadAhead(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + chain := newFakeChain(1000, 1, 5*time.Millisecond) + chain.announceAhead = 5 + monitor := newTestMonitor(t, chain, true, 4, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go chain.produce(ctx, 10*time.Millisecond) + go monitor.Run(ctx) + defer monitor.Stop() + + time.Sleep(1500 * time.Millisecond) + assert.Greater(t, monitor.LatestBlockNum().Uint64(), uint64(1050)) +} + +// TestMonitorPrefetchSlowChainNoExtraCalls checks the prefetcher stays out +// of the way of a monitor at the head of a slow chain: workers never ask for +// blocks the chain hasn't made, and in polling mode the head isn't polled. +func TestMonitorPrefetchSlowChainNoExtraCalls(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + type calls struct{ blockNumber, blockByNumber, filterLogs int64 } + + run := func(streaming bool, concurrency int) calls { + chain := newFakeChain(1000, 1, 5*time.Millisecond) + monitor := newTestMonitor(t, chain, streaming, concurrency, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go chain.produce(ctx, 200*time.Millisecond) + go monitor.Run(ctx) + defer monitor.Stop() + + time.Sleep(2 * time.Second) + return calls{ + blockNumber: chain.blockNumberCalls.Load(), + blockByNumber: chain.blockByNumberCalls.Load(), + filterLogs: chain.filterLogsCalls.Load(), + } + } + + for _, streaming := range []bool{false, true} { + t.Run(fmt.Sprintf("streaming=%v", streaming), func(t *testing.T) { + off := run(streaming, 0) + on := run(streaming, 4) + t.Logf("prefetch=0: %+v", off) + t.Logf("prefetch=4: %+v", on) + + assert.Zero(t, off.blockNumber) + if streaming { + // the stream supplies the head + assert.Zero(t, on.blockNumber) + } else { + // at most the one poll at startup, before the monitor has found + // out it's at the head + assert.LessOrEqual(t, on.blockNumber, int64(1)) + } + + // ~10 blocks made in the run: allow a couple of calls of jitter + // between the two runs, well short of a block per worker + assert.InDelta(t, off.blockByNumber, on.blockByNumber, 3) + assert.InDelta(t, off.filterLogs, on.filterLogs, 3) + }) + } +} + +func TestMonitorPrefetchShutdownNoGoroutineLeak(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + chain := newFakeChain(1000, 100, 5*time.Millisecond) + baseline := runtime.NumGoroutine() + + monitor := newTestMonitor(t, chain, false, 4, 0) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- monitor.Run(ctx) + }() + + time.Sleep(300 * time.Millisecond) + cancel() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Monitor.Run() didn't return within timeout") + } + + // Run joins its goroutines; allow runtime cleanup before comparing counts. + // NOTE: poll here rather than with assert.Eventually, whose condition runs + // in a goroutine of its own and so always counts one extra. + deadline := time.Now().Add(2 * time.Second) + for runtime.NumGoroutine() > baseline && time.Now().Before(deadline) { + runtime.GC() + time.Sleep(50 * time.Millisecond) + } + if n := runtime.NumGoroutine(); n > baseline { + buf := make([]byte, 1<<20) + t.Fatalf("%d goroutine(s) leaked:\n%s", n-baseline, buf[:runtime.Stack(buf, true)]) + } +} + +// A peer can complete an old-fork fetch between DEL and the confirmation read, +// and a cache deletion can fail. Neither may cause a false canonical removal. +func TestMonitorPrefetchRefetchBypassesCache(t *testing.T) { + for _, deleteFails := range []bool{false, true} { + t.Run(fmt.Sprintf("deleteFails=%v", deleteFails), func(t *testing.T) { + chain := newFakeChain(1000, 20, time.Millisecond) + oldNext, ok := chain.byNumber(big.NewInt(1001)) + require.True(t, ok) + chain.reorgFrom(1000) + + backend, err := memcache.NewBackend(512) + require.NoError(t, err) + key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(1001)) + racingBackend := &repopulatingBackend{ + Backend: backend, key: key, payload: oldNext.payload(), deleteFails: deleteFails, + } + require.NoError(t, racingBackend.SetEx(context.Background(), key, []byte(oldNext.payload()), time.Minute)) + monitor := newTestMonitor(t, chain, false, 1, 1, racingBackend) + sub := monitor.Subscribe("TestMonitorPrefetchRefetchBypassesCache") + defer sub.Unsubscribe() + runMonitorForTest(t, monitor) + + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + for { + select { + case blocks := <-sub.Blocks(): + for _, b := range blocks { + require.Equal(t, ethmonitor.Added, b.Event, "removed canonical block %d", b.NumberU64()) + require.Equal(t, chain.hashAt(b.NumberU64()), b.Hash()) + if b.NumberU64() >= 1001 { + require.Positive(t, racingBackend.deletes.Load(), "did not exercise the mismatch confirmation") + return + } + } + case <-timer.C: + t.Fatal("monitor did not advance through the stale cache entry") + } + } + }) + } +} + +func TestMonitorPrefetchInvalidBlockRecovery(t *testing.T) { + for _, withLogs := range []bool{false, true} { + for _, cached := range []bool{false, true} { + t.Run(fmt.Sprintf("withLogs=%v/cached=%v", withLogs, cached), func(t *testing.T) { + chain := newFakeChain(1000, 30, time.Millisecond) + initial := newTestMonitor(t, chain, false, 4, 0) + opts := initial.Options() + opts.WithLogs = withLogs + provider := &invalidBlockProvider{fakeProvider: initial.Provider().(*fakeProvider), target: 1000} + if cached { + key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(1000)) + require.NoError(t, opts.CacheBackend.SetEx(context.Background(), key, []byte(`{"number":"0x3e8"}`), time.Minute)) + // The origin is healthy; only the existing cache entry is invalid. + provider.injected.Store(true) + } else { + // Exercise the automatic memory cache with one invalid origin response. + opts.CacheBackend = nil + } + monitor, err := ethmonitor.NewMonitor(provider, opts) + require.NoError(t, err) + runMonitorForTest(t, monitor) + require.Eventually(t, func() bool { + return monitor.LatestReadyBlock() != nil && monitor.LatestBlockNum().Uint64() == chain.head() + }, 2*time.Second, time.Millisecond) + require.True(t, provider.injected.Load()) + for _, block := range monitor.Chain().Blocks() { + require.Equal(t, chain.hashAt(block.NumberU64()), block.Hash()) + } + }) + } + } +} + +func TestMonitorPrefetchIdleAfterBacklog(t *testing.T) { + chain := newFakeChain(1000, 12, time.Millisecond) + monitor := newTestMonitor(t, chain, false, 4, 0) + provider := &idleHeadProvider{fakeProvider: monitor.Provider().(*fakeProvider), missed: make(chan struct{})} + monitor, err := ethmonitor.NewMonitor(provider, monitor.Options()) + require.NoError(t, err) + runMonitorForTest(t, monitor) + + // Wait for the serial loop's first miss after draining the backlog, rather + // than sampling during the last block's processing. + select { + case <-provider.missed: + case <-time.After(2 * time.Second): + t.Fatal("monitor did not drain the backlog") + } + before := chain.blockNumberCalls.Load() + time.Sleep(150 * time.Millisecond) + // Permit a head poll that was already selected when the miss occurred. + require.LessOrEqual(t, chain.blockNumberCalls.Load()-before, int64(1)) +} + +func TestMonitorPrefetchFatalExitStopsWorkers(t *testing.T) { + chain := newFakeChain(1000, 20, time.Millisecond) + backend, err := memcache.NewBackend(512) + require.NoError(t, err) + initial := newTestMonitor(t, chain, false, 4, 0, &invalidLogsBackend{backend}) + opts := initial.Options() + opts.BlockRetentionLimit = 2 // Four queued events; failed logs prevent dequeue. + monitor, err := ethmonitor.NewMonitor(initial.Provider().(ethrpc.RawInterface), opts) + require.NoError(t, err) + sub := monitor.Subscribe("TestMonitorPrefetchFatalExitStopsWorkers") + defer sub.Unsubscribe() + + for i := 0; i < 2; i++ { + done := runMonitorForTest(t, monitor) + select { + case err := <-done: + require.ErrorIs(t, err, ethmonitor.ErrFatal) + case <-time.After(2 * time.Second): + t.Fatal("expected a fatal publish error") + } + require.False(t, monitor.IsRunning()) + before := chain.blockNumberCalls.Load() + time.Sleep(150 * time.Millisecond) + require.Equal(t, before, chain.blockNumberCalls.Load(), "prefetch continued after Run returned") + } +} + +func TestMonitorPrefetchRestart(t *testing.T) { + for _, streaming := range []bool{false, true} { + t.Run(fmt.Sprintf("streaming=%v", streaming), func(t *testing.T) { + chain := newFakeChain(1000, 10, time.Millisecond) + monitor := newTestMonitor(t, chain, streaming, 4, 0) + ctx, cancel := context.WithCancel(context.Background()) + produced := make(chan struct{}) + go func() { + defer close(produced) + chain.produce(ctx, 20*time.Millisecond) + }() + defer func() { cancel(); <-produced }() + + for i := 0; i < 2; i++ { + target := chain.head() + done := runMonitorForTest(t, monitor) + require.Eventually(t, func() bool { + return monitor.LatestBlockNum().Uint64() >= target + }, 2*time.Second, time.Millisecond) + monitor.Stop() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("monitor did not stop") + } + // Make a gap that the next run must ingest. + require.Eventually(t, func() bool { + return chain.head() >= monitor.LatestBlockNum().Uint64()+3 + }, time.Second, time.Millisecond) + } + }) + } +} + +func runMonitorForTest(t *testing.T, monitor *ethmonitor.Monitor) <-chan error { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- monitor.Run(ctx) + close(done) + }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("monitor did not shut down") + } + }) + return done +} + +type repopulatingBackend struct { + cachestore.Backend + key string + payload []byte + deleteFails bool + once sync.Once + deletes atomic.Int64 +} + +func (b *repopulatingBackend) Delete(ctx context.Context, key string) error { + if key != b.key { + return b.Backend.Delete(ctx, key) + } + b.deletes.Add(1) + if b.deleteFails { + return errors.New("simulated cache deletion failure") + } + if err := b.Backend.Delete(ctx, key); err != nil { + return err + } + b.once.Do(func() { _ = b.Backend.SetEx(ctx, key, b.payload, time.Minute) }) + return nil +} + +type invalidBlockProvider struct { + *fakeProvider + target uint64 + injected atomic.Bool +} + +func (p *invalidBlockProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json.RawMessage, error) { + payload, err := p.fakeProvider.RawBlockByNumber(ctx, num) + if err == nil && num != nil && num.Uint64() == p.target && p.injected.CompareAndSwap(false, true) { + return json.RawMessage(fmt.Sprintf(`{"number":"0x%x"}`, p.target)), nil + } + return payload, err +} + +type idleHeadProvider struct { + *fakeProvider + missed chan struct{} + once sync.Once +} + +func (p *idleHeadProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json.RawMessage, error) { + payload, err := p.fakeProvider.RawBlockByNumber(ctx, num) + if errors.Is(err, ethereum.NotFound) { + p.once.Do(func() { close(p.missed) }) + } + return payload, err +} + +type invalidLogsBackend struct{ cachestore.Backend } + +func (b *invalidLogsBackend) GetOrSetWithLockEx(ctx context.Context, key string, getter func(context.Context, string) (any, error), ttl time.Duration) (any, error) { + if strings.Contains(key, ":Logs:") { + return []byte("invalid logs"), nil + } + return b.Backend.GetOrSetWithLockEx(ctx, key, getter, ttl) +} + +func newTestMonitor(t *testing.T, chain *fakeChain, streaming bool, prefetchConcurrency, prefetchWindow int, cacheBackend ...cachestore.Backend) *ethmonitor.Monitor { + t.Helper() + + provider := &fakeProvider{ + MockRawInterface: mocks.NewMockRawInterface(gomock.NewController(t)), + chain: chain, + streaming: streaming, + } + + opts := ethmonitor.DefaultOptions + opts.PollingInterval = 20 * time.Millisecond + opts.Timeout = 2 * time.Second + opts.WithLogs = true + opts.StartBlockNumber = new(big.Int).SetUint64(chain.base) + opts.PrefetchConcurrency = prefetchConcurrency + opts.PrefetchWindow = prefetchWindow + if len(cacheBackend) > 0 { + opts.CacheBackend = cacheBackend[0] + } + + monitor, err := ethmonitor.NewMonitor(provider, opts) + require.NoError(t, err) + return monitor +} + +// fakeChain is an in-memory chain served with a fixed latency per call. +type fakeChain struct { + base uint64 + latency time.Duration + + // announceAhead makes the chain announce heads this many blocks past the + // blocks it serves. + announceAhead uint64 + + // node calls served, by method + blockNumberCalls atomic.Int64 + blockByNumberCalls atomic.Int64 + filterLogsCalls atomic.Int64 + + mu sync.Mutex + canonical []common.Hash // canonical[i] is block base+i + blocks map[common.Hash]fakeBlock + fork int + heads []*fakeSubscription +} + +type fakeBlock struct { + num uint64 + hash common.Hash + parent common.Hash +} + +func newFakeChain(base uint64, n int, latency time.Duration) *fakeChain { + c := &fakeChain{base: base, latency: latency, blocks: map[common.Hash]fakeBlock{}} + for i := 0; i < n; i++ { + c.appendLocked() + } + return c +} + +func (c *fakeChain) appendLocked() fakeBlock { + num := c.base + uint64(len(c.canonical)) + b := fakeBlock{ + num: num, + hash: crypto.Keccak256Hash([]byte(fmt.Sprintf("fork:%d/block:%d", c.fork, num))), + } + if len(c.canonical) > 0 { + b.parent = c.canonical[len(c.canonical)-1] + } + c.canonical = append(c.canonical, b.hash) + c.blocks[b.hash] = b + return b +} + +func (c *fakeChain) produce(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.mu.Lock() + b := c.appendLocked() + heads := append([]*fakeSubscription{}, c.heads...) + c.mu.Unlock() + + header := &types.Header{Number: new(big.Int).SetUint64(b.num + c.announceAhead)} + for _, sub := range heads { + sub.send(header) + } + } + } +} + +// reorgFrom replaces the canonical chain from block num onwards with a new +// fork of the same length. +func (c *fakeChain) reorgFrom(num uint64) { + c.mu.Lock() + defer c.mu.Unlock() + + n := len(c.canonical) + c.canonical = c.canonical[:num-c.base] + c.fork++ + for len(c.canonical) < n { + c.appendLocked() + } +} + +func (c *fakeChain) head() uint64 { + c.mu.Lock() + defer c.mu.Unlock() + return c.base + uint64(len(c.canonical)) - 1 +} + +func (c *fakeChain) hashAt(num uint64) common.Hash { + c.mu.Lock() + defer c.mu.Unlock() + return c.canonical[num-c.base] +} + +func (c *fakeChain) byNumber(num *big.Int) (fakeBlock, bool) { + c.mu.Lock() + defer c.mu.Unlock() + i := uint64(len(c.canonical) - 1) + if num != nil { + if num.Uint64() < c.base || num.Uint64()-c.base > i { + return fakeBlock{}, false + } + i = num.Uint64() - c.base + } + return c.blocks[c.canonical[i]], true +} + +func (c *fakeChain) byHash(hash common.Hash) (fakeBlock, bool) { + c.mu.Lock() + defer c.mu.Unlock() + b, ok := c.blocks[hash] + return b, ok +} + +func (c *fakeChain) wait(ctx context.Context) error { + select { + case <-time.After(c.latency): + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (b fakeBlock) payload() json.RawMessage { + zero := common.Hash{}.Hex() + return json.RawMessage(fmt.Sprintf(`{`+ + `"number":"0x%x","hash":"%s","parentHash":"%s",`+ + `"sha3Uncles":"%s","miner":"0x0000000000000000000000000000000000000000",`+ + `"stateRoot":"%s","transactionsRoot":"%s","receiptsRoot":"%s",`+ + `"logsBloom":"0x%s","difficulty":"0x0","gasLimit":"0x1c9c380","gasUsed":"0x0",`+ + `"timestamp":"0x%x","extraData":"0x","transactions":[],"uncles":[]}`, + b.num, b.hash.Hex(), b.parent.Hex(), + zero, zero, zero, zero, + strings.Repeat("0", 512), + b.num, + )) +} + +// fakeProvider serves a fakeChain. Methods the monitor is not expected to +// call fall through to the gomock mock, which fails the test. +type fakeProvider struct { + *mocks.MockRawInterface + chain *fakeChain + streaming bool +} + +var _ ethrpc.RawInterface = &fakeProvider{} + +func (p *fakeProvider) ChainID(ctx context.Context) (*big.Int, error) { + return big.NewInt(1), nil +} + +func (p *fakeProvider) IsStreamingEnabled() bool { + return p.streaming +} + +func (p *fakeProvider) BlockNumber(ctx context.Context) (uint64, error) { + p.chain.blockNumberCalls.Add(1) + if err := p.chain.wait(ctx); err != nil { + return 0, err + } + return p.chain.head() + p.chain.announceAhead, nil +} + +func (p *fakeProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json.RawMessage, error) { + p.chain.blockByNumberCalls.Add(1) + if err := p.chain.wait(ctx); err != nil { + return nil, err + } + b, ok := p.chain.byNumber(num) + if !ok { + return nil, ethereum.NotFound + } + return b.payload(), nil +} + +func (p *fakeProvider) RawBlockByHash(ctx context.Context, hash common.Hash) (json.RawMessage, error) { + if err := p.chain.wait(ctx); err != nil { + return nil, err + } + b, ok := p.chain.byHash(hash) + if !ok { + return nil, ethereum.NotFound + } + return b.payload(), nil +} + +func (p *fakeProvider) RawFilterLogs(ctx context.Context, q ethereum.FilterQuery) (json.RawMessage, error) { + p.chain.filterLogsCalls.Add(1) + if err := p.chain.wait(ctx); err != nil { + return nil, err + } + return json.RawMessage(`[]`), nil +} + +func (p *fakeProvider) SubscribeNewHeads(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) { + sub := &fakeSubscription{ch: ch, err: make(chan error), done: make(chan struct{})} + p.chain.mu.Lock() + p.chain.heads = append(p.chain.heads, sub) + p.chain.mu.Unlock() + return sub, nil +} + +type fakeSubscription struct { + ch chan<- *types.Header + err chan error + done chan struct{} + once sync.Once +} + +func (s *fakeSubscription) send(header *types.Header) { + select { + case s.ch <- header: + case <-s.done: + } +} + +func (s *fakeSubscription) Unsubscribe() { + s.once.Do(func() { close(s.done) }) +} + +func (s *fakeSubscription) Err() <-chan error { + return s.err +} + +// eventLog rebuilds the chain a subscriber sees from the monitor's events, +// checking each event is consistent with what came before. +type eventLog struct { + mu sync.Mutex + chain []fakeBlock +} + +func newEventLog() *eventLog { + return &eventLog{} +} + +func (l *eventLog) apply(t *testing.T, blocks ethmonitor.Blocks) { + l.mu.Lock() + defer l.mu.Unlock() + + for _, b := range blocks { + switch b.Event { + case ethmonitor.Added: + if n := len(l.chain); n > 0 { + assert.Equal(t, l.chain[n-1].hash, b.ParentHash(), "added block %d does not extend the chain", b.NumberU64()) + } + l.chain = append(l.chain, fakeBlock{num: b.NumberU64(), hash: b.Hash(), parent: b.ParentHash()}) + case ethmonitor.Removed: + n := len(l.chain) + if assert.Greater(t, n, 0, "removed block %d from an empty chain", b.NumberU64()) { + assert.Equal(t, l.chain[n-1].hash, b.Hash(), "removed block %d is not the head", b.NumberU64()) + l.chain = l.chain[:n-1] + } + } + } +} + +func (l *eventLog) head() common.Hash { + l.mu.Lock() + defer l.mu.Unlock() + if len(l.chain) == 0 { + return common.Hash{} + } + return l.chain[len(l.chain)-1].hash +} + +func (l *eventLog) blocks() map[uint64]common.Hash { + l.mu.Lock() + defer l.mu.Unlock() + out := make(map[uint64]common.Hash, len(l.chain)) + for _, b := range l.chain { + out[b.num] = b.hash + } + return out +} From 1c355320ee6f02939bbf513bdbd465659d345b57 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Wed, 30 Sep 2026 14:08:14 -0400 Subject: [PATCH 02/11] test(ethmonitor): avoid timing-dependent polling counts --- ethmonitor/prefetch_test.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/ethmonitor/prefetch_test.go b/ethmonitor/prefetch_test.go index 51d4216d..1c84edc0 100644 --- a/ethmonitor/prefetch_test.go +++ b/ethmonitor/prefetch_test.go @@ -186,7 +186,7 @@ func TestMonitorPrefetchSlowChainNoExtraCalls(t *testing.T) { t.Skip("Skipping in short mode") } - type calls struct{ blockNumber, blockByNumber, filterLogs int64 } + type calls struct{ blockNumber, foundBlocks, filterLogs, farAheadBlocks int64 } run := func(streaming bool, concurrency int) calls { chain := newFakeChain(1000, 1, 5*time.Millisecond) @@ -200,9 +200,10 @@ func TestMonitorPrefetchSlowChainNoExtraCalls(t *testing.T) { time.Sleep(2 * time.Second) return calls{ - blockNumber: chain.blockNumberCalls.Load(), - blockByNumber: chain.blockByNumberCalls.Load(), - filterLogs: chain.filterLogsCalls.Load(), + blockNumber: chain.blockNumberCalls.Load(), + foundBlocks: chain.foundBlockByNumberCalls.Load(), + filterLogs: chain.filterLogsCalls.Load(), + farAheadBlocks: chain.farAheadBlockByNumberCalls.Load(), } } @@ -214,6 +215,8 @@ func TestMonitorPrefetchSlowChainNoExtraCalls(t *testing.T) { t.Logf("prefetch=4: %+v", on) assert.Zero(t, off.blockNumber) + assert.Zero(t, off.farAheadBlocks) + assert.Zero(t, on.farAheadBlocks, "prefetch asked for a block beyond the next block at the head") if streaming { // the stream supplies the head assert.Zero(t, on.blockNumber) @@ -223,9 +226,10 @@ func TestMonitorPrefetchSlowChainNoExtraCalls(t *testing.T) { assert.LessOrEqual(t, on.blockNumber, int64(1)) } - // ~10 blocks made in the run: allow a couple of calls of jitter - // between the two runs, well short of a block per worker - assert.InDelta(t, off.blockByNumber, on.blockByNumber, 3) + // Compare requests that found blocks. Ordinary polling retries for the + // next missing block depend on timer scheduling and cannot be compared + // across independent runs. Allow jitter in the number of produced blocks. + assert.InDelta(t, off.foundBlocks, on.foundBlocks, 3) assert.InDelta(t, off.filterLogs, on.filterLogs, 3) }) } @@ -543,9 +547,11 @@ type fakeChain struct { announceAhead uint64 // node calls served, by method - blockNumberCalls atomic.Int64 - blockByNumberCalls atomic.Int64 - filterLogsCalls atomic.Int64 + blockNumberCalls atomic.Int64 + blockByNumberCalls atomic.Int64 + foundBlockByNumberCalls atomic.Int64 + farAheadBlockByNumberCalls atomic.Int64 + filterLogsCalls atomic.Int64 mu sync.Mutex canonical []common.Hash // canonical[i] is block base+i @@ -701,6 +707,9 @@ func (p *fakeProvider) BlockNumber(ctx context.Context) (uint64, error) { func (p *fakeProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json.RawMessage, error) { p.chain.blockByNumberCalls.Add(1) + if num != nil && num.Uint64() > p.chain.head()+1 { + p.chain.farAheadBlockByNumberCalls.Add(1) + } if err := p.chain.wait(ctx); err != nil { return nil, err } @@ -708,6 +717,7 @@ func (p *fakeProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json if !ok { return nil, ethereum.NotFound } + p.chain.foundBlockByNumberCalls.Add(1) return b.payload(), nil } From 987262d69722820de861cc166cf4672f63fc1e56 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Fri, 2 Oct 2026 14:39:30 -0400 Subject: [PATCH 03/11] fix(ethmonitor): reject malformed logs before caching --- ethmonitor/ethmonitor.go | 21 ++++- ethmonitor/prefetch_internal_test.go | 135 +++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 2 deletions(-) diff --git a/ethmonitor/ethmonitor.go b/ethmonitor/ethmonitor.go index c1c70b84..041c7104 100644 --- a/ethmonitor/ethmonitor.go +++ b/ethmonitor/ethmonitor.go @@ -851,6 +851,8 @@ func (m *Monitor) logTopics() [][]common.Hash { } func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics [][]common.Hash, blockBloom types.Bloom) ([]types.Log, []byte, error) { + var fetchedLogs []types.Log + getter := func(ctx context.Context, _ string) ([]byte, error) { if m.options.DebugLogging { m.log.Debug(fmt.Sprintf("ethmonitor: filterLogs is calling origin for block hash %s", blockHash)) @@ -873,6 +875,12 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics // ensure we do not incorrectly cache an empty block-logs response as valid. return nil, fmt.Errorf("ethmonitor: filterLogs detected empty block-logs response but block bloom is set, ignoring node response") } + // Validate before caching so a malformed response cannot block log + // backfilling until cache expiry. + fetchedLogs, err = m.unmarshalLogs(logsPayload) + if err != nil { + return nil, err + } return logsPayload, nil } @@ -881,8 +889,7 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics if err != nil { return nil, resp, err } - logs, err := m.unmarshalLogs(resp) - return logs, resp, err + return fetchedLogs, resp, nil } key := CacheKeyBlockLogs(m.chainID, blockHash, topics) @@ -890,7 +897,17 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics if err != nil { return nil, resp, err } + if fetchedLogs != nil { + return fetchedLogs, resp, nil + } logs, err := m.unmarshalLogs(resp) + if err != nil { + // Recover entries cached by peers or older monitors that did not + // validate logs before writing them. + if deleteErr := m.cache.Delete(ctx, key); deleteErr != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting invalid logs cache for block hash %s due to: '%v'", blockHash.Hex(), deleteErr)) + } + } return logs, resp, err } diff --git a/ethmonitor/prefetch_internal_test.go b/ethmonitor/prefetch_internal_test.go index 259ae05e..0bdb593a 100644 --- a/ethmonitor/prefetch_internal_test.go +++ b/ethmonitor/prefetch_internal_test.go @@ -5,10 +5,14 @@ import ( "encoding/json" "fmt" "math/big" + "sync/atomic" "testing" "time" "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) @@ -35,6 +39,137 @@ func TestPrefetchRejectsInvalidBlockBeforeCaching(t *testing.T) { } } +func TestMonitorRecoversInvalidPrefetchedLogs(t *testing.T) { + for _, payload := range []string{`{}`, `[{}]`} { + for _, cached := range []bool{false, true} { + t.Run(fmt.Sprintf("payload=%s/cached=%v", payload, cached), func(t *testing.T) { + const first, target, last = uint64(1000), uint64(1002), uint64(1004) + targetHash := common.BigToHash(new(big.Int).SetUint64(target)) + expectedLogs := []types.Log{{ + Address: common.HexToAddress("0x1234"), + Topics: []common.Hash{common.HexToHash("0xabcd")}, + Data: []byte{1, 2}, + TxHash: common.HexToHash("0x5678"), + BlockHash: targetHash, + BlockNumber: target, + }} + validPayload, err := json.Marshal(expectedLogs) + require.NoError(t, err) + + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + provider.EXPECT().ChainID(gomock.Any()).Return(big.NewInt(1), nil).AnyTimes() + provider.EXPECT().IsStreamingEnabled().Return(false).AnyTimes() + provider.EXPECT().BlockNumber(gomock.Any()).Return(last, nil).AnyTimes() + provider.EXPECT().RawBlockByNumber(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, number *big.Int) (json.RawMessage, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if number.Uint64() > last { + return nil, ethereum.NotFound + } + var bloom types.Bloom + if number.Uint64() == target { + bloom = types.BytesToBloom([]byte{1}) + } + header := &types.Header{ + Number: number, + ParentHash: common.BigToHash(new(big.Int).Sub(number, big.NewInt(1))), + Difficulty: big.NewInt(0), + GasLimit: 30_000_000, + Bloom: bloom, + } + header.SetHash(common.BigToHash(number)) + return json.Marshal(header) + }, + ).AnyTimes() + var originCalls atomic.Int64 + provider.EXPECT().RawFilterLogs(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, query ethereum.FilterQuery) (json.RawMessage, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if *query.BlockHash != targetHash { + return json.RawMessage(`[]`), nil + } + if originCalls.Add(1) == 1 && !cached { + return json.RawMessage(payload), nil + } + return json.RawMessage(validPayload), nil + }, + ).AnyTimes() + + opts := DefaultOptions + opts.WithLogs = true + opts.PrefetchConcurrency = 1 + opts.StartBlockNumber = new(big.Int).SetUint64(first) + opts.PollingInterval = 10 * time.Millisecond + opts.CacheExpiry = time.Hour + // Leave CacheBackend unset to exercise automatic memory caching. + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID = big.NewInt(1) + key := CacheKeyBlockLogs(monitor.chainID, targetHash, monitor.logTopics()) + if cached { + require.NoError(t, monitor.cache.SetEx(context.Background(), key, []byte(payload), time.Hour)) + } else { + // Complete the speculative fetch before starting the serial loop, + // so the worker deterministically receives the malformed response. + monitor.prefetch.fetch(context.Background(), prefetchJob{num: target}) + require.Equal(t, int64(1), originCalls.Load()) + _, found, err := monitor.cache.Get(context.Background(), key) + require.NoError(t, err) + require.False(t, found, "failed prefetch must not cache malformed logs") + } + + sub := monitor.Subscribe("TestMonitorRecoversInvalidPrefetchedLogs") + defer sub.Unsubscribe() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- monitor.Run(ctx) }() + defer func() { + cancel() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Error("monitor did not stop") + } + }() + + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + next := first + for next <= last { + select { + case blocks := <-sub.Blocks(): + for _, block := range blocks { + require.Equal(t, Added, block.Event) + require.True(t, block.OK) + require.Equal(t, next, block.NumberU64(), "blocks must publish in order") + if next == target { + require.Equal(t, expectedLogs, block.Logs) + } + next++ + } + case <-timer.C: + t.Fatalf("publication stalled at block %d; origin calls: %d", next, originCalls.Load()) + } + } + if !cached { + require.GreaterOrEqual(t, originCalls.Load(), int64(2), "malformed prefetch must allow an origin retry") + } else { + require.Positive(t, originCalls.Load(), "invalid cache entry must allow an origin retry") + } + stored, found, err := monitor.cache.Get(context.Background(), key) + require.NoError(t, err) + require.True(t, found) + require.JSONEq(t, string(validPayload), string(stored)) + }) + } + } +} + func TestPrefetchPanicStopsWorkers(t *testing.T) { provider := mocks.NewMockRawInterface(gomock.NewController(t)) provider.EXPECT().BlockNumber(gomock.Any()).DoAndReturn(func(context.Context) (uint64, error) { From 247c2e19340d297056992200e4bf2fd675bc3622 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Mon, 5 Oct 2026 10:10:50 -0400 Subject: [PATCH 04/11] ethmonitor: make the prefetched reorg test deterministic TestMonitorPrefetchReorg/prefetched=true raced the monitor: it waited for a block 2-4 past the monitor head to show up in the cache, assuming the monitor (~15ms a block) couldn't reach it first. On slow CI runners (macos-latest) the monitor either drained the fixed chain before the condition was seen, failing the test, or reached the reorg block first, passing without exercising the stale prefetched block at all. Hold block 1060 on the fake chain instead, so the monitor stalls before it while the workers prefetch past it. Reorg from 1062 once it is cached from the old fork, then release. Also assert the reorg was actually exercised: a block from the abandoned fork must be published and removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- ethmonitor/prefetch_test.go | 94 ++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 18 deletions(-) diff --git a/ethmonitor/prefetch_test.go b/ethmonitor/prefetch_test.go index 1c84edc0..06f86822 100644 --- a/ethmonitor/prefetch_test.go +++ b/ethmonitor/prefetch_test.go @@ -107,35 +107,38 @@ func testMonitorPrefetchReorg(t *testing.T, concurrency int, prefetched bool) { } }() + // To reorg at a block the prefetcher cached from the abandoned fork, the + // monitor must not reach that block first. Rather than racing it, hold + // block 1060 so the monitor stalls before it while the workers fill the + // window past it, then reorg from 1062 and let the monitor continue. + const heldBlock, stalePrefetched = 1060, 1062 + if prefetched { + chain.hold(heldBlock) + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() go monitor.Run(ctx) defer monitor.Stop() - require.Eventually(t, func() bool { - return monitor.LatestBlockNum().Uint64() >= 1050 - }, 10*time.Second, time.Millisecond) - var reorgAt uint64 if prefetched { - // the monitor takes ~15ms a block here, so a block cached two or - // more past its head is still ahead of it when we reorg require.Eventually(t, func() bool { - head := monitor.LatestBlockNum().Uint64() - for n := head + 4; n >= head+2; n-- { - key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), new(big.Int).SetUint64(n)) - if _, ok, _ := cache.Get(context.Background(), key); ok { - reorgAt = n - return true - } - } - return false - }, 5*time.Second, time.Millisecond) + key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(stalePrefetched)) + _, ok, _ := cache.Get(context.Background(), key) + return ok + }, 10*time.Second, time.Millisecond, "block %d was never prefetched", stalePrefetched) + require.Less(t, monitor.LatestBlockNum().Uint64(), uint64(heldBlock)) + reorgAt = stalePrefetched } else { + require.Eventually(t, func() bool { + return monitor.LatestBlockNum().Uint64() >= 1050 + }, 10*time.Second, time.Millisecond) reorgAt = monitor.LatestBlockNum().Uint64() - 2 } chain.reorgFrom(reorgAt) t.Logf("reorged chain from block %d, monitor head %d", reorgAt, monitor.LatestBlockNum().Uint64()) + chain.release(heldBlock) require.Eventually(t, func() bool { head := monitor.LatestReadyBlock() @@ -154,6 +157,11 @@ func testMonitorPrefetchReorg(t *testing.T, concurrency int, prefetched bool) { for num, hash := range events.blocks() { assert.Equal(t, chain.hashAt(num), hash, "subscriber block %d", num) } + + // both cases publish abandoned-fork blocks before the reorg is seen: in + // the prefetched case the stale block extends the head, so it is accepted + // and must be reverted once the new fork shows up. + assert.Positive(t, events.removedCount(), "the reorg was never exercised") } // TestMonitorPrefetchHeadAhead has the node announce heads it cannot serve @@ -558,6 +566,7 @@ type fakeChain struct { blocks map[common.Hash]fakeBlock fork int heads []*fakeSubscription + held map[uint64]chan struct{} // block numbers not served until released } type fakeBlock struct { @@ -623,6 +632,44 @@ func (c *fakeChain) reorgFrom(num uint64) { } } +// hold stops the chain serving block num by number until release(num). +func (c *fakeChain) hold(num uint64) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + c.held = map[uint64]chan struct{}{} + } + c.held[num] = make(chan struct{}) +} + +func (c *fakeChain) release(num uint64) { + c.mu.Lock() + defer c.mu.Unlock() + if ch, ok := c.held[num]; ok { + close(ch) + delete(c.held, num) + } +} + +// waitIfHeld blocks while block num is held. +func (c *fakeChain) waitIfHeld(ctx context.Context, num *big.Int) error { + if num == nil { + return nil + } + c.mu.Lock() + ch := c.held[num.Uint64()] + c.mu.Unlock() + if ch == nil { + return nil + } + select { + case <-ch: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + func (c *fakeChain) head() uint64 { c.mu.Lock() defer c.mu.Unlock() @@ -713,6 +760,9 @@ func (p *fakeProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json if err := p.chain.wait(ctx); err != nil { return nil, err } + if err := p.chain.waitIfHeld(ctx, num); err != nil { + return nil, err + } b, ok := p.chain.byNumber(num) if !ok { return nil, ethereum.NotFound @@ -773,8 +823,9 @@ func (s *fakeSubscription) Err() <-chan error { // eventLog rebuilds the chain a subscriber sees from the monitor's events, // checking each event is consistent with what came before. type eventLog struct { - mu sync.Mutex - chain []fakeBlock + mu sync.Mutex + chain []fakeBlock + removed int } func newEventLog() *eventLog { @@ -798,10 +849,17 @@ func (l *eventLog) apply(t *testing.T, blocks ethmonitor.Blocks) { assert.Equal(t, l.chain[n-1].hash, b.Hash(), "removed block %d is not the head", b.NumberU64()) l.chain = l.chain[:n-1] } + l.removed++ } } } +func (l *eventLog) removedCount() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.removed +} + func (l *eventLog) head() common.Hash { l.mu.Lock() defer l.mu.Unlock() From 77bbef8b2ac2a3fcb70b360a9b2ff2ffacd1be69 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Mon, 5 Oct 2026 15:39:01 -0400 Subject: [PATCH 05/11] fix(ethmonitor): confirm cached parents and track canonical incarnations --- ethmonitor/bootstrap.go | 6 +- ethmonitor/canonical_state_test.go | 179 +++++++++++++++++++++++++++++ ethmonitor/chain.go | 53 +++++++-- ethmonitor/ethmonitor.go | 19 +-- ethmonitor/prefetch_test.go | 134 +++++++++++++++++---- 5 files changed, 350 insertions(+), 41 deletions(-) create mode 100644 ethmonitor/canonical_state_test.go diff --git a/ethmonitor/bootstrap.go b/ethmonitor/bootstrap.go index c4c28f6d..d616c6e2 100644 --- a/ethmonitor/bootstrap.go +++ b/ethmonitor/bootstrap.go @@ -38,8 +38,9 @@ func (c *Chain) bootstrapBlocks(blocks Blocks) error { return nil } - if len(blocks) == 1 { + if len(blocks) == 1 && blocks[0].Event != Added { c.blocks = blocks.Copy() + c.blocks[0].canonicalState = nil return nil } @@ -51,7 +52,7 @@ func (c *Chain) bootstrapBlocks(blocks Blocks) error { for _, b := range blocks { if b.Event == Added { - err := c.push(b) + _, err := c.push(b) if err != nil { return fmt.Errorf("ethmonitor: bootstrap failed to build canonical chain: %w", err) } @@ -101,5 +102,6 @@ func (b *Block) UnmarshalJSON(data []byte) error { b.Event = s.Event b.Logs = s.Logs b.OK = s.OK + b.canonicalState = nil return nil } diff --git a/ethmonitor/canonical_state_test.go b/ethmonitor/canonical_state_test.go new file mode 100644 index 00000000..e6ca0e64 --- /dev/null +++ b/ethmonitor/canonical_state_test.go @@ -0,0 +1,179 @@ +package ethmonitor + +import ( + "encoding/json" + "fmt" + "math/big" + "testing" + + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/stretchr/testify/require" +) + +func canonicalTestBlock(num int64) *Block { + return &Block{ + Block: types.NewBlockWithHeader(&types.Header{ + Number: big.NewInt(num), + BlockHash: common.BigToHash(big.NewInt(num)), + ParentHash: common.BigToHash(big.NewInt(num - 1)), + Time: uint64(num), + }), + Event: Added, + OK: true, + } +} + +func pushCanonicalTestBlock(t *testing.T, chain *Chain, input *Block) *Block { + t.Helper() + chain.push(input) + block := chain.Head() + require.NotNil(t, block) + require.Equal(t, input.Hash(), block.Hash()) + return block +} + +// The interface keeps the before-fix proof executable without the new method. +func canonicalTestState(t *testing.T, block *Block) (uint64, bool) { + t.Helper() + state, ok := any(block).(interface{ CanonicalState() (uint64, bool) }) + if !ok { + return 0, false + } + return state.CanonicalState() +} + +func TestBlockCanonicalStateRetention(t *testing.T) { + for _, depth := range []int64{1, 3} { + t.Run(fmt.Sprintf("evictionDepth=%d", depth), func(t *testing.T) { + chain := newChain(10, false) + block := pushCanonicalTestBlock(t, chain, canonicalTestBlock(100)) + copy := chain.Blocks().Copy()[0] + incarnation, canonical := canonicalTestState(t, block) + require.Positive(t, incarnation) + require.True(t, canonical) + for num := int64(101); num <= 109+depth; num++ { + pushCanonicalTestBlock(t, chain, canonicalTestBlock(num)) + } + require.Nil(t, chain.GetBlock(block.Hash())) + for _, snapshot := range []*Block{block, copy} { + got, canonical := canonicalTestState(t, snapshot) + require.Equal(t, incarnation, got) + require.True(t, canonical, "retention eviction was mistaken for removal") + } + }) + } +} + +func TestBlockCanonicalStateReadoption(t *testing.T) { + chain := newChain(10, false) + input := canonicalTestBlock(100) + block := pushCanonicalTestBlock(t, chain, input) + shallow := *block + copy := chain.Blocks().Copy()[0] + incarnation, canonical := canonicalTestState(t, block) + require.Positive(t, incarnation) + require.True(t, canonical) + removed := *chain.pop() + removed.Event = Removed + for _, snapshot := range []*Block{block, &shallow, copy, &removed} { + got, canonical := canonicalTestState(t, snapshot) + require.Equal(t, incarnation, got) + require.False(t, canonical, "snapshot did not observe the actual removal") + } + + // Reusing either the original input or a removed snapshot must create a fresh + // owned incarnation without mutating old Added/Removed event copies. + for _, reused := range []*Block{input, copy} { + fresh := pushCanonicalTestBlock(t, chain, reused) + got, canonical := canonicalTestState(t, fresh) + require.Greater(t, got, incarnation) + require.True(t, canonical) + for _, snapshot := range []*Block{block, &shallow, copy, &removed} { + got, canonical := canonicalTestState(t, snapshot) + require.Equal(t, incarnation, got) + require.False(t, canonical, "fresh readoption revived an old event") + } + chain.pop() + } +} + +func TestBlockCanonicalStateBootstrap(t *testing.T) { + for _, count := range []int{1, 3} { + for _, serialized := range []bool{false, true} { + t.Run(fmt.Sprintf("blocks=%d/JSON=%v", count, serialized), func(t *testing.T) { + inputs := make(Blocks, count) + for i := range inputs { + inputs[i] = canonicalTestBlock(int64(100 + i)) + } + chain := newChain(10, true) + if serialized { + data, err := json.Marshal(inputs) + require.NoError(t, err) + require.NoError(t, chain.BootstrapFromBlocksJSON(data)) + } else { + require.NoError(t, chain.BootstrapFromBlocks(inputs)) + } + for _, block := range chain.Blocks().Copy() { + incarnation, canonical := canonicalTestState(t, block) + require.Positive(t, incarnation) + require.True(t, canonical) + data, err := json.Marshal(block) + require.NoError(t, err) + require.NotContains(t, string(data), "incarnation") + require.NoError(t, json.Unmarshal(data, block)) + incarnation, canonical = canonicalTestState(t, block) + require.Zero(t, incarnation, "serialized state preserved runtime ownership") + require.False(t, canonical) + } + }) + } + } +} + +func TestBlockCanonicalStateConcurrentRemoval(t *testing.T) { + chain := newChain(10, false) + input := canonicalTestBlock(100) + block := pushCanonicalTestBlock(t, chain, input) + incarnation, _ := canonicalTestState(t, block) + require.Positive(t, incarnation) + state := any(block).(interface{ CanonicalState() (uint64, bool) }) + start, done := make(chan struct{}), make(chan struct{}) + result := make(chan error, 1) + go func() { + close(start) + removed := false + for { + got, canonical := state.CanonicalState() + if got != incarnation || (removed && canonical) { + result <- fmt.Errorf("old incarnation changed or revived: id=%d canonical=%v", got, canonical) + return + } + removed = removed || !canonical + select { + case <-done: + result <- nil + return + default: + } + } + }() + <-start + for i := 0; i < 100; i++ { + chain.pop() + pushCanonicalTestBlock(t, chain, input) + } + close(done) + require.NoError(t, <-result) + got, canonical := state.CanonicalState() + require.Equal(t, incarnation, got) + require.False(t, canonical) +} + +func TestBlockCanonicalStateUntracked(t *testing.T) { + for _, block := range []*Block{nil, canonicalTestBlock(100)} { + incarnation, canonical := canonicalTestState(t, block) + require.Zero(t, incarnation) + require.False(t, canonical) + } +} diff --git a/ethmonitor/chain.go b/ethmonitor/chain.go index 8685a862..7233b5d0 100644 --- a/ethmonitor/chain.go +++ b/ethmonitor/chain.go @@ -7,6 +7,7 @@ import ( "math/big" "slices" "sync" + "sync/atomic" "github.com/0xsequence/ethkit/go-ethereum" "github.com/0xsequence/ethkit/go-ethereum/common" @@ -28,6 +29,7 @@ type Chain struct { mu sync.RWMutex averageBlockTime float64 // in seconds + lastIncarnation uint64 } func newChain(retentionLimit int, bootstrapMode bool) *Chain { @@ -59,7 +61,7 @@ func newChain(retentionLimit int, bootstrapMode bool) *Chain { // } // Push to the top of the stack -func (c *Chain) push(nextBlock *Block) error { +func (c *Chain) push(nextBlock *Block) (*Block, error) { c.mu.Lock() defer c.mu.Unlock() @@ -70,12 +72,12 @@ func (c *Chain) push(nextBlock *Block) error { // Assert pointing at prev block if nextBlock.ParentHash() != headBlock.Hash() { - return ErrUnexpectedParentHash + return nil, ErrUnexpectedParentHash } // Assert block numbers are in sequence if nextBlock.NumberU64() != headBlock.NumberU64()+1 { - return ErrUnexpectedBlockNumber + return nil, ErrUnexpectedBlockNumber } // Update average block time @@ -86,14 +88,20 @@ func (c *Chain) push(nextBlock *Block) error { } } + // Each adoption owns its state so reusing an input cannot revive old events. + c.lastIncarnation++ + block := *nextBlock + block.canonicalState = &blockCanonicalState{incarnation: c.lastIncarnation} + block.canonicalState.canonical.Store(true) + // Add to head of stack - c.blocks = append(c.blocks, nextBlock) + c.blocks = append(c.blocks, &block) if len(c.blocks) > c.retentionLimit { c.blocks[0] = nil c.blocks = c.blocks[1:] } - return nil + return &block, nil } // Pop from the top of the stack @@ -107,6 +115,9 @@ func (c *Chain) pop() *Block { n := len(c.blocks) - 1 block := c.blocks[n] + if block.canonicalState != nil { + block.canonicalState.canonical.Store(false) + } c.blocks[n] = nil c.blocks = c.blocks[:n] return block @@ -215,6 +226,9 @@ const ( Removed ) +// Block contains a monitored block and its event data. +// Construct values with keyed composite literals: private canonical state makes +// positional literals unsupported. type Block struct { *types.Block @@ -228,6 +242,26 @@ type Block struct { // OK flag which represents the block is ready for broadcasting OK bool + + canonicalState *blockCanonicalState +} + +type blockCanonicalState struct { + incarnation uint64 + canonical atomic.Bool +} + +// CanonicalState reports the monitor-assigned incarnation and whether it has +// remained canonical without a known removal. Zero means the block is untracked. +// Retention eviction preserves this state; in-memory copies share removal updates. +// Incarnations are local to a monitor chain. Serialized blocks are untracked +// until accepted by a monitor. +func (b *Block) CanonicalState() (incarnation uint64, canonical bool) { + if b == nil || b.canonicalState == nil { + return 0, false + } + state := b.canonicalState + return state.incarnation, state.canonical.Load() } type Blocks []*Block @@ -560,10 +594,11 @@ func (blocks Blocks) Copy() Blocks { } nb[i] = &Block{ - Block: b.Block, - Event: b.Event, - Logs: logs, - OK: b.OK, + Block: b.Block, + Event: b.Event, + Logs: logs, + OK: b.OK, + canonicalState: b.canonicalState, } } diff --git a/ethmonitor/ethmonitor.go b/ethmonitor/ethmonitor.go index 041c7104..d59c4898 100644 --- a/ethmonitor/ethmonitor.go +++ b/ethmonitor/ethmonitor.go @@ -640,10 +640,10 @@ func (m *Monitor) monitor(ctx context.Context, wg *sync.WaitGroup) error { // fetch the next block, either via the stream or via a poll nextBlock, nextBlockPayload, miss, err := m.fetchNextBlock(ctx, false) - // with prefetching, a cached block may be a stale entry from an abandoned - // fork, written by a prefetch worker of this or a peer monitor. confirm a - // parent mismatch with the node before treating it as a reorg. - if err == nil && m.prefetch != nil && headBlock != nil && nextBlock.ParentHash() != headBlock.Hash() { + // A cached block may be a stale entry from an abandoned fork, written by + // this or a peer monitor even when local prefetching is disabled. Confirm + // a parent mismatch with the node before treating it as a reorg. + if err == nil && m.cache != nil && headBlock != nil && nextBlock.ParentHash() != headBlock.Hash() { nextBlock, nextBlockPayload, miss, err = m.refetchNextBlock(ctx, nextBlock) } @@ -730,9 +730,11 @@ func (m *Monitor) buildCanonicalChain(ctx context.Context, nextBlock *types.Bloc if headBlock == nil || nextBlock.ParentHash() == headBlock.Hash() { // block-chaining it up - block := &Block{Event: Added, Block: nextBlock} - events = append(events, block) - return events, m.chain.push(block) + block, err := m.chain.push(&Block{Event: Added, Block: nextBlock}) + if err != nil { + return events, err + } + return append(events, block), nil } // next block doest match prevHash, therefore we must pop our previous block and recursively @@ -778,8 +780,7 @@ func (m *Monitor) buildCanonicalChain(ctx context.Context, nextBlock *types.Bloc return events, err } - block := &Block{Event: Added, Block: nextBlock} - err = m.chain.push(block) + block, err := m.chain.push(&Block{Event: Added, Block: nextBlock}) if err != nil { return events, err } diff --git a/ethmonitor/prefetch_test.go b/ethmonitor/prefetch_test.go index 06f86822..d1713576 100644 --- a/ethmonitor/prefetch_test.go +++ b/ethmonitor/prefetch_test.go @@ -284,40 +284,132 @@ func TestMonitorPrefetchShutdownNoGoroutineLeak(t *testing.T) { // A peer can complete an old-fork fetch between DEL and the confirmation read, // and a cache deletion can fail. Neither may cause a false canonical removal. func TestMonitorPrefetchRefetchBypassesCache(t *testing.T) { - for _, deleteFails := range []bool{false, true} { - t.Run(fmt.Sprintf("deleteFails=%v", deleteFails), func(t *testing.T) { - chain := newFakeChain(1000, 20, time.Millisecond) - oldNext, ok := chain.byNumber(big.NewInt(1001)) - require.True(t, ok) - chain.reorgFrom(1000) + for _, concurrency := range []int{0, 1} { + for _, deleteFails := range []bool{false, true} { + t.Run(fmt.Sprintf("prefetch=%d/deleteFails=%v", concurrency, deleteFails), func(t *testing.T) { + chain := newFakeChain(1000, 20, time.Millisecond) + oldNext, ok := chain.byNumber(big.NewInt(1001)) + require.True(t, ok) + chain.reorgFrom(1000) + + backend, err := memcache.NewBackend(512) + require.NoError(t, err) + key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(1001)) + racingBackend := &repopulatingBackend{ + Backend: backend, key: key, payload: oldNext.payload(), deleteFails: deleteFails, + } + require.NoError(t, racingBackend.SetEx(context.Background(), key, []byte(oldNext.payload()), time.Minute)) + monitor := newTestMonitor(t, chain, false, concurrency, 1, racingBackend) + sub := monitor.Subscribe("TestMonitorPrefetchRefetchBypassesCache") + defer sub.Unsubscribe() + runMonitorForTest(t, monitor) - backend, err := memcache.NewBackend(512) - require.NoError(t, err) - key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(1001)) - racingBackend := &repopulatingBackend{ - Backend: backend, key: key, payload: oldNext.payload(), deleteFails: deleteFails, + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + for { + select { + case blocks := <-sub.Blocks(): + for _, b := range blocks { + require.Equal(t, ethmonitor.Added, b.Event, "removed canonical block %d", b.NumberU64()) + require.Equal(t, chain.hashAt(b.NumberU64()), b.Hash()) + if b.NumberU64() >= 1001 { + require.Positive(t, racingBackend.deletes.Load(), "did not exercise the mismatch confirmation") + return + } + } + case <-timer.C: + t.Fatal("monitor did not advance through the stale cache entry") + } + } + }) + } + } +} + +// Cache confirmation must preserve genuine reorgs, including when prefetching +// is disabled. An uncached monitor still follows the direct-origin path. +func TestMonitorParentMismatchReorg(t *testing.T) { + cases := []struct { + concurrency int + cached bool + }{{0, false}, {0, true}, {1, true}} + for _, tc := range cases { + t.Run(fmt.Sprintf("prefetch=%d/cached=%v", tc.concurrency, tc.cached), func(t *testing.T) { + chain := newFakeChain(1000, 1, time.Millisecond) + oldHash := chain.hashAt(1000) + var monitor *ethmonitor.Monitor + var tracker *repopulatingBackend + if tc.cached { + backend, err := memcache.NewBackend(512) + require.NoError(t, err) + tracker = &repopulatingBackend{Backend: backend, key: ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(1001)), deleteFails: true} + monitor = newTestMonitor(t, chain, false, tc.concurrency, 1, tracker) + } else { + monitor = newTestMonitor(t, chain, false, tc.concurrency, 1) + require.Nil(t, monitor.Options().CacheBackend) } - require.NoError(t, racingBackend.SetEx(context.Background(), key, []byte(oldNext.payload()), time.Minute)) - monitor := newTestMonitor(t, chain, false, 1, 1, racingBackend) - sub := monitor.Subscribe("TestMonitorPrefetchRefetchBypassesCache") + sub := monitor.Subscribe("TestMonitorParentMismatchReorg") defer sub.Unsubscribe() runMonitorForTest(t, monitor) - - timer := time.NewTimer(5 * time.Second) + state := func(block *ethmonitor.Block) (uint64, bool) { + t.Helper() + witness, ok := any(block).(interface{ CanonicalState() (uint64, bool) }) + require.True(t, ok) + return witness.CanonicalState() + } + var initial *ethmonitor.Block + var oldIncarnation uint64 + select { + case blocks := <-sub.Blocks(): + require.Len(t, blocks, 1) + require.Equal(t, ethmonitor.Added, blocks[0].Event) + require.Equal(t, oldHash, blocks[0].Hash()) + initial = blocks[0] + var canonical bool + oldIncarnation, canonical = state(initial) + require.Positive(t, oldIncarnation) + require.True(t, canonical) + case <-time.After(5 * time.Second): + t.Fatal("missing initial canonical block") + } + chain.reorgFrom(1000) + chain.mu.Lock() + chain.appendLocked() + chain.mu.Unlock() + timer := time.NewTimer(8 * time.Second) defer timer.Stop() + removed, added := 0, 0 for { select { case blocks := <-sub.Blocks(): for _, b := range blocks { - require.Equal(t, ethmonitor.Added, b.Event, "removed canonical block %d", b.NumberU64()) - require.Equal(t, chain.hashAt(b.NumberU64()), b.Hash()) - if b.NumberU64() >= 1001 { - require.Positive(t, racingBackend.deletes.Load(), "did not exercise the mismatch confirmation") + if b.Event == ethmonitor.Removed { + require.Equal(t, uint64(1000), b.NumberU64()) + require.Equal(t, oldHash, b.Hash()) + incarnation, canonical := state(b) + require.Equal(t, oldIncarnation, incarnation) + require.False(t, canonical) + _, canonical = state(initial) + require.False(t, canonical, "queued Added did not observe removal") + removed++ + } else { + require.Equal(t, chain.hashAt(b.NumberU64()), b.Hash()) + incarnation, canonical := state(b) + require.Greater(t, incarnation, oldIncarnation) + require.True(t, canonical) + added++ + } + if b.Event == ethmonitor.Added && b.NumberU64() == 1001 { + require.Equal(t, 1, removed) + require.Equal(t, 2, added) + if tracker != nil { + require.Positive(t, tracker.deletes.Load(), "cached parent mismatch was not confirmed") + } return } } case <-timer.C: - t.Fatal("monitor did not advance through the stale cache entry") + t.Fatal("real reorg did not recover to new canonical block 1001") } } }) From 95199aee607ec7075cc95de8c1780b617cac57e2 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 12:56:09 -0400 Subject: [PATCH 06/11] Harden ethreceipts rollback, filter ownership and cancellation (#220) * fix(ethreceipts): harden rollback, filter ownership and cancellation * fix(ethreceipts): preserve retry candidates and filter identity Keep successful canceled retries tied to their learned block generation, reopen canonical adopted blocks after retention, and support stable custom value filter removal. Document supported limits and add regression coverage. * ci: test Go 1.25 and 1.26 * fix(ethreceipts): retry current candidates after stale RPC receipts * refactor(ethreceipts): pass block processing contexts explicitly * refactor(ethreceipts): restore breaker retries with v0.3.2 * test(ethreceipts): organize regressions by behavior * refactor(ethreceipts): clarify pending receipt retries * fix(ethreceipts): harden subscription cleanup and finality * fix(ethreceipts): preserve fetch filter compatibility * fix(ethmonitor): reject invalid empty log responses --- .github/workflows/ci.yml | 2 +- Makefile | 5 +- ethmonitor/ethmonitor.go | 21 +- ethmonitor/prefetch_internal_test.go | 17 +- ethreceipts/ethreceipts.go | 493 ++++++++----- ethreceipts/ethreceipts_test.go | 901 +++++++++-------------- ethreceipts/filterer.go | 53 +- ethreceipts/finalizer.go | 154 ++-- ethreceipts/finalizer_test.go | 215 ++++++ ethreceipts/helpers_test.go | 231 ++++++ ethreceipts/lifecycle_test.go | 282 +++++++ ethreceipts/ownership_regression_test.go | 572 ++++++++++++++ ethreceipts/pending_receipts_test.go | 357 +++++++++ ethreceipts/receipt.go | 36 +- ethreceipts/receipt_fetch_test.go | 258 +++++++ ethreceipts/reorg_test.go | 773 +++++++++++++++++++ ethreceipts/stale_receipt_test.go | 170 +++++ ethreceipts/subscription.go | 765 +++++++++++++------ go.mod | 2 +- go.sum | 5 +- 20 files changed, 4251 insertions(+), 1061 deletions(-) create mode 100644 ethreceipts/finalizer_test.go create mode 100644 ethreceipts/helpers_test.go create mode 100644 ethreceipts/lifecycle_test.go create mode 100644 ethreceipts/ownership_regression_test.go create mode 100644 ethreceipts/pending_receipts_test.go create mode 100644 ethreceipts/receipt_fetch_test.go create mode 100644 ethreceipts/reorg_test.go create mode 100644 ethreceipts/stale_receipt_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11046a93..6bf49709 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - go-version: [1.24.x,1.25.x] + go-version: [1.25.x,1.26.x] os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} diff --git a/Makefile b/Makefile index 6621a01d..5697fe25 100644 --- a/Makefile +++ b/Makefile @@ -56,13 +56,14 @@ install: # Run baseline tests test: check-testchain-running go-test +# Packages share one testchain and deployment sender, so serialize package binaries. # Go test short-hand, and skip testing go-ethereum go-test: test-clean - GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -race -run=$(TEST) `go list ./... | grep -v go-ethereum` + GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -p=1 -race -run=$(TEST) `go list ./... | grep -v go-ethereum` # Go test short-hand, including go-ethereum go-test-all: test-clean - GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -run=$(TEST) ./... + GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -p=1 -run=$(TEST) ./... test-clean: GOGC=off go clean -testcache diff --git a/ethmonitor/ethmonitor.go b/ethmonitor/ethmonitor.go index d59c4898..37acf283 100644 --- a/ethmonitor/ethmonitor.go +++ b/ethmonitor/ethmonitor.go @@ -869,16 +869,9 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics if err != nil { return nil, err } - if blockBloom != (types.Bloom{}) && (len(logsPayload) == 0 || (len(logsPayload) == 2 && logsPayload[0] == '[' && logsPayload[1] == ']')) { - // If we have no logs and the block bloom is set, then we need to return an error - // as the node is incorrectly telling us the block-logs response is '[]' but in fact - // the block log bloom filter tells us we should be expecting logs. We do this to - // ensure we do not incorrectly cache an empty block-logs response as valid. - return nil, fmt.Errorf("ethmonitor: filterLogs detected empty block-logs response but block bloom is set, ignoring node response") - } // Validate before caching so a malformed response cannot block log // backfilling until cache expiry. - fetchedLogs, err = m.unmarshalLogs(logsPayload) + fetchedLogs, err = m.unmarshalLogs(logsPayload, blockBloom) if err != nil { return nil, err } @@ -901,7 +894,7 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics if fetchedLogs != nil { return fetchedLogs, resp, nil } - logs, err := m.unmarshalLogs(resp) + logs, err := m.unmarshalLogs(resp, blockBloom) if err != nil { // Recover entries cached by peers or older monitors that did not // validate logs before writing them. @@ -1424,11 +1417,19 @@ func (m *Monitor) unmarshalBlock(blockPayload []byte) (*types.Block, error) { return block, nil } -func (m *Monitor) unmarshalLogs(logsPayload []byte) ([]types.Log, error) { +func (m *Monitor) unmarshalLogs(logsPayload []byte, blockBloom types.Bloom) ([]types.Log, error) { var logs []types.Log err := json.Unmarshal(logsPayload, &logs) if err != nil { return nil, err } + // JSON null decodes without error, but a logs response must be an array. + if logs == nil { + return nil, fmt.Errorf("ethmonitor: filterLogs expected a JSON array of block logs") + } + // Apply the bloom guard to decoded responses, including cache hits. + if len(logs) == 0 && blockBloom != (types.Bloom{}) { + return nil, fmt.Errorf("ethmonitor: filterLogs detected empty block-logs response but block bloom is set, ignoring node response") + } return logs, nil } diff --git a/ethmonitor/prefetch_internal_test.go b/ethmonitor/prefetch_internal_test.go index 0bdb593a..736f3e6b 100644 --- a/ethmonitor/prefetch_internal_test.go +++ b/ethmonitor/prefetch_internal_test.go @@ -40,7 +40,7 @@ func TestPrefetchRejectsInvalidBlockBeforeCaching(t *testing.T) { } func TestMonitorRecoversInvalidPrefetchedLogs(t *testing.T) { - for _, payload := range []string{`{}`, `[{}]`} { + for _, payload := range []string{`{}`, `[{}]`, `null`, `[]`, `[ ]`} { for _, cached := range []bool{false, true} { t.Run(fmt.Sprintf("payload=%s/cached=%v", payload, cached), func(t *testing.T) { const first, target, last = uint64(1000), uint64(1002), uint64(1004) @@ -114,12 +114,12 @@ func TestMonitorRecoversInvalidPrefetchedLogs(t *testing.T) { require.NoError(t, monitor.cache.SetEx(context.Background(), key, []byte(payload), time.Hour)) } else { // Complete the speculative fetch before starting the serial loop, - // so the worker deterministically receives the malformed response. + // so the worker deterministically receives the invalid response. monitor.prefetch.fetch(context.Background(), prefetchJob{num: target}) require.Equal(t, int64(1), originCalls.Load()) _, found, err := monitor.cache.Get(context.Background(), key) require.NoError(t, err) - require.False(t, found, "failed prefetch must not cache malformed logs") + require.False(t, found, "failed prefetch must not cache invalid logs") } sub := monitor.Subscribe("TestMonitorRecoversInvalidPrefetchedLogs") @@ -157,7 +157,7 @@ func TestMonitorRecoversInvalidPrefetchedLogs(t *testing.T) { } } if !cached { - require.GreaterOrEqual(t, originCalls.Load(), int64(2), "malformed prefetch must allow an origin retry") + require.GreaterOrEqual(t, originCalls.Load(), int64(2), "invalid prefetch must allow an origin retry") } else { require.Positive(t, originCalls.Load(), "invalid cache entry must allow an origin retry") } @@ -187,7 +187,14 @@ func TestPrefetchPanicStopsWorkers(t *testing.T) { defer close(done) monitor.prefetch.run(ctx) }() - defer func() { cancel(); <-done }() + defer func() { + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Error("prefetch did not stop after cancellation") + } + }() select { case <-done: case <-time.After(time.Second): diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index cd7854f0..7c411654 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -40,36 +40,27 @@ const ( ) type Options struct { - // .. + // MaxConcurrentFetchReceiptWorkers limits concurrent receipt RPC requests. MaxConcurrentFetchReceiptWorkers int - // .. + // MaxConcurrentFilterWorkers limits concurrent block and filter processing. MaxConcurrentFilterWorkers int - // MaxConcurrentSearchOnChainWorkers is the maximum amount of concurrent - // on-chain searches (this is per subscriber) + // MaxConcurrentSearchOnChainWorkers limits on-chain searches per subscriber. MaxConcurrentSearchOnChainWorkers int - // .. + // PastReceiptsCacheSize limits the number of cached receipts. PastReceiptsCacheSize int - // .. + // NumBlocksToFinality is the number of blocks after mining required for finality. + // Values <= 0 select the network's finality policy when Run starts. NumBlocksToFinality int - // FilterMaxWaitNumBlocks is the maximum amount of blocks a filter will wait between getting - // a receipt filter match, before the filter will unsubscribe itself and stop listening. - // This value may be overriden by setting FilterCond#MaxListenNumBlocks on per-filter basis. - // - // NOTE: - // * value of -1 will use NumBlocksToFinality*2 - // * value of 0 will set no limit, so filter will always listen [default] - // * value of N will set the N number of blocks without results before unsubscribing between iterations + // FilterMaxWaitNumBlocks sets the default block wait between filter matches. + // Zero disables the limit. Individual filters can override it with MaxWait. FilterMaxWaitNumBlocks int - // Cache backend ... - // CacheBackend cachestore.Backend - - // Alerter config via github.com/goware/alerter + // Alerter receives listener and subscriber alerts. Alerter util.Alerter } @@ -82,7 +73,7 @@ type ReceiptsListener struct { chainID *big.Int br *breaker.Breaker - // fetchSem is used to limit amount of concurrenct fetch requests + // fetchSem limits concurrent receipt RPC requests. fetchSem chan struct{} // pastReceipts is a cache of past requested receipts @@ -93,15 +84,17 @@ type ReceiptsListener struct { // for us if they end up turning up. notFoundTxnHashes cachestore.Store[uint64] - // ... subscribers []*subscriber registerFiltersCh chan registerFilters filterSem chan struct{} - ctx context.Context - ctxStop context.CancelFunc - running int32 - mu sync.RWMutex + ctxStop context.CancelFunc + running int32 + lifecycleMu sync.Mutex // protects Run/Stop cancellation handle and running transitions + receiptMu sync.Mutex + blockStates map[common.Hash]blockState + reorgRevision uint64 // removal sequence used to reject hash queries started before invalidation + mu sync.RWMutex } var ( @@ -139,12 +132,12 @@ func NewReceiptsListener(log *slog.Logger, provider ethrpc.Interface, monitor *e return nil, err } - notFoundTxnHashes, err := memcache.NewCacheWithSize[uint64](uint32(5000)) //, cachestore.WithDefaultKeyExpiry(2*time.Minute)) + notFoundTxnHashes, err := memcache.NewCacheWithSize[uint64](uint32(5000)) if err != nil { return nil, err } - // max ~12s total wait time before giving up + // Retry transient RPC failures with exponential backoff. br := breaker.New(log, 200*time.Millisecond, 1.2, 20) return &ReceiptsListener{ @@ -160,6 +153,7 @@ func NewReceiptsListener(log *slog.Logger, provider ethrpc.Interface, monitor *e subscribers: make([]*subscriber, 0), registerFiltersCh: make(chan registerFilters, maxFiltersPerListener), filterSem: make(chan struct{}, opts.MaxConcurrentFilterWorkers), + blockStates: make(map[common.Hash]blockState), }, nil } @@ -186,32 +180,43 @@ func (l *ReceiptsListener) lazyInit(ctx context.Context) error { l.options.NumBlocksToFinality = ethrpc.DefaultNumBlocksToFinality } + for _, sub := range l.subscribers { + sub.finalizer.setFinality(l.options.NumBlocksToFinality) + } return nil } func (l *ReceiptsListener) Run(ctx context.Context) error { + l.lifecycleMu.Lock() if l.IsRunning() { + l.lifecycleMu.Unlock() return fmt.Errorf("ethreceipts: already running") } - - l.ctx, l.ctxStop = context.WithCancel(ctx) - + runCtx, cancel := context.WithCancel(ctx) + l.ctxStop = cancel atomic.StoreInt32(&l.running, 1) - defer atomic.StoreInt32(&l.running, 0) - - if err := l.lazyInit(ctx); err != nil { - slog.Error("ethreceipts: lazyInit failed", slog.String("error", err.Error())) + l.lifecycleMu.Unlock() + defer func() { + cancel() + l.lifecycleMu.Lock() + l.ctxStop = nil + atomic.StoreInt32(&l.running, 0) + l.lifecycleMu.Unlock() + }() + if err := l.lazyInit(runCtx); err != nil { return err } - l.log.Info("ethreceipts: running") - - return l.listener() + return l.listener(runCtx) } - func (l *ReceiptsListener) Stop() { l.log.Info("ethreceipts: stop") - l.ctxStop() + l.lifecycleMu.Lock() + cancel := l.ctxStop + l.lifecycleMu.Unlock() + if cancel != nil { + cancel() + } } func (l *ReceiptsListener) IsRunning() bool { @@ -237,11 +242,11 @@ func (l *ReceiptsListener) Subscribe(filterQueries ...FilterQuery) Subscription finalizer: &finalizer{ numBlocksToFinality: big.NewInt(int64(l.options.NumBlocksToFinality)), queue: []finalTxn{}, - txns: map[common.Hash]struct{}{}, + txns: map[receiptKey]struct{}{}, }, } - subscriber.unsubscribe = func() { + subscriber.unsubscribe = sync.OnceFunc(func() { close(subscriber.done) subscriber.ch.Close() subscriber.ch.Flush() @@ -255,7 +260,7 @@ func (l *ReceiptsListener) Subscribe(filterQueries ...FilterQuery) Subscription return } } - } + }) l.subscribers = append(l.subscribers, subscriber) @@ -302,14 +307,22 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFinality(ctx context.Conte func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context, filter FilterQuery, optFilterFinalize ...bool) (*Receipt, WaitReceiptFinalityFunc, error) { // Fetch method searches for just a single filter match. If you'd like to keep the filter // open to listen to many similar receipts, use .Subscribe(filter) directly instead. - query := filter.LimitOne(true).SearchCache(true) - if len(optFilterFinalize) > 0 && optFilterFinalize[0] { - query = query.Finalize(true) - } - - filterer, ok := query.(Filterer) + source, ok := filter.(Filterer) if !ok { - return nil, nil, fmt.Errorf("ethreceipts: unable to cast Filterer from FilterQuery") + // Builders may expose a Filterer only after their normal option chaining. + filter = filter.LimitOne(true).SearchCache(true) + if len(optFilterFinalize) > 0 && optFilterFinalize[0] { + filter = filter.Finalize(true) + } + source, ok = filter.(Filterer) + if !ok { + return nil, nil, fmt.Errorf("ethreceipts: unable to cast Filterer from FilterQuery") + } + } + filterer := snapshotFilter(source) + filterer.LimitOne(true).SearchCache(true) + if len(optFilterFinalize) > 0 && optFilterFinalize[0] { + filterer.Finalize(true) } condMaxWait := 0 @@ -319,23 +332,23 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context condTxnHash := "" if filterer.Cond().TxnHash != nil { condTxnHash = (*filterer.Cond().TxnHash).String() - query = query.QueryOnChainTxnHash(true) + filterer.QueryOnChainTxnHash(true) } - sub := l.Subscribe(query) - - // Use a WaitGroup to ensure the goroutine cleans up before the function returns - var wg sync.WaitGroup + sub := l.Subscribe(filterer) + workerDone := make(chan struct{}) exhausted := make(chan struct{}) mined := make(chan Receipt, 2) finalized := make(chan Receipt, 1) - found := uint32(0) finalityFunc := func(ctx context.Context) (*Receipt, error) { - // Wait for the goroutine to finish its cleanup before proceeding in finalityFunc, - // ensuring Unsubscribe has been called if the goroutine exited. - wg.Wait() + // Wait for the worker's cleanup, including Unsubscribe, before returning finality. + select { + case <-workerDone: + case <-ctx.Done(): + return nil, ctx.Err() + } select { case <-ctx.Done(): return nil, ctx.Err() @@ -355,12 +368,8 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context } } - // TODO/NOTE: perhaps in an extended node failure. could there be a scenario - // where filterer.Exhausted is never hit? and this subscription never unsubscribes..? - // don't think so, but we can double check. - wg.Add(1) go func() { - defer wg.Done() + defer close(workerDone) defer sub.Unsubscribe() defer close(mined) defer close(finalized) @@ -372,6 +381,8 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context } }() + found := false + filterExhausted := filterer.Exhausted() for { select { case <-ctx.Done(): @@ -381,19 +392,14 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // Subscription closed externally (less likely here, but good practice) return - case <-filterer.Exhausted(): - // Exhausted, check if we ever found a match. - if atomic.LoadUint32(&found) == 0 { - // Never found a match, signal exhaustion and exit. - close(exhausted) + case <-filterExhausted: + // Handle the closed exhaustion channel once. Already mined receipts + // keep waiting for finality until the subscription or context closes. + filterExhausted = nil + close(exhausted) + if !found { return } - // Found a match previously, but now exhausted. - // Allow loop to continue briefly to let finalizer potentially finish, - // but the finalized channel will eventually be closed if no final receipt comes. - // The finalityFunc will handle the exhausted state if needed. - // We signal exhaustion mainly for the initial return value check. - close(exhausted) case receipt, ok := <-sub.TransactionReceipt(): if !ok { @@ -401,7 +407,9 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context return } - atomic.StoreUint32(&found, 1) + found = true + // Helper state stays private; public receipts retain the resolved source filter. + receipt.Filter = source if receipt.Final { // Send to mined (in case caller only waits for mined) @@ -438,19 +446,17 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // Wait for the first mined receipt or an exit signal select { case <-ctx.Done(): - wg.Wait() // Ensure cleanup + <-workerDone return nil, nil, ctx.Err() case <-sub.Done(): - wg.Wait() // Ensure cleanup + <-workerDone return nil, nil, ErrSubscriptionClosed case <-exhausted: - // Exhausted before finding *any* receipt. - // finalityFunc will handle waiting and returning the exhaustion error. return nil, finalityFunc, superr.Wrap(ErrFilterExhausted, fmt.Errorf("txnHash=%s maxWait=%d", condTxnHash, condMaxWait)) case receipt, ok := <-mined: if !ok { // Mined channel closed without sending, implies goroutine exited early. - wg.Wait() // Ensure cleanup + <-workerDone // Check if exhaustion occurred select { case <-exhausted: @@ -459,8 +465,6 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context return nil, nil, ErrSubscriptionClosed } } - // Got the first mined receipt. Return it and the finality func. - // The finalityFunc will use wg.Wait() internally. return &receipt, finalityFunc, nil } } @@ -468,12 +472,17 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // fetchTransactionReceipt from the rpc provider, up to some amount of concurrency. When forceFetch is passed, // it indicates that we have high conviction that the receipt should be available, as the monitor has found // this transaction hash. -func (l *ReceiptsListener) fetchTransactionReceipt(ctx context.Context, txnHash common.Hash, forceFetch bool) (*types.Receipt, error) { +func (l *ReceiptsListener) fetchTransactionReceipt(ctx context.Context, txnHash common.Hash, forceFetch bool, expectedBlock ...blockRef) (*types.Receipt, error) { + l.receiptMu.Lock() + started := l.reorgRevision + l.receiptMu.Unlock() timeStart := time.Now() for { select { case l.fetchSem <- struct{}{}: goto start + case <-ctx.Done(): + return nil, ctx.Err() case <-time.After(1 * time.Minute): elapsed := time.Since(timeStart) l.log.Warn(fmt.Sprintf("fetchTransactionReceipt(%s) waiting for fetch semaphore for %s", txnHash.String(), elapsed)) @@ -496,7 +505,27 @@ start: txnHashHex := txnHash.String() + expected := blockRef{} + if len(expectedBlock) > 0 { + expected = expectedBlock[0] + } + l.receiptMu.Lock() + if expected.hash != (common.Hash{}) && !l.currentBlock(expected.hash, expected.generation) { + l.receiptMu.Unlock() + errCh <- ethereum.NotFound + return + } receipt, ok, _ := l.pastReceipts.Get(ctx, txnHashHex) + if ok && l.blockStates[receipt.BlockHash].removed { + ok = false + l.pastReceipts.Delete(ctx, txnHashHex) + } + if ok && !l.validFetchedBlock(receipt.BlockHash, expected, started) { + l.receiptMu.Unlock() + errCh <- ethereum.NotFound + return + } + l.receiptMu.Unlock() if ok { resultCh <- receipt return @@ -546,9 +575,26 @@ start: return superr.Wrap(fmt.Errorf("failed to fetch receipt %s", txnHash), err) } - l.pastReceipts.Set(ctx, txnHashHex, receipt) - l.notFoundTxnHashes.Delete(ctx, txnHashHex) - + if receipt == nil || receipt.TxHash != txnHash || receipt.BlockNumber == nil { + return fmt.Errorf("ethreceipts: invalid receipt for txn %s", txnHash) + } + l.receiptMu.Lock() + valid := l.validFetchedBlock(receipt.BlockHash, expected, started) + // A stale provider response does not invalidate a current monitored candidate. + retryable := !valid && expected.hash != (common.Hash{}) && l.currentBlock(expected.hash, expected.generation) + if valid { + l.pastReceipts.Set(ctx, txnHashHex, receipt) + l.notFoundTxnHashes.Delete(ctx, txnHashHex) + } + l.receiptMu.Unlock() + if !valid { + if retryable { + errCh <- fmt.Errorf("ethreceipts: stale receipt for txn %s in block %s, expected current block %s", txnHash, receipt.BlockHash, expected.hash) + } else { + errCh <- ethereum.NotFound + } + return nil + } resultCh <- receipt return nil }) @@ -568,15 +614,25 @@ start: } } -func (l *ReceiptsListener) listener() error { +func (l *ReceiptsListener) listener(runCtx context.Context) error { + runCtx, cancel := context.WithCancel(runCtx) + defer cancel() monitor := l.monitor.Subscribe("ethreceipts") defer monitor.Unsubscribe() - latestBlockNum := l.LatestBlockNum().Uint64() + g, ctx := errgroup.WithContext(runCtx) + // Monitor closure must cancel RPC work even while both processing loops wait. + g.Go(func() error { + select { + case <-monitor.Done(): + cancel() + case <-ctx.Done(): + } + return nil + }) + latestBlockNum := l.latestBlockNum(ctx).Uint64() l.log.Debug(fmt.Sprintf("latestBlockNum %d", latestBlockNum)) - g, ctx := errgroup.WithContext(l.ctx) - // Listen on filter registration to search cached and on-chain receipts // At filter subscription registration time, we first search our local cache // of recent blocks retained by the monitor, and then we query on-chain for @@ -591,6 +647,7 @@ func (l *ReceiptsListener) listener() error { return nil case <-monitor.Done(): + cancel() l.log.Info("ethreceipts: receipt listener is stopped because monitor signaled its stopping") return nil @@ -623,7 +680,7 @@ func (l *ReceiptsListener) listener() error { // Search our local blocks cache from monitor retention list, and notify subscriber // of any matches found by publishing receipts. - matchedList, err := l.processBlocks(blocks, []*subscriber{reg.subscriber}, [][]Filterer{filters}) + matchedList, err := l.processCachedBlocks(ctx, blocks, []*subscriber{reg.subscriber}, [][]Filterer{filters}) if err != nil { l.log.Warn(fmt.Sprintf("ethreceipts: failed to process blocks during new filter registration: %v", err)) } @@ -649,6 +706,7 @@ func (l *ReceiptsListener) listener() error { return nil case <-monitor.Done(): + cancel() l.log.Info("ethreceipts: receipt listener is stopped because monitor signaled its stopping") return nil @@ -658,19 +716,15 @@ func (l *ReceiptsListener) listener() error { continue } - latestBlockNum = l.LatestBlockNum().Uint64() + latestBlockNum = l.latestBlockNum(ctx).Uint64() // pass blocks across filters of subscribers l.mu.Lock() - if len(l.subscribers) == 0 { - l.mu.Unlock() - continue - } subscribers := make([]*subscriber, len(l.subscribers)) copy(subscribers, l.subscribers) filters := make([][]Filterer, len(l.subscribers)) for i := 0; i < len(subscribers); i++ { - filters[i] = subscribers[i].Filters() + filters[i] = subscribers[i].filterers() } l.mu.Unlock() @@ -683,13 +737,7 @@ func (l *ReceiptsListener) listener() error { l.notFoundTxnHashes.Delete(ctx, txn.Hash().Hex()) } case ethmonitor.Removed: - // delete past receipts of removed blocks reorg = true - for _, txn := range block.Transactions() { - txnHashHex := txn.Hash().Hex() - l.pastReceipts.Delete(ctx, txnHashHex) - l.notFoundTxnHashes.Delete(ctx, txnHashHex) - } } } @@ -697,7 +745,7 @@ func (l *ReceiptsListener) listener() error { if reorg { for _, list := range filters { for _, filterer := range list { - if f, ok := filterer.(*filter); ok { + if f := builtinFilter(filterer); f != nil { f.setStartBlockNum(latestBlockNum) f.setLastMatchBlockNum(0) } @@ -706,7 +754,7 @@ func (l *ReceiptsListener) listener() error { } // Match blocks against subscribers[i] X filters[i][..] - matchedList, err := l.processBlocks(blocks, subscribers, filters) + matchedList, err := l.processBlocks(ctx, blocks, subscribers, filters) if err != nil { l.log.Warn(fmt.Sprintf("ethreceipts: failed to process blocks: %v", err)) } @@ -716,7 +764,7 @@ func (l *ReceiptsListener) listener() error { for y, matched := range list { filterer := filters[x][y] if matched || filterer.StartBlockNum() == 0 { - if f, ok := filterer.(*filter); ok { + if f := builtinFilter(filterer); f != nil { if f.StartBlockNum() == 0 { f.setStartBlockNum(latestBlockNum) } @@ -725,14 +773,12 @@ func (l *ReceiptsListener) listener() error { } } } else { - // NOTE: even if a filter is exhausted, the finalizer will still run - // for those transactions which were previously mined and marked by the finalizer. - // Therefore, the code below will not impact the functionality of the finalizer. + // Exhaustion stops matching but preserves queued finality. maxWait := l.getMaxWaitBlocks(filterer.Options().MaxWait) blockNum := max(filterer.StartBlockNum(), filterer.LastMatchBlockNum()) if maxWait != 0 && (latestBlockNum-blockNum) >= maxWait { - f, _ := filterer.(*filter) + f := builtinFilter(filterer) if f == nil { panic("ethreceipts: unexpected") } @@ -741,9 +787,9 @@ func (l *ReceiptsListener) listener() error { l.log.Debug(fmt.Sprintf("filter exhausted! last block matched:%d maxWait:%d filterID:%d", filterer.LastMatchBlockNum(), maxWait, filterer.FilterID())) subscriber := subscribers[x] - subscriber.RemoveFilter(filterer) + subscriber.exhaustFilter(filterer) - if f, ok := filterer.(*filter); ok { + if f := builtinFilter(filterer); f != nil { f.closeExhausted() } } @@ -755,40 +801,68 @@ func (l *ReceiptsListener) listener() error { } }) - // TODO/NOTE: perhaps in an extended node failure. could there be a scenario - // where filterer.Exhausted is never hit? and this subscription never unsubscribes..? - // TODO: we ultimately need to check the monitor and if we get no new blocks for a period - // of time, then we can assume node problems.. even more helpful woudl be if the monitor - // gave us an error count of node failures, and we'd listen on that, and if we hit a threshold - // and our block number doesn't change after a period of time, then we return an error - // that we're exhausted due to a node failure. + // MaxWait counts blocks; callers should use a context deadline to bound stalls. return g.Wait() } // processBlocks attempts to match blocks against subscriber[i] X filterers[i].. list of filters. There is // a corresponding list of filters[i] for each subscriber[i]. -func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { +func (l *ReceiptsListener) processBlocks(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { + return l.processBlockEvents(ctx, blocks, subscribers, filterers, true) +} + +// Cached snapshots may match receipts, but cannot authorize canonical re-adoption. +func (l *ReceiptsListener) processCachedBlocks(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { + return l.processBlockEvents(ctx, blocks, subscribers, filterers, false) +} +func (l *ReceiptsListener) processBlockEvents(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer, canonicalEvents bool) ([][]bool, error) { // oks is the 'ok' match of the filterers [][]Filterer results oks := make([][]bool, len(filterers)) for i, f := range filterers { oks[i] = make([]bool, len(f)) } + // Invalidate every rollback before any retry, addition or finality advancement. + removed := make(map[common.Hash]blockRef) + for _, block := range blocks { + if block.Event == ethmonitor.Removed { + ref := l.invalidateBlock(ctx, block) + removed[block.Hash()] = ref + for _, sub := range subscribers { + sub.rollbackBlock(ref) + } + } + } if len(subscribers) == 0 || len(filterers) == 0 { + if canonicalEvents { + for _, block := range blocks { + if block.Event == ethmonitor.Added { + l.acceptBlock(block) + } + } + } return oks, nil } - // check each block against each subscriber X filter for _, block := range blocks { + if ctx.Err() != nil { + return oks, ctx.Err() + } // report if the txn was removed reorged := block.Event == ethmonitor.Removed + if canonicalEvents && !reorged { + l.acceptBlock(block) + } + generation := l.blockGeneration(block.Hash()) + if reorged { + generation = removed[block.Hash()].generation + } - // TODOXXX: feels wasteful to build all receipts for all subscribers every time, but its okay for now receipts := make([]Receipt, len(block.Transactions())) logs := groupLogsByTransaction(block.Logs) - // build unfiltered complete receipts for each txn which include the transaction and the logs + // Build transaction candidates with the logs from the monitored block. for i, txn := range block.Transactions() { txnLog, ok := logs[txn.Hash().Hex()] if !ok { @@ -800,6 +874,9 @@ func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers [ logs: txnLog, chainID: l.chainID, transaction: txn, + blockNum: block.Number(), + blockHash: block.Hash(), + generation: generation, } } @@ -807,8 +884,12 @@ func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers [ // and if there is a match notify the subscriber with the receipts. var wg sync.WaitGroup for i, sub := range subscribers { - l.filterSem <- struct{}{} - + select { + case l.filterSem <- struct{}{}: + case <-ctx.Done(): + wg.Wait() + return oks, ctx.Err() + } wg.Add(1) go func(i int, sub *subscriber) { defer func() { @@ -817,21 +898,24 @@ func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers [ }() // retry pending receipts first - retryCtx, cancel := context.WithTimeout(l.ctx, 5*time.Second) - sub.retryPendingReceipts(retryCtx) // TODOXXXPETER: what is this pending receipts thing..? hmpf.. - cancel() + if !reorged { + retryCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + sub.retryPendingReceipts(retryCtx) + cancel() + } // filter matcher and notify subscriber of receipts if matched - matched, err := sub.matchFiltersAndPublish(l.ctx, filterers[i], receipts) + matched, err := sub.matchFiltersAndPublish(ctx, filterers[i], receipts) if err != nil { l.log.Warn(fmt.Sprintf("error while processing filters: %s", err)) } oks[i] = matched // check subscriber to finalize any receipts - err = sub.finalizeReceipts(block.Number()) - if err != nil { - l.log.Error(fmt.Sprintf("finalizeReceipts failed: %v", err)) + if !reorged && ctx.Err() == nil { + if err = sub.finalizeReceipts(block.Number()); err != nil { + l.log.Error(fmt.Sprintf("finalizeReceipts failed: %v", err)) + } } }(i, sub) } @@ -885,6 +969,9 @@ func (l *ReceiptsListener) queryFilterOnChain(ctx context.Context, subscriber *s return gctx.Err() } + l.receiptMu.Lock() + started := l.reorgRevision + l.receiptMu.Unlock() var r *types.Receipt var err error @@ -909,15 +996,24 @@ func (l *ReceiptsListener) queryFilterOnChain(ctx context.Context, subscriber *s return nil } + l.receiptMu.Lock() + valid := l.validFetchedBlock(r.BlockHash, blockRef{}, started) + generation := l.blockStates[r.BlockHash].generation + l.receiptMu.Unlock() + if !valid { + return nil + } // Found the receipt, update last match block num and continue - if f, ok := item.filterer.(*filter); ok { - f.setLastMatchBlockNum(r.BlockNumber.Uint64()) + if f := builtinFilter(item.filterer); f != nil { + if r.BlockNumber != nil { + f.setLastMatchBlockNum(r.BlockNumber.Uint64()) + } } receipt := Receipt{ - receipt: r, + receipt: r, + generation: generation, // NOTE: we do not include the transaction at this point, as we don't have it. - // transaction: txn, Final: l.isBlockFinal(r.BlockNumber), } @@ -954,7 +1050,7 @@ func (l *ReceiptsListener) getMaxWaitBlocks(maxWait *int) uint64 { } func (l *ReceiptsListener) isBlockFinal(blockNum *big.Int) bool { - latestBlockNum := l.LatestBlockNum() + latestBlockNum := l.monitor.LatestBlockNum() if latestBlockNum == nil || blockNum == nil { return false } @@ -963,10 +1059,12 @@ func (l *ReceiptsListener) isBlockFinal(blockNum *big.Int) bool { l.mu.RLock() defer l.mu.RUnlock() - return diff.Cmp(big.NewInt(int64(l.options.NumBlocksToFinality))) >= 0 + return l.options.NumBlocksToFinality > 0 && diff.Cmp(big.NewInt(int64(l.options.NumBlocksToFinality))) >= 0 } -func (l *ReceiptsListener) LatestBlockNum() *big.Int { +func (l *ReceiptsListener) LatestBlockNum() *big.Int { return l.latestBlockNum(context.Background()) } + +func (l *ReceiptsListener) latestBlockNum(ctx context.Context) *big.Int { // return immediately if the monitor has a latest block number latestBlockNum := l.monitor.LatestBlockNum() if latestBlockNum != nil && latestBlockNum.Cmp(big.NewInt(0)) > 0 { @@ -977,7 +1075,11 @@ func (l *ReceiptsListener) LatestBlockNum() *big.Int { maxWaitTime := 30 * time.Second period := 250 * time.Millisecond for { - time.Sleep(period) + select { + case <-time.After(period): + case <-ctx.Done(): + return big.NewInt(0) + } latestBlockNum := l.monitor.LatestBlockNum() if latestBlockNum != nil && latestBlockNum.Cmp(big.NewInt(0)) > 0 { @@ -994,24 +1096,23 @@ func (l *ReceiptsListener) LatestBlockNum() *big.Int { func getChainID(ctx context.Context, provider ethrpc.Interface) (*big.Int, error) { var chainID *big.Int - - // provide plenty of time for breaker to succeed err := breaker.Do(ctx, func() error { - ctx, cancel := context.WithTimeout(ctx, 4*time.Second) + requestCtx, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() - id, err := provider.ChainID(ctx) + id, err := provider.ChainID(requestCtx) if err != nil { return err } chainID = id return nil - }, nil, 1*time.Second, 2, 10) - + }, nil, time.Second, 2, 10) if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } return nil, err } - return chainID, nil } @@ -1025,20 +1126,6 @@ func collectOk[T any](in []T, oks []bool, okCond bool) []T { return out } -// func txnLogs(blockLogs []types.Log, txnHash ethkit.Hash) []*types.Log { -// txnLogs := []*types.Log{} -// for i, log := range blockLogs { -// if log.TxHash == txnHash { -// log := log // copy -// txnLogs = append(txnLogs, &log) -// if i+1 >= len(blockLogs) || blockLogs[i+1].TxHash != txnHash { -// break -// } -// } -// } -// return txnLogs -// } - func groupLogsByTransaction(logs []types.Log) map[string][]*types.Log { var out = make(map[string][]*types.Log) for _, log := range logs { @@ -1056,12 +1143,80 @@ func groupLogsByTransaction(logs []types.Log) map[string][]*types.Log { return out } -// func blockLogsCount(numTxns int, logs []types.Log) uint { -// var max uint = uint(numTxns) -// for _, log := range logs { -// if log.TxIndex+1 > max { -// max = log.TxIndex + 1 -// } -// } -// return max -// } +// A removed hash can later be canonical again. Generations invalidate work +// started before rollback even when its transaction and block hash are unchanged. +type blockRef struct { + hash common.Hash + generation uint64 +} +type blockState struct { + generation uint64 + removed bool + invalidatedAt uint64 +} + +func (l *ReceiptsListener) currentBlock(hash common.Hash, generation uint64) bool { + state := l.blockStates[hash] + return !state.removed && state.generation == generation +} +func (l *ReceiptsListener) isCurrentBlock(hash common.Hash, generation uint64) bool { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + return l.currentBlock(hash, generation) +} +func (l *ReceiptsListener) blockGeneration(hash common.Hash) uint64 { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + return l.blockStates[hash].generation +} + +// Caller holds receiptMu. Hash queries do not know their block before fetching, +// so reject results from blocks invalidated since the RPC started as well. +func (l *ReceiptsListener) validFetchedBlock(hash common.Hash, expected blockRef, started uint64) bool { + state := l.blockStates[hash] + if state.removed || state.invalidatedAt > started { + return false + } + return expected.hash == (common.Hash{}) || (hash == expected.hash && state.generation == expected.generation) +} +func (l *ReceiptsListener) acceptBlock(block *ethmonitor.Block) { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + hash := block.Hash() + state := l.blockStates[hash] + if state.removed { + incarnation, canonical := block.CanonicalState() + if incarnation == 0 { + // Legacy/manual blocks have no incarnation evidence; only a + // positive retained lookup can authorize their re-adoption. + retained := l.monitor.GetBlock(hash) + canonical = retained != nil && retained.Event == ethmonitor.Added + } + if canonical { + state.removed = false + l.blockStates[hash] = state + } + } +} +func (l *ReceiptsListener) invalidateBlock(ctx context.Context, block *ethmonitor.Block) blockRef { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + state := l.blockStates[block.Hash()] + if !state.removed { + l.reorgRevision++ + state.generation++ + state.invalidatedAt = l.reorgRevision + state.removed = true + l.blockStates[block.Hash()] = state + } + removed := blockRef{block.Hash(), state.generation - 1} + for _, txn := range block.Transactions() { + hash := txn.Hash().Hex() + cached, found, _ := l.pastReceipts.Get(ctx, hash) + if found && cached.BlockHash == block.Hash() { + l.pastReceipts.Delete(ctx, hash) + } + l.notFoundTxnHashes.Delete(ctx, hash) + } + return removed +} diff --git a/ethreceipts/ethreceipts_test.go b/ethreceipts/ethreceipts_test.go index 9c508a4e..d2bfc3ae 100644 --- a/ethreceipts/ethreceipts_test.go +++ b/ethreceipts/ethreceipts_test.go @@ -20,10 +20,12 @@ import ( "github.com/0xsequence/ethkit/ethrpc" "github.com/0xsequence/ethkit/ethtest" "github.com/0xsequence/ethkit/ethtxn" + "github.com/0xsequence/ethkit/ethwallet" "github.com/0xsequence/ethkit/go-ethereum/common" "github.com/0xsequence/ethkit/go-ethereum/core/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) var ( @@ -72,6 +74,109 @@ func newProvider(t *testing.T) *ethrpc.Provider { return provider } +// Service errors cancel receipt waits and are checked after the worker exits. +func runReceiptTestService(t *testing.T, ctx context.Context, cancel context.CancelFunc, name string, run func(context.Context) error) { + t.Helper() + done := make(chan error, 1) + go func() { + err := run(ctx) + if ctx.Err() != nil && (err == nil || errors.Is(err, ctx.Err())) { + done <- nil + return + } + if err == nil { + err = fmt.Errorf("%s stopped unexpectedly", name) + } else { + err = fmt.Errorf("%s: %w", name, err) + } + done <- err + cancel() + }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Errorf("%s did not stop after cancellation", name) + } + }) +} + +func restoreReceiptTestProvider(t *testing.T, ctx context.Context, cancel context.CancelFunc, provider *ethrpc.Provider) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + select { + case <-ctx.Done(): + case <-time.After(20 * time.Second): + provider.SetHTTPClient(newFlakyHTTPClient(0.0)) + } + }() + t.Cleanup(func() { + cancel() + <-done + }) +} + +func waitForFlakyProviderReceipts(t *testing.T, ctx context.Context, provider *ethrpc.Provider, listener *ethreceipts.ReceiptsListener, sender *ethwallet.Wallet, recipient common.Address) { + t.Helper() + nonce, err := sender.GetNonce(ctx) + require.NoError(t, err) + + txns := make([]*types.Transaction, 50) + for i := range txns { + txns[i], err = sender.NewTransaction(ctx, ðtxn.TransactionRequest{ + To: &recipient, + ETHValue: ethtest.ETHValue(0.01), + GasLimit: 120_000, + Nonce: big.NewInt(int64(nonce + uint64(i))), + }) + require.NoError(t, err) + } + + g, workerCtx := errgroup.WithContext(ctx) + for _, txn := range txns { + ready := make(chan struct{}) + g.Go(func() error { + sub := listener.Subscribe(ethreceipts.FilterTxnHash(txn.Hash())) + defer sub.Unsubscribe() + close(ready) + + select { + case <-workerCtx.Done(): + return workerCtx.Err() + case <-sub.Done(): + return fmt.Errorf("subscription closed before receipt for %s", txn.Hash()) + case receipt, ok := <-sub.TransactionReceipt(): + if !ok { + return fmt.Errorf("receipt channel closed for %s", txn.Hash()) + } + if receipt.TransactionHash() != txn.Hash() || receipt.Status() != types.ReceiptStatusSuccessful || receipt.Reorged { + return fmt.Errorf("unexpected receipt for %s: hash=%s status=%d reorged=%t", txn.Hash(), receipt.TransactionHash(), receipt.Status(), receipt.Reorged) + } + return nil + case <-time.After(300 * time.Second): + return fmt.Errorf("timeout waiting for receipt for %s", txn.Hash()) + } + }) + g.Go(func() error { + select { + case <-workerCtx.Done(): + return workerCtx.Err() + case <-ready: + } + _, _, err := ethtxn.SendTransaction(workerCtx, provider, txn) + if err != nil { + return fmt.Errorf("send transaction %s: %w", txn.Hash(), err) + } + return nil + }) + } + require.NoError(t, g.Wait()) +} + func newFlakyHTTPClient(failureRate float32) *http.Client { return &http.Client{ Timeout: 10 * time.Second, @@ -118,12 +223,7 @@ func TestFetchTransactionReceiptBasic(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) require.Zero(t, monitor.NumSubscribers()) @@ -134,12 +234,7 @@ func TestFetchTransactionReceiptBasic(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) assert.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // // Setup test wallet @@ -174,54 +269,29 @@ func TestFetchTransactionReceiptBasic(t *testing.T) { txnHashes = append(txnHashes, txn.Hash()) } - // dispatch txns in the background - go func() { + workers, workerCtx := errgroup.WithContext(ctx) + workers.Go(func() error { for _, txn := range txns { - _, _, err = wallet.SendTransaction(ctx, txn) - require.NoError(t, err) - // time.Sleep(500 * time.Millisecond) + if _, _, err := wallet.SendTransaction(workerCtx, txn); err != nil { + return fmt.Errorf("send transaction %s: %w", txn.Hash(), err) + } } - }() - - // ensure all txns made it - // delay processing if we want to make sure SearchCache works - // time.Sleep(2 * time.Second) - // for _, txnHash := range txnHashes { - // receipt, err := provider.TransactionReceipt(context.Background(), txnHash) - // require.NoError(t, err) - // require.True(t, receipt.Status == 1) - // } - - // Let's listen for all the txns - var wg sync.WaitGroup - for i, txnHash := range txnHashes { - wg.Add(1) - go func(i int, txnHash common.Hash) { - defer wg.Done() + return nil + }) - receipt, err := receiptsListener.FetchTransactionReceipt(ctx, txnHash, 7) - require.NoError(t, err) - require.NotNil(t, receipt) - require.True(t, receipt.Status() == types.ReceiptStatusSuccessful) - require.False(t, receipt.Final) - t.Logf("=> MINED %d :: %s", i, receipt.TransactionHash().String()) - - // receipt, waitFinality, err := receiptsListener.FetchTransactionReceiptWithFinality(ctx, txnHash, 7) - // require.NoError(t, err) - // require.NotNil(t, receipt) - // require.True(t, receipt.Status() == types.ReceiptStatusSuccessful) - // require.False(t, receipt.Final) - // t.Logf("=> MINED %d :: %s", i, receipt.TransactionHash().String()) - // - // finalReceipt, err := waitFinality(context.Background()) - // require.NoError(t, err) - // require.NotNil(t, finalReceipt) - // require.True(t, finalReceipt.Status() == types.ReceiptStatusSuccessful) - // require.True(t, finalReceipt.Final) - // t.Logf("=> FINAL %d :: %s", i, receipt.TransactionHash().String()) - }(i, txnHash) + for _, txnHash := range txnHashes { + workers.Go(func() error { + receipt, err := receiptsListener.FetchTransactionReceipt(workerCtx, txnHash, 7) + if err != nil { + return fmt.Errorf("fetch receipt %s: %w", txnHash, err) + } + if receipt == nil || receipt.TransactionHash() != txnHash || receipt.Status() != types.ReceiptStatusSuccessful || receipt.Final { + return fmt.Errorf("unexpected mined receipt for %s: %v", txnHash, receipt) + } + return nil + }) } - wg.Wait() + require.NoError(t, workers.Wait()) time.Sleep(2 * time.Second) @@ -292,12 +362,7 @@ func TestFetchTransactionReceiptBlast(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 @@ -306,12 +371,7 @@ func TestFetchTransactionReceiptBlast(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) assert.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // // Setup wallets @@ -345,176 +405,193 @@ func TestFetchTransactionReceiptBlast(t *testing.T) { txns[5].Hash(), txns[2].Hash(), txns[8].Hash(), txns[3].Hash(), } - var count uint64 - - var wg sync.WaitGroup - for i, txnHash := range txnHashes { - wg.Add(1) - go func(i int, txnHash common.Hash) { - defer wg.Done() - - receipt, receiptFinality, err := receiptsListener.FetchTransactionReceiptWithFinality(ctx, txnHash) - assert.NoError(t, err) - assert.NotNil(t, receipt) - assert.True(t, receipt.Status() == types.ReceiptStatusSuccessful) - - finalReceipt, err := receiptFinality(context.Background()) - require.NoError(t, err) - require.True(t, finalReceipt.Status() == types.ReceiptStatusSuccessful) - - t.Logf("=> %d :: %s", i, receipt.TransactionHash().String()) - - atomic.AddUint64(&count, 1) - }(i, txnHash) + workers, workerCtx := errgroup.WithContext(ctx) + for _, txnHash := range txnHashes { + workers.Go(func() error { + receipt, waitFinality, err := receiptsListener.FetchTransactionReceiptWithFinality(workerCtx, txnHash) + if err != nil { + return fmt.Errorf("fetch receipt %s: %w", txnHash, err) + } + if receipt == nil || receipt.TransactionHash() != txnHash || receipt.Status() != types.ReceiptStatusSuccessful { + return fmt.Errorf("unexpected receipt for %s: %v", txnHash, receipt) + } + finalReceipt, err := waitFinality(workerCtx) + if err != nil { + return fmt.Errorf("final receipt %s: %w", txnHash, err) + } + if finalReceipt == nil || finalReceipt.TransactionHash() != txnHash || finalReceipt.Status() != types.ReceiptStatusSuccessful || !finalReceipt.Final { + return fmt.Errorf("unexpected final receipt for %s: %v", txnHash, finalReceipt) + } + return nil + }) } - wg.Wait() - - require.Equal(t, int(count), len(txnHashes)) + require.NoError(t, workers.Wait()) } func TestReceiptsListenerFilters(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() - // - // Setup ReceiptsListener - // provider := testchain.Provider - monitorOptions := ethmonitor.DefaultOptions - // monitorOptions.Logger = log monitorOptions.WithLogs = true monitorOptions.BlockRetentionLimit = 1000 monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) - assert.NoError(t, err) - - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + require.NoError(t, err) + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 listenerOptions.FilterMaxWaitNumBlocks = 4 - receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) - assert.NoError(t, err) - - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() - - // - // Setup wallets - // - - // create and fund a few wallets to send from - fromWallets, _ := testchain.DummyWallets(3, 100) - testchain.FundAddresses(ethtest.WalletAddresses(fromWallets), 10) - - // create a few wallets to send to - toWallets, _ := testchain.DummyWallets(3, 200) - - // prepare and sign bunch of txns - values := []*big.Int{} - for range fromWallets { - values = append(values, ethtest.ETHValue(0.1)) - } + require.NoError(t, err) + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) + fromWallets, err := testchain.DummyWallets(3, 100) + require.NoError(t, err) + require.NoError(t, testchain.FundAddresses(ethtest.WalletAddresses(fromWallets), 10)) + toWallets, err := testchain.DummyWallets(3, 200) + require.NoError(t, err) + values := []*big.Int{ethtest.ETHValue(0.1), ethtest.ETHValue(0.1), ethtest.ETHValue(0.1)} _, txns, err := ethtest.PrepareBlastSendTransactions(ctx, fromWallets, ethtest.WalletAddresses(toWallets), values) - assert.NoError(t, err) + require.NoError(t, err) - // send the txns -- these will be async, so we can just blast synchronously - // and not have to do it in a goroutine + fromFilter := ethreceipts.FilterFrom(fromWallets[1].Address()).LimitOne(true).ID(1111).MaxWait(0) + toFilter := ethreceipts.FilterTo(toWallets[1].Address()).ID(1112).MaxWait(0) + removedFilter := ethreceipts.FilterTxnHash(txns[2].Hash()).ID(2222) + addedFilter := ethreceipts.FilterTxnHash(txns[4].Hash()).ID(4444) + sub := receiptsListener.Subscribe(fromFilter, toFilter, removedFilter) + defer sub.Unsubscribe() + + sub2 := receiptsListener.Subscribe() + secondFilter := ethreceipts.FilterTxnHash(txns[3].Hash()) + sub2.AddFilter(secondFilter) + defer sub2.Unsubscribe() + + thirdFilter := ethreceipts.FilterTxnHash(txns[2].Hash()).ID(3333) + expiringFilter := ethreceipts.FilterFrom(ethkit.Address{4, 2, 4, 2}).ID(8888).MaxWait(4) + sub3 := receiptsListener.Subscribe(thirdFilter, expiringFilter) + defer sub3.Unsubscribe() + + // All live filters are installed before any transaction can be mined. + require.Eventually(t, func() bool { + return receiptsListener.IsRunning() && monitor.NumSubscribers() == 1 + }, 5*time.Second, 10*time.Millisecond) for _, txn := range txns { _, _, err := ethtxn.SendTransaction(ctx, provider, txn) - assert.NoError(t, err) + require.NoError(t, err) } - // - // Subscribe to a filter on the receipt listener - // - fmt.Println("listening for txns..") - - sub := receiptsListener.Subscribe( - ethreceipts.FilterFrom(fromWallets[1].Address()).LimitOne(true), - ethreceipts.FilterTo(toWallets[1].Address()), - ethreceipts.FilterTxnHash(txns[2].Hash()).ID(2222), //.Finalize(true) is set by default for FilterTxnHash - ) - - sub2 := receiptsListener.Subscribe() - sub2.AddFilter(ethreceipts.FilterTxnHash(txns[3].Hash())) - - sub3 := receiptsListener.Subscribe( - ethreceipts.FilterTxnHash(txns[2].Hash()).ID(3333), - - // will end up not being found and timeout after MaxWait - ethreceipts.FilterFrom(ethkit.Address{4, 2, 4, 2}).MaxWait(4), - ) - - go func() { - time.Sleep(5 * time.Second) - fmt.Println("==> delaying to find", txns[4].Hash().String()) - sub.AddFilter(ethreceipts.FilterTxnHash(txns[4].Hash()).ID(4444)) - }() - - go func() { - for r := range sub2.TransactionReceipt() { - fmt.Println("sub2, got receipt", r.TransactionHash(), "final?", r.Final) + type delivery struct { + subscription string + hash common.Hash + filterID uint64 + final bool + } + expected := make(map[delivery]ethreceipts.FilterQuery) + expect := func(subscription string, txnIndex int, filter ethreceipts.FilterQuery, states ...bool) { + for _, final := range states { + key := delivery{subscription, txns[txnIndex].Hash(), filter.(ethreceipts.Filterer).FilterID(), final} + expected[key] = filter } - }() - - go func() { - for r := range sub3.TransactionReceipt() { - fmt.Println("sub3, got receipt", r.TransactionHash(), "final?", r.Final, "id?", r.FilterID()) //, "maxWait hit?", r.Filter.IsExpired()) + } + // LimitOne selects the first matching nonce; the recipient filter selects all three senders. + expect("sub", 3, fromFilter, false) + for _, i := range []int{1, 4, 7} { + expect("sub", i, toFilter, false) + } + expect("sub", 2, removedFilter, false) + expect("sub", 4, addedFilter, false, true) + expect("sub2", 3, secondFilter, false, true) + expect("sub3", 2, thirdFilter, false, true) + + seen := make(map[delivery]bool) + added := false + removed := false + record := func(subscription string, receipt ethreceipts.Receipt) { + key := delivery{subscription, receipt.TransactionHash(), receipt.FilterID(), receipt.Final} + filter, ok := expected[key] + require.True(t, ok, "unexpected delivery: %+v", key) + require.False(t, seen[key], "duplicate delivery: %+v", key) + require.Same(t, filter, receipt.Filter, "wrong filter owner: %+v", key) + require.Equal(t, uint64(types.ReceiptStatusSuccessful), receipt.Status()) + require.False(t, receipt.Reorged) + require.NotZero(t, receipt.BlockHash()) + seen[key] = true + + if subscription == "sub" && receipt.FilterID() == 2222 { + // Remove only this owner after its mined event, before it can finalize. + sub.RemoveFilter(receipt.Filter) + require.NotContains(t, sub.Filters(), receipt.Filter) + removed = true } - }() + if subscription == "sub" && receipt.FilterID() == 1112 && receipt.TransactionHash() == txns[4].Hash() { + // This transaction is now in the monitor cache; registration must find it there. + sub.AddFilter(addedFilter) + require.Contains(t, sub.Filters(), addedFilter) + added = true + } + } -loop: - for { + exhausted := expiringFilter.(ethreceipts.Filterer).Exhausted() + expired := false + for len(seen) != len(expected) || !expired { select { - case <-ctx.Done(): - fmt.Println("ctx done") - break loop - + missing := make([]delivery, 0) + for key := range expected { + if !seen[key] { + missing = append(missing, key) + } + } + t.Fatalf("missing receipt deliveries: %v; expired=%t; context=%v", missing, expired, ctx.Err()) case <-sub.Done(): - fmt.Println("sub done") - break loop - + t.Fatal("subscription closed before expected deliveries") + case <-sub2.Done(): + t.Fatal("second subscription closed before expected deliveries") + case <-sub3.Done(): + t.Fatal("third subscription closed before expected deliveries") case receipt, ok := <-sub.TransactionReceipt(): - if !ok { - continue - } - - fmt.Println("=> sub, got receipt", receipt.TransactionHash(), "final?", receipt.Final, "id?", receipt.FilterID(), "status?", receipt.Status()) - - // txn := receipt.Transaction - // txnMsg := receipt.Message - - fmt.Println("=> filter matched!", receipt.From(), receipt.TransactionHash()) - fmt.Println("=> receipt status?", receipt.Status()) - - fmt.Println("==> len filters", len(sub.Filters())) - if receipt.TransactionHash() == txns[2].Hash() { - sub.RemoveFilter(receipt.Filter) - } - fmt.Println("==> len filters", len(sub.Filters())) - - fmt.Println("") - - // expecting to be finished with listening for events after a few seconds - case <-time.After(15 * time.Second): - sub.Unsubscribe() - + require.True(t, ok, "receipt channel closed before expected deliveries") + record("sub", receipt) + case receipt, ok := <-sub2.TransactionReceipt(): + require.True(t, ok, "second receipt channel closed before expected deliveries") + record("sub2", receipt) + case receipt, ok := <-sub3.TransactionReceipt(): + require.True(t, ok, "third receipt channel closed before expected deliveries") + record("sub3", receipt) + case <-exhausted: + expired = true + exhausted = nil } } + + require.True(t, added, "cached transaction filter was not added") + require.True(t, removed, "mined transaction filter was not removed") + require.Eventually(t, func() bool { + return len(sub.Filters()) == 1 && len(sub2.Filters()) == 0 && len(sub3.Filters()) == 0 + }, 5*time.Second, 10*time.Millisecond) + require.Same(t, toFilter, sub.Filters()[0]) + + // The unbounded subscription queue forwards asynchronously; observe it long enough + // to reject queued duplicates or finalization of the removed owner. + quiet := time.NewTimer(2 * time.Second) + defer quiet.Stop() + select { + case <-ctx.Done(): + t.Fatalf("context ended while checking for extra deliveries: %v", ctx.Err()) + case receipt, ok := <-sub.TransactionReceipt(): + require.True(t, ok, "receipt channel closed unexpectedly") + record("sub", receipt) + case receipt, ok := <-sub2.TransactionReceipt(): + require.True(t, ok, "second receipt channel closed unexpectedly") + record("sub2", receipt) + case receipt, ok := <-sub3.TransactionReceipt(): + require.True(t, ok, "third receipt channel closed unexpectedly") + record("sub3", receipt) + case <-quiet.C: + } } func TestReceiptsListenerERC20(t *testing.T) { @@ -544,12 +621,7 @@ func TestReceiptsListenerERC20(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 @@ -558,12 +630,7 @@ func TestReceiptsListenerERC20(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) assert.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // // Subscribe to a filter on the receipt listener @@ -602,26 +669,20 @@ func TestReceiptsListenerERC20(t *testing.T) { num := int64(2000) erc20Receipts := make([]*types.Receipt, 0) - var erc20ReceiptsMu sync.Mutex receipt := erc20Mock.Mint(t, wallet, num) erc20Receipts = append(erc20Receipts, receipt) erc20Mock.GetBalance(t, wallet.Address(), num) - go func() { - total := int64(0) - for i := 0; i < 5; i++ { - n := int64(40 + i) - total += n - - erc20ReceiptsMu.Lock() - receipt := erc20Mock.Transfer(t, wallet, wallet2.Address(), n) - erc20Receipts = append(erc20Receipts, receipt) - erc20ReceiptsMu.Unlock() + total := int64(0) + for i := 0; i < 5; i++ { + n := int64(40 + i) + total += n - erc20Mock.GetBalance(t, wallet2.Address(), total) - } - }() + receipt := erc20Mock.Transfer(t, wallet, wallet2.Address(), n) + erc20Receipts = append(erc20Receipts, receipt) + erc20Mock.GetBalance(t, wallet2.Address(), total) + } // // Listener loop @@ -667,20 +728,29 @@ loop: } } - // NOTE: expecting receipts twice. Once on mine, once on finalize. - for _, mr := range matchedReceipts { - found := false - for _, r := range erc20Receipts { - if mr.TransactionHash() == r.TxHash { - found = true - } - } - assert.True(t, found, "looking for matched receipt %s", mr.TransactionHash().String()) + // Each submitted transaction must have one mined and one final event, even + // when MaxWait expires before the queued finality threshold is reached. + expectedHashes := make(map[common.Hash]bool, len(erc20Receipts)) + for _, receipt := range erc20Receipts { + expectedHashes[receipt.TxHash] = true } - - t.Logf("matchedCount: %d", matchedCount) - t.Logf("erc20Receipts: %d", len(erc20Receipts)) - require.Equal(t, matchedCount, len(erc20Receipts)*2) + type receiptState struct { + hash common.Hash + final bool + } + counts := make(map[receiptState]int) + for _, receipt := range matchedReceipts { + require.True(t, expectedHashes[receipt.TransactionHash()], "unexpected transaction %s", receipt.TransactionHash()) + require.Equal(t, uint64(9999), receipt.FilterID()) + require.Equal(t, uint64(types.ReceiptStatusSuccessful), receipt.Status()) + require.False(t, receipt.Reorged) + counts[receiptState{receipt.TransactionHash(), receipt.Final}]++ + } + for hash := range expectedHashes { + require.Equal(t, 1, counts[receiptState{hash, false}], "mined delivery for %s", hash) + require.Equal(t, 1, counts[receiptState{hash, true}], "final delivery for %s", hash) + } + require.Equal(t, len(erc20Receipts)*2, matchedCount) } func TestFiltersAddDeadlock(t *testing.T) { @@ -696,12 +766,7 @@ func TestFiltersAddDeadlock(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 @@ -780,10 +845,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(goodProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -792,81 +854,18 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, goodProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) t.Run("Wait for txn receipts with a flaky monitor and a healthy provider", func(t *testing.T) { @@ -885,10 +884,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(flakyProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -897,10 +893,7 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, goodProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) @@ -909,81 +902,15 @@ func TestFlakyProvider(t *testing.T) { t.Logf("Setting provider to flaky state") flakyProvider.SetHTTPClient(newFlakyHTTPClient(1.0)) - go func() { - // After 20 seconds, restore the provider to more reliable state - time.Sleep(20 * time.Second) - t.Logf("Restoring provider to healthy state") - flakyProvider.SetHTTPClient(newFlakyHTTPClient(0.0)) - }() + restoreReceiptTestProvider(t, ctx, cancel, flakyProvider) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - // Add a bunch of subscribers that will each send a txn and wait for it - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) t.Run("Wait for txn receipts with a healthy monitor and a flaky provider", func(t *testing.T) { @@ -1002,10 +929,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(goodProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -1014,10 +938,7 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, flakyProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) @@ -1026,81 +947,15 @@ func TestFlakyProvider(t *testing.T) { t.Logf("Setting provider to flaky state") flakyProvider.SetHTTPClient(newFlakyHTTPClient(1.0)) - go func() { - // After 20 seconds, restore the provider to more reliable state - time.Sleep(20 * time.Second) - t.Logf("Restoring provider to healthy state") - flakyProvider.SetHTTPClient(newFlakyHTTPClient(0.0)) - }() + restoreReceiptTestProvider(t, ctx, cancel, flakyProvider) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - // Add a bunch of subscribers that will each send a txn and wait for it - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) t.Run("Wait for txn receipts with a flaky monitor and a flaky provider", func(t *testing.T) { @@ -1119,10 +974,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(flakyProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -1131,10 +983,7 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, flakyProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) @@ -1143,80 +992,14 @@ func TestFlakyProvider(t *testing.T) { t.Logf("Setting provider to flaky state") flakyProvider.SetHTTPClient(newFlakyHTTPClient(1.0)) - go func() { - // After 20 seconds, restore the provider to more reliable state - time.Sleep(20 * time.Second) - t.Logf("Restoring provider to healthy state") - flakyProvider.SetHTTPClient(newFlakyHTTPClient(0.0)) - }() + restoreReceiptTestProvider(t, ctx, cancel, flakyProvider) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - // Add a bunch of subscribers that will each send a txn and wait for it - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) } diff --git a/ethreceipts/filterer.go b/ethreceipts/filterer.go index fd48c20d..28046694 100644 --- a/ethreceipts/filterer.go +++ b/ethreceipts/filterer.go @@ -120,22 +120,21 @@ type FilterQuery interface { QueryOnChain(func(context.Context) (*types.Receipt, error)) FilterQuery MaxWait(int) FilterQuery - // DEPRECATED: please use QueryOnChainTxnHash instead, which is the same thing, renamed to be more clear - // in addition, see new QueryChain(fn) as additional feature. + // Deprecated: use QueryOnChainTxnHash instead. SearchOnChain(bool) FilterQuery } type FilterOptions struct { - // .. + // ID is a caller-supplied label, not a unique registration identity. ID uint64 - // .. + // Finalize requests a final receipt after the mined receipt. Finalize bool - // . + // LimitOne stops matching after one mined receipt and preserves queued finality. LimitOne bool - // .. + // SearchCache searches the monitor's retained blocks when the filter is registered. SearchCache bool // QueryOnChainTxnHash will query the chain for the txn hash at the start @@ -144,17 +143,16 @@ type FilterOptions struct { // mined before the filter was created. QueryOnChainTxnHash bool - // .. + // QueryOnChain optionally fetches a receipt when the filter is registered. QueryOnChain func(context.Context) (*types.Receipt, error) - // MaxWait filter option waits some number of blocks without a filter match after - // which point will auto-unsubscribe the filter. This is useful to help automatically - // remove filters which likely won't come up. + // MaxWait limits blocks without a match before matching is exhausted. + // Receipts already queued for finality are still finalized. // // nil : use the ReceiptsListener option FilterMaxWaitNumBlocks value as the default - // -1 : set value to ReceiptsListener option NumFinality * 3 + // <0 : use ReceiptsListener option NumBlocksToFinality * 2 // 0 : option is disabled, and has no limit on wait. filters need to be manually unsubscribed - // N : a specified number of blocks without a match before unsusbcribe + // N : a specified number of blocks without a match before exhaustion MaxWait *int } @@ -169,6 +167,7 @@ type FilterCond struct { type filter struct { options FilterOptions cond FilterCond + match func(context.Context, Receipt) (bool, error) // startBlockNum is the first block number observed once filter is active startBlockNum uint64 @@ -206,7 +205,7 @@ func (f *filter) SearchCache(searchCache bool) FilterQuery { return f } -// DEPRECATED: please use QueryChainForTxnHash instead, which is the same thing, renamed to be more clear +// Deprecated: use QueryOnChainTxnHash instead. func (f *filter) SearchOnChain(searchOnChain bool) FilterQuery { f.options.QueryOnChainTxnHash = searchOnChain return f @@ -240,6 +239,9 @@ func (f *filter) Cond() FilterCond { } func (f *filter) Match(ctx context.Context, receipt Receipt) (bool, error) { + if f.match != nil { + return f.match(ctx, receipt) + } c := f.cond if c.TxnHash != nil { @@ -302,3 +304,28 @@ func (f *filter) closeExhausted() { func (f *filter) Exhausted() <-chan struct{} { return f.exhausted } + +// Fetch helpers own their options, counters, and exhaustion signal. A custom +// Filterer keeps its matching behavior without receiving helper option mutations. +func snapshotFilter(source Filterer) *filter { + options := source.Options() + if options.MaxWait != nil { + value := *options.MaxWait + options.MaxWait = &value + } + snapshot := &filter{options: options, cond: source.Cond(), exhausted: make(chan struct{})} + if builtIn, ok := source.(*filter); ok { + snapshot.match = builtIn.match + } else { + snapshot.match = source.Match + } + return snapshot +} + +func builtinFilter(filterer Filterer) *filter { + if owner, ok := filterer.(*filterOwner); ok { + filterer = owner.Filterer + } + builtIn, _ := filterer.(*filter) + return builtIn +} diff --git a/ethreceipts/finalizer.go b/ethreceipts/finalizer.go index c9ffa2db..85ed7dcf 100644 --- a/ethreceipts/finalizer.go +++ b/ethreceipts/finalizer.go @@ -2,96 +2,132 @@ package ethreceipts import ( "math/big" - "sort" "sync" - "github.com/0xsequence/ethkit" + "github.com/0xsequence/ethkit/go-ethereum/common" ) +// Public filter IDs are labels; distinct filters can share one label. +type receiptKey struct { + txnHash common.Hash + blockHash common.Hash + generation uint64 + owner *filterOwner +} + +func receiptOwner(receipt Receipt, owner *filterOwner) receiptKey { + return receiptKey{receipt.TransactionHash(), receipt.BlockHash(), receipt.generation, owner} +} + type finalizer struct { queue []finalTxn - txns map[ethkit.Hash]struct{} + txns map[receiptKey]struct{} numBlocksToFinality *big.Int mu sync.Mutex } - type finalTxn struct { receipt Receipt blockNum *big.Int } -func (f *finalizer) len() int { +func (f *finalizer) setFinality(num int) { f.mu.Lock() defer f.mu.Unlock() - return len(f.queue) + f.numBlocksToFinality = big.NewInt(int64(num)) } - -// func (f *finalizer) lastBlockNum() *big.Int { -// f.mu.Lock() -// defer f.mu.Unlock() -// if len(f.queue) == 0 { -// return big.NewInt(0) -// } -// return f.queue[0].blockNum -// } - -func (f *finalizer) enqueue(filterID uint64, receipt Receipt, blockNum *big.Int) { +func (f *finalizer) enqueue(owner *filterOwner, receipt Receipt, blockNum *big.Int) { f.mu.Lock() defer f.mu.Unlock() - - if receipt.Final { - // do not enqueue if the receipt is already final + if receipt.Final || blockNum == nil { return } - - txnHash := receipt.TransactionHash() - - // txn id based on the hash + filterID to ensure we get finalize callback for any unique filterID - txnID := txnHash - if filterID > 0 { - for i := 0; i < 8; i++ { - txnID[i] = txnID[i] + byte(filterID>>uint(i)) - } - } - - if _, ok := f.txns[txnID]; ok { - // update the blockNum if we already have this txn, as it could have been included - // again after a reorg in a new block + key := receiptOwner(receipt, owner) + if _, ok := f.txns[key]; ok { for i, entry := range f.queue { - if entry.receipt.TransactionHash() == txnHash { - f.queue[i] = finalTxn{receipt, blockNum} + if receiptOwner(entry.receipt, entry.receipt.owner) == key { + f.queue[i] = finalTxn{receipt, new(big.Int).Set(blockNum)} + return } } - return } - - // append new - f.queue = append(f.queue, finalTxn{receipt, blockNum}) - f.txns[txnID] = struct{}{} - - // sort block order from oldest to newest in case of a reorg - if len(f.queue) >= 2 && f.queue[0].blockNum.Cmp(f.queue[1].blockNum) < 0 { - sort.SliceStable(f.queue, func(i, j int) bool { - return f.queue[i].blockNum.Cmp(f.queue[j].blockNum) < 0 - }) + f.queue = append(f.queue, finalTxn{receipt, new(big.Int).Set(blockNum)}) + f.txns[key] = struct{}{} +} +func (f *finalizer) invalidateBlock(block blockRef) []Receipt { + f.mu.Lock() + defer f.mu.Unlock() + var invalidated []Receipt + retained := f.queue[:0] + for _, txn := range f.queue { + if txn.receipt.BlockHash() == block.hash && txn.receipt.generation == block.generation { + invalidated = append(invalidated, txn.receipt) + delete(f.txns, receiptOwner(txn.receipt, txn.receipt.owner)) + } else { + retained = append(retained, txn) + } } + clear(f.queue[len(retained):]) + f.queue = retained + return invalidated } - -func (f *finalizer) dequeue(currentBlockNum *big.Int) []finalTxn { +func (f *finalizer) invalidateOwner(owner *filterOwner) { f.mu.Lock() defer f.mu.Unlock() - - finalTxns := []finalTxn{} - + retained := f.queue[:0] for _, txn := range f.queue { - if currentBlockNum.Cmp(big.NewInt(0).Add(txn.blockNum, f.numBlocksToFinality)) > 0 { - finalTxns = append(finalTxns, txn) + if txn.receipt.owner == owner { + delete(f.txns, receiptOwner(txn.receipt, owner)) + } else { + retained = append(retained, txn) } } - - if len(finalTxns) > 0 { - f.queue = f.queue[len(finalTxns):] + clear(f.queue[len(retained):]) + f.queue = retained +} +func (f *finalizer) hasOwner(owner *filterOwner) bool { + f.mu.Lock() + defer f.mu.Unlock() + for _, txn := range f.queue { + if txn.receipt.owner == owner { + return true + } } - - return finalTxns + return false +} +func (f *finalizer) findOwner(filter Filterer) *filterOwner { + f.mu.Lock() + defer f.mu.Unlock() + for _, txn := range f.queue { + if sameFilter(txn.receipt.Filter, filter) { + return txn.receipt.owner + } + } + return nil +} +func (f *finalizer) clear() { + f.mu.Lock() + defer f.mu.Unlock() + f.queue = nil + clear(f.txns) +} +func (f *finalizer) dequeue(currentBlockNum *big.Int) []finalTxn { + f.mu.Lock() + defer f.mu.Unlock() + // Zero is unresolved until Run selects the network's finality policy. + if f.numBlocksToFinality == nil || f.numBlocksToFinality.Sign() <= 0 { + return nil + } + var finalized []finalTxn + retained := f.queue[:0] + for _, txn := range f.queue { + if currentBlockNum.Cmp(new(big.Int).Add(txn.blockNum, f.numBlocksToFinality)) >= 0 { + finalized = append(finalized, txn) + delete(f.txns, receiptOwner(txn.receipt, txn.receipt.owner)) + } else { + retained = append(retained, txn) + } + } + clear(f.queue[len(retained):]) + f.queue = retained + return finalized } diff --git a/ethreceipts/finalizer_test.go b/ethreceipts/finalizer_test.go new file mode 100644 index 00000000..70eb3c00 --- /dev/null +++ b/ethreceipts/finalizer_test.go @@ -0,0 +1,215 @@ +package ethreceipts + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestFinalizerFinalityBoundary(t *testing.T) { + tx, _, _ := hardeningTxn(t, 3000) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions(), b, hardeningBlock(101), hardeningBlock(102)) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + // Queue the receipt as it would have been at block 100, before head 102. + s.finalizer.enqueue(owner, Receipt{receipt: hardeningReceipt(b, tx), owner: owner, Filter: owner.Filterer}, b.Number()) + if err := s.finalizeReceipts(big.NewInt(101)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + if !l.isBlockFinal(b.Number()) { + t.Fatal("block is not final at the two-block threshold") + } + if err := s.finalizeReceipts(big.NewInt(102)); err != nil { + t.Fatal(err) + } + r := hardeningRead(t, s) + if !r.Final || r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q { + t.Fatal("wrong final receipt at the two-block threshold") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} + +func TestFetchFinalityAfterExhaustion(t *testing.T) { + tx, _, _ := hardeningTxn(t, 3001) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + hardeningStart(t, l) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + mined, waitFinal, err := l.FetchTransactionReceiptWithFinality(ctx, tx.Hash(), 1) + if err != nil { + t.Fatal(err) + } + if mined == nil || mined.Final || mined.Reorged || mined.BlockHash() != b.Hash() { + t.Fatalf("wrong mined receipt: %+v", mined) + } + l.mu.Lock() + s := l.subscribers[0] + l.mu.Unlock() + f := builtinFilter(s.Filters()[0]) + // A shallow reorg can exhaust matching while the mined block's finality + // remains queued. Use the same exhaustion steps as the listener. + s.exhaustFilter(f) + f.closeExhausted() + done := make(chan struct{}) + var final *Receipt + var finalErr error + go func() { + defer close(done) + final, finalErr = waitFinal(ctx) + }() + select { + case <-done: + t.Fatalf("finality stopped before its threshold: %v", finalErr) + case <-time.After(100 * time.Millisecond): + } + if err := s.finalizeReceipts(big.NewInt(102)); err != nil { + t.Fatal(err) + } + select { + case <-done: + if finalErr != nil || final == nil || !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || final.BlockHash() != b.Hash() { + t.Fatalf("wrong final receipt after exhaustion: %+v, error: %v", final, finalErr) + } + case <-ctx.Done(): + t.Fatal("finality did not complete after exhaustion") + } + if l.NumSubscribers() != 0 { + t.Error("fetch helper did not unsubscribe after finality") + } +} + +func TestHardeningSubscribeBeforeRunFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 2) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, DefaultOptions, b, hardeningBlock(101)) + s := l.Subscribe(FilterTxnHash(tx.Hash())).(*subscriber) + defer s.Unsubscribe() + hardeningStart(t, l) + if r := hardeningRead(t, s); r.Final { + t.Fatal("receipt final before network threshold") + } + processed := make(chan struct{}) + s.AddFilter(FilterLogs(func([]*types.Log) bool { return false }).SearchCache(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { close(processed); return nil, nil })) + select { + case <-processed: + case <-time.After(time.Second): + t.Fatal("registration did not complete") + } + hardeningNoReceipt(t, s) + if len(s.Filters()) != 2 { + t.Fatal("premature finality removed transaction filter") + } +} + +func TestHardeningFinalizerOrderAndOwners(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 4) + tx2, _, _ := hardeningTxn(t, 5) + newer, older := hardeningBlock(101, tx1), hardeningBlock(100, tx2) + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + return hardeningReceipt(newer, tx1), nil + } + return hardeningReceipt(older, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) + q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + for _, r := range []*types.Receipt{hardeningReceipt(newer, tx1), hardeningReceipt(older, tx2)} { + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: r}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + hardeningRead(t, s) + } + for _, tc := range []struct { + head int64 + tx *types.Transaction + b *ethmonitor.Block + }{{102, tx2, older}, {103, tx1, newer}} { + if err := s.finalizeReceipts(big.NewInt(tc.head)); err != nil { + t.Fatal(err) + } + owners := map[Filterer]bool{} + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.TransactionHash() != tc.tx.Hash() || r.BlockHash() != tc.b.Hash() { + t.Errorf("wrong final at %d: %+v", tc.head, r) + } + owners[r.Filter] = true + case <-time.After(100 * time.Millisecond): + t.Errorf("missing owner final at %d", tc.head) + } + } + if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { + t.Errorf("finality lost distinct owners with same public ID at %d", tc.head) + } + } + hardeningNoReceipt(t, s) +} + +func TestHardeningExhaustedOwnerKeepsQueuedFinal(t *testing.T) { + tx, _, _ := hardeningTxn(t, 17) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).MaxWait(1) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + // Exhaustion removes matching and closes its signal, but documented pending + // finality remains owned. Exercise that state without relying on a live chain. + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + q.(*filter).closeExhausted() + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Filter != q || final.BlockHash() != b.Hash() { + t.Fatal("exhaustion canceled a queued final") + } + hardeningNoReceipt(t, s) +} + +func TestHardeningClearCancelsExhaustedFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 18) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + q.(*filter).closeExhausted() + s.ClearFilters() + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} diff --git a/ethreceipts/helpers_test.go b/ethreceipts/helpers_test.go new file mode 100644 index 00000000..e01ab671 --- /dev/null +++ b/ethreceipts/helpers_test.go @@ -0,0 +1,231 @@ +package ethreceipts + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/ethrpc" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/0xsequence/ethkit/go-ethereum/crypto" + "github.com/goware/breaker" +) + +// Only the RPC methods used by the listener are provided. No external node is needed. +type hardeningProvider struct { + ethrpc.RawInterface + chainID func(context.Context) (*big.Int, error) + receipt func(context.Context, common.Hash) (*types.Receipt, error) +} + +func (p *hardeningProvider) ChainID(ctx context.Context) (*big.Int, error) { + if p.chainID != nil { + return p.chainID(ctx) + } + return big.NewInt(1), nil +} + +func (p *hardeningProvider) TransactionReceipt(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if p.receipt != nil { + return p.receipt(ctx, h) + } + return nil, ethereum.NotFound +} + +func hardeningBlock(num int64, txns ...*types.Transaction) *ethmonitor.Block { + b := types.NewBlockWithHeader(&types.Header{Number: big.NewInt(num), BlockHash: common.BigToHash(big.NewInt(num)), ParentHash: common.BigToHash(big.NewInt(num - 1)), GasLimit: 30_000_000, Time: uint64(num)}) + return ðmonitor.Block{Block: b.WithBody(types.Body{Transactions: txns}), Event: ethmonitor.Added, OK: true} +} + +func hardeningTxn(t *testing.T, nonce uint64) (*types.Transaction, common.Address, common.Address) { + t.Helper() + key, err := crypto.HexToECDSA("1111111111111111111111111111111111111111111111111111111111111111") + if err != nil { + t.Fatal(err) + } + from := crypto.PubkeyToAddress(key.PublicKey) + to := common.HexToAddress("0x2222222222222222222222222222222222222222") + tx, err := types.SignTx(types.NewTransaction(nonce, to, big.NewInt(1), 21000, big.NewInt(1), nil), types.NewEIP155Signer(big.NewInt(1)), key) + if err != nil { + t.Fatal(err) + } + return tx, from, to +} + +func hardeningReceipt(b *ethmonitor.Block, tx *types.Transaction) *types.Receipt { + return &types.Receipt{TxHash: tx.Hash(), BlockHash: b.Hash(), BlockNumber: b.Number(), Status: 1} +} + +func hardeningListener(t *testing.T, p *hardeningProvider, opts Options, blocks ...*ethmonitor.Block) *ReceiptsListener { + t.Helper() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger = log + mo.WithLogs = true + mo.BlockRetentionLimit = 100 + mo.Bootstrap = true + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if len(blocks) == 0 { + blocks = []*ethmonitor.Block{hardeningBlock(100)} + } + if err = m.Chain().BootstrapFromBlocks(blocks); err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + l.br = breaker.New(log, 0, 1, 0) + return l +} + +func hardeningStart(t *testing.T, l *ReceiptsListener) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + var runErr error + t.Cleanup(func() { + cancel() + l.Stop() + select { + case <-done: + if runErr != nil && !errors.Is(runErr, context.Canceled) { + t.Errorf("Run: %v", runErr) + } + case <-time.After(time.Second): + t.Error("Run did not stop") + } + }) + go func() { + defer close(done) + runErr = l.Run(ctx) + }() + deadline := time.After(time.Second) + for l.monitor.NumSubscribers() != 1 { + select { + case <-done: + t.Fatalf("Run exited before subscription: %v", runErr) + case <-deadline: + t.Fatal("Run did not subscribe to monitor") + default: + time.Sleep(time.Millisecond) + } + } +} + +func hardeningRead(t *testing.T, s Subscription) Receipt { + t.Helper() + select { + case r := <-s.TransactionReceipt(): + return r + case <-time.After(time.Second): + t.Fatal("missing receipt") + return Receipt{} + } +} + +func hardeningNoReceipt(t *testing.T, s Subscription) { + t.Helper() + select { + case r := <-s.TransactionReceipt(): + t.Errorf("unexpected receipt: txn=%s block=%s final=%v reorged=%v owner=%p", r.TransactionHash(), r.BlockHash(), r.Final, r.Reorged, r.Filter) + case <-time.After(30 * time.Millisecond): + } +} + +func hardeningProcess(t *testing.T, l *ReceiptsListener, s *subscriber, blocks ...*ethmonitor.Block) { + t.Helper() + if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + t.Fatal(err) + } +} + +func hardeningOptions() Options { opts := DefaultOptions; opts.NumBlocksToFinality = 2; return opts } + +// The real monitor polls this provider and builds/broadcasts reorg events. Tests +// advance its canonical RPC responses without changing monitor production APIs. +type ownershipLiveProvider struct { + *hardeningProvider + mu sync.Mutex + canonical map[uint64]*types.Block + blocks map[common.Hash]*types.Block +} + +func (p *ownershipLiveProvider) advance(blocks ...*types.Block) { + p.mu.Lock() + defer p.mu.Unlock() + for _, b := range blocks { + p.canonical[b.NumberU64()] = b + p.blocks[b.Hash()] = b + } +} + +func ownershipBlockPayload(b *types.Block) (json.RawMessage, error) { + if b == nil { + return nil, ethereum.NotFound + } + header := b.Header() + header.Difficulty = big.NewInt(0) + payload, err := json.Marshal(header) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err = json.Unmarshal(payload, &fields); err != nil { + return nil, err + } + fields["transactions"], err = json.Marshal(b.Transactions()) + if err != nil { + return nil, err + } + fields["uncles"] = json.RawMessage(`[]`) + return json.Marshal(fields) +} + +func (p *ownershipLiveProvider) RawBlockByNumber(_ context.Context, num *big.Int) (json.RawMessage, error) { + p.mu.Lock() + b := p.canonical[num.Uint64()] + p.mu.Unlock() + return ownershipBlockPayload(b) +} + +func (p *ownershipLiveProvider) RawBlockByHash(_ context.Context, hash common.Hash) (json.RawMessage, error) { + p.mu.Lock() + b := p.blocks[hash] + p.mu.Unlock() + return ownershipBlockPayload(b) +} + +func (p *ownershipLiveProvider) RawFilterLogs(context.Context, ethereum.FilterQuery) (json.RawMessage, error) { + return json.RawMessage(`[]`), nil +} + +func receiptsFixCollect(t *testing.T, s Subscription) []Receipt { + t.Helper() + var receipts []Receipt + timer := time.NewTimer(40 * time.Millisecond) + defer timer.Stop() + for { + select { + case r, ok := <-s.TransactionReceipt(): + if !ok { + t.Fatal("receipt channel closed unexpectedly") + } + receipts = append(receipts, r) + case <-timer.C: + return receipts + } + } +} diff --git a/ethreceipts/lifecycle_test.go b/ethreceipts/lifecycle_test.go new file mode 100644 index 00000000..63907f7f --- /dev/null +++ b/ethreceipts/lifecycle_test.go @@ -0,0 +1,282 @@ +package ethreceipts + +import ( + "context" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/goware/breaker" +) + +func TestSubscriptionUnsubscribe(t *testing.T) { + for _, workers := range []int{1, 8} { + name := "repeated" + if workers > 1 { + name = "concurrent" + } + t.Run(name, func(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + s := l.Subscribe() + other := l.Subscribe() + defer other.Unsubscribe() + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + defer func() { + if p := recover(); p != nil { + t.Errorf("Unsubscribe panicked: %v", p) + } + }() + <-start + s.Unsubscribe() + s.Unsubscribe() + }() + } + close(start) + wg.Wait() + if l.NumSubscribers() != 1 { + t.Fatal("Unsubscribe did not remove exactly one subscription") + } + select { + case <-s.Done(): + default: + t.Error("unsubscribed Done channel remains open") + } + select { + case _, ok := <-s.TransactionReceipt(): + if ok { + t.Error("unsubscribed receipt channel remains open") + } + case <-time.After(time.Second): + t.Error("unsubscribed receipt channel did not close") + } + select { + case <-other.Done(): + t.Error("Unsubscribe closed an unrelated subscription") + default: + } + }) + } +} + +func TestHardeningStopCancelsStartup(t *testing.T) { + entered := make(chan struct{}) + p := &hardeningProvider{chainID: func(ctx context.Context) (*big.Int, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() }} + l := hardeningListener(t, p, DefaultOptions) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + <-entered + l.Stop() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + cancel() + <-done + t.Error("Stop did not cancel startup ChainID request") + } +} + +func TestHardeningStopCancelsStartupBackoff(t *testing.T) { + entered := make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{chainID: func(context.Context) (*big.Int, error) { + if calls.Add(1) == 1 { + close(entered) + return nil, errors.New("transient ChainID failure") + } + return nil, breaker.ErrFatal + }} + l := hardeningListener(t, p, DefaultOptions) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + <-entered + // Allow the failed attempt to enter its one-second backoff. + time.Sleep(100 * time.Millisecond) + l.Stop() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("startup cancellation error: %v", err) + } + case <-time.After(500 * time.Millisecond): + cancel() + <-done + t.Error("Stop waited for startup ChainID backoff") + } + if calls.Load() != 1 { + t.Errorf("ChainID retried after cancellation: %d calls", calls.Load()) + } +} + +func TestHardeningMonitorStopCancelsRegistration(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + entered := make(chan struct{}) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return false }).QueryOnChain(func(ctx context.Context) (*types.Receipt, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() })) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { done <- l.Run(context.Background()) }() + <-entered + l.monitor.UnsubscribeAll(errors.New("monitor closed")) + select { + case <-done: + case <-time.After(100 * time.Millisecond): + l.Stop() + <-done + t.Error("monitor closure did not cancel registration") + } +} + +func TestHardeningFetchSemaphoreCancellation(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + close(entered) + select { + case <-release: + return &types.Receipt{TxHash: h, BlockNumber: big.NewInt(100)}, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + }} + opts := hardeningOptions() + opts.MaxConcurrentFetchReceiptWorkers = 1 + l := hardeningListener(t, p, opts) + first := make(chan error, 1) + go func() { _, err := l.fetchTransactionReceipt(context.Background(), common.Hash{1}, true); first <- err }() + <-entered + ctx, cancel := context.WithCancel(context.Background()) + cancel() + second := make(chan error, 1) + go func() { _, err := l.fetchTransactionReceipt(ctx, common.Hash{2}, true); second <- err }() + blocked := false + select { + case err := <-second: + if !errors.Is(err, context.Canceled) { + t.Errorf("cancellation error: %v", err) + } + case <-time.After(100 * time.Millisecond): + blocked = true + } + close(release) + <-first + if blocked { + <-second + t.Error("canceled fetch waited for unrelated fetch slot") + } +} + +func TestHardeningFinalityWaitCancellation(t *testing.T) { + tx, _, _ := hardeningTxn(t, 10) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + hardeningStart(t, l) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + mined, wait, err := l.FetchTransactionReceiptWithFinality(parent, tx.Hash()) + if err != nil { + t.Fatal(err) + } + if mined.Final { + t.Fatal("initial receipt final") + } + ctx, finalCancel := context.WithCancel(context.Background()) + finalCancel() + done := make(chan error, 1) + go func() { _, err := wait(ctx); done <- err }() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("wait error: %v", err) + } + case <-time.After(100 * time.Millisecond): + cancel() + <-done + t.Error("finality waiter ignored its own context") + } +} + +func TestHardeningConcurrentRunStop(t *testing.T) { + for i := 0; i < 5; i++ { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + _, l.ctxStop = context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var wg sync.WaitGroup + start := make(chan struct{}) + for j := 0; j < 16; j++ { + wg.Add(1) + go func() { defer wg.Done(); <-start; _ = l.Run(ctx) }() + } + wg.Add(1) + go func() { + defer wg.Done() + <-start + for j := 0; j < 16; j++ { + l.Stop() + } + }() + close(start) + wg.Wait() + } +} + +func TestHardeningMonitorClosureCancelsHeadWait(t *testing.T) { + p := &hardeningProvider{} + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger = log + mo.WithLogs = true + mo.BlockRetentionLimit = 100 + mo.Bootstrap = true + monitor, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, monitor, hardeningOptions()) + if err != nil { + t.Fatal(err) + } + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + deadline := time.After(time.Second) + for monitor.NumSubscribers() != 1 { + select { + case <-deadline: + t.Fatal("listener did not subscribe") + default: + time.Sleep(time.Millisecond) + } + } + monitor.UnsubscribeAll(errors.New("monitor closed before first block")) + select { + case <-done: + case <-time.After(100 * time.Millisecond): + cancel() + // Baseline LatestBlockNum has a fixed 30-second wait; let it finish + // without mutating the monitor's bootstrap state during concurrent reads. + select { + case <-done: + case <-time.After(32 * time.Second): + t.Fatal("Run did not clean up") + } + t.Error("monitor closure did not cancel waiting for first head") + } +} diff --git a/ethreceipts/ownership_regression_test.go b/ethreceipts/ownership_regression_test.go new file mode 100644 index 00000000..b3d1cd5d --- /dev/null +++ b/ethreceipts/ownership_regression_test.go @@ -0,0 +1,572 @@ +package ethreceipts + +import ( + "context" + "errors" + "fmt" + "math/big" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningAddressFilters(t *testing.T) { + tx, from, to := hardeningTxn(t, 3) + b := hardeningBlock(100, tx) + for _, tc := range []struct { + name string + q FilterQuery + }{{"from", FilterFrom(from)}, {"to", FilterTo(to)}} { + t.Run(tc.name, func(t *testing.T) { + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions(), b) + s := l.Subscribe(tc.q.SearchCache(true)) + defer s.Unsubscribe() + hardeningStart(t, l) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() { + t.Fatalf("wrong matching transaction: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("signed address filter missed cached transaction") + } + }) + } +} + +func TestHardeningLimitOneSnapshots(t *testing.T) { + for _, sameBlock := range []bool{true, false} { + t.Run(map[bool]string{true: "same_block", false: "retained_blocks"}[sameBlock], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 7) + tx2, _, _ := hardeningTxn(t, 8) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + blocks := ethmonitor.Blocks{b1, b2} + if sameBlock { + b1 = hardeningBlock(100, tx1, tx2) + b2 = b1 + blocks = ethmonitor.Blocks{b1} + } + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + return hardeningReceipt(b1, tx1), nil + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true)).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + r := hardeningRead(t, s) + if r.TransactionHash() != tx1.Hash() { + t.Errorf("LimitOne did not select first match: %s", r.TransactionHash()) + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestHardeningConcurrentLimitOne(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 12) + tx2, _, _ := hardeningTxn(t, 13) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + close(entered) + select { + case <-release: + return hardeningReceipt(b1, tx1), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + done := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{fs}) + done <- err + }() + <-entered + if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b2}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + close(release) + <-done + r := hardeningRead(t, s) + if r.TransactionHash() != tx1.Hash() || r.Filter != q { + t.Errorf("concurrent LimitOne selected different transaction: %+v", r) + } + hardeningNoReceipt(t, s) + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.TransactionHash() != tx1.Hash() || final.Filter != q { + t.Errorf("wrong claimed final: %+v", final) + } + hardeningNoReceipt(t, s) +} + +func TestHardeningExplicitRemovalCancelsOnlyOwner(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove_one", true: "clear_all"}[clearAll], func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 16) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(1).Finalize(true) + q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(2).Finalize(true) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + hardeningRead(t, s) + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q1.(Filterer)) + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + if !clearAll { + r := hardeningRead(t, s) + if !r.Final || r.Filter != q2 || r.BlockHash() != b.Hash() { + t.Errorf("removal lost surviving owner: %+v", r) + } + } + hardeningNoReceipt(t, s) + }) + } +} + +type ownershipValueFilter struct { + Filterer + labels []string + match func(context.Context, Receipt) (bool, error) +} + +func (f ownershipValueFilter) Match(ctx context.Context, r Receipt) (bool, error) { + return f.match(ctx, r) +} + +func TestOwnershipCustomValueFilter(t *testing.T) { + tx, _, _ := hardeningTxn(t, 100) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions(), b) + base := FilterLogs(func([]*types.Log) bool { return false }).SearchCache(true).(Filterer) + var matches atomic.Int32 + q := ownershipValueFilter{base, []string{"custom"}, func(_ context.Context, r Receipt) (bool, error) { + matches.Add(1) + return r.TransactionHash() == tx.Hash(), nil + }} + s := l.Subscribe(q) + defer s.Unsubscribe() + gotFilter, ok := s.Filters()[0].(ownershipValueFilter) + if !ok || gotFilter.Filterer != base || gotFilter.labels[0] != "custom" { + t.Fatal("public Filters lost original custom value") + } + hardeningStart(t, l) + r := hardeningRead(t, s) + original, ok := r.Filter.(ownershipValueFilter) + if !ok || original.Filterer != base || original.labels[0] != "custom" || original.match == nil { + t.Fatal("receipt lost original custom Filterer value") + } + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || matches.Load() == 0 { + t.Fatal("custom Match behavior lost") + } + s.ClearFilters() +} + +type ownershipCollectionFilter struct { + Filterer + cancel context.CancelFunc +} + +func (f *ownershipCollectionFilter) Match(context.Context, Receipt) (bool, error) { + if f.cancel != nil { + f.cancel() + return false, nil + } + return false, errors.New("temporary match failure") +} + +func TestOwnershipCollectionAbort(t *testing.T) { + for _, canceled := range []bool{false, true} { + t.Run(map[bool]string{false: "matcher_error", true: "cancellation"}[canceled], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 101) + tx2, _, _ := hardeningTxn(t, 102) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + bad := &ownershipCollectionFilter{Filterer: FilterLogs(func([]*types.Log) bool { return false }).(Filterer)} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if canceled { + bad.cancel = cancel + } + s := l.Subscribe(q, bad).(*subscriber) + defer s.Unsubscribe() + _, err := s.matchFiltersAndPublish(ctx, s.Filters(), []Receipt{{receipt: hardeningReceipt(b1, tx1)}}) + if err == nil { + t.Error("expected matching abort") + } + if canceled && !errors.Is(err, context.Canceled) { + t.Errorf("cancellation error: %v", err) + } + if len(s.inFlight) != 0 || len(s.claims) != 0 { + t.Errorf("aborted collection retained work: inFlight=%d claims=%d", len(s.inFlight), len(s.claims)) + } + hardeningNoReceipt(t, s) + s.RemoveFilter(bad) + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b2, tx2)}}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("unused claim blocked later transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("aborted reservation blocked later processing") + } + }) + } +} + +func TestOwnershipQueryReuse(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove", true: "clear"}[clearAll], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 103) + tx2, _, _ := hardeningTxn(t, 104) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b1, tx1)}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q.(Filterer)) + } + if len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 { + t.Error("explicit removal retained canceled ownership") + } + s.AddFilter(q) + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b2, tx2)}}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q || r.Final { + t.Error("reused query kept old selection") + } + case <-time.After(100 * time.Millisecond): + t.Error("reused LimitOne query missed second transaction") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("canceled owner finalized in new lifetime") + } + case <-time.After(100 * time.Millisecond): + t.Error("new owner did not finalize") + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestOwnershipReaddRejectsOldWorker(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove", true: "clear"}[clearAll], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 105) + tx2, _, _ := hardeningTxn(t, 106) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + close(entered) + select { + case <-release: + return hardeningReceipt(b1, tx1), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{s.Filters()}) + done <- err + }() + <-entered + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q.(Filterer)) + } + s.AddFilter(q) + hardeningProcess(t, l, s, b2) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("new lifetime selected old transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("new lifetime suppressed by old claim") + } + close(release) + <-done + hardeningNoReceipt(t, s) + if len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 { + t.Error("completed/canceled lifetime retained state") + } + }) + } +} + +func TestOwnershipCompletionReleasesState(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + s := l.Subscribe().(*subscriber) + defer s.Unsubscribe() + for i := 0; i < 32; i++ { + tx, _, _ := hardeningTxn(t, uint64(200+i)) + b := hardeningBlock(100, tx) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(i%4 != 0) + s.AddFilter(q) + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + switch i % 4 { + case 1: + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + case 2: + s.RemoveFilter(q.(Filterer)) + case 3: + s.ClearFilters() + } + if len(s.Filters()) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 { + t.Fatalf("completion %d retained state: active=%d claims=%d inFlight=%d deliveries=%d", i, len(s.Filters()), len(s.claims), len(s.inFlight), len(s.deliveries)) + } + if len(s.finalizer.queue) != 0 || len(s.finalizer.txns) != 0 || len(s.pendingReceipts) != 0 { + t.Fatalf("completion %d retained queued ownership", i) + } + } +} + +func TestReceiptsFixCustomValueRemoval(t *testing.T) { + for _, pointer := range []bool{false, true} { + for _, exhausted := range []bool{false, true} { + for _, source := range []string{"original", "Filters", "Receipt.Filter"} { + t.Run(fmt.Sprintf("pointer=%v/exhausted=%v/%s", pointer, exhausted, source), func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 801) + tx2, _, _ := hardeningTxn(t, 802) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + custom := func(label string) Filterer { + return ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).ID(7).(Filterer), []string{label}, func(context.Context, Receipt) (bool, error) { return label != "", nil }} + } + value := custom("selected").(ownershipValueFilter) + var q Filterer = value + if pointer { + q = &value + } + other := custom("surviving") + s := l.Subscribe(q, other).(*subscriber) + defer s.Unsubscribe() + first, second := s.filterers()[0].(*filterOwner), s.filterers()[1].(*filterOwner) + fromFilters := s.Filters()[0] + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(b1, tx1)) + var mined Receipt + for i := 0; i < 2; i++ { + r := hardeningRead(t, s) + if r.owner == first { + mined = r + if pointer { + if r.Filter != q || fromFilters != q { + t.Fatal("public pointer identity changed") + } + } else if r.Filter.(ownershipValueFilter).Filterer != value.Filterer || fromFilters.(ownershipValueFilter).Filterer != value.Filterer { + t.Fatal("original public custom value changed") + } + } + } + if exhausted { + s.exhaustFilter(first) + value.Filterer.(*filter).closeExhausted() + } + candidate := q + switch source { + case "Filters": + candidate = fromFilters + case "Receipt.Filter": + candidate = mined.Filter + } + s.RemoveFilter(candidate) + if s.hasFilter(first) || s.finalizer.hasOwner(first) || !s.hasFilter(second) || !s.finalizer.hasOwner(second) { + t.Error("public removal did not isolate selected registration/finality") + } + process(hardeningReceipt(b2, tx2)) + late := receiptsFixCollect(t, s) + if len(late) != 1 || late[0].owner != second || late[0].TransactionHash() != tx2.Hash() || late[0].Final || late[0].Reorged { + t.Error("selected cancellation changed later surviving delivery") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(t, s) + seen := make(map[common.Hash]bool) + for _, r := range finals { + if r.owner != second || !r.Final || r.Reorged || seen[r.TransactionHash()] { + t.Error("removed custom value still finalized or surviving final duplicated") + } + seen[r.TransactionHash()] = true + } + if len(finals) != 2 || !seen[tx1.Hash()] || !seen[tx2.Hash()] { + t.Error("custom removal lost surviving finals") + } + }) + } + } + } +} + +func TestReceiptsFixSharedBaseAliases(t *testing.T) { + tx, _, _ := hardeningTxn(t, 803) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + base := FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).(Filterer) + first := ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + second := ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + other := ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer), []string{"other"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + s := l.Subscribe(first, second, other).(*subscriber) + defer s.Unsubscribe() + owners := s.filterers() + if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + hardeningRead(t, s) + } + // Shared base signals define aliases: old RemoveFilter removes the first + // matching registration. Independent callback identities use distinct bases. + s.RemoveFilter(second) + if s.hasFilter(owners[0].(*filterOwner)) || s.finalizer.hasOwner(owners[0].(*filterOwner)) || !s.hasFilter(owners[1].(*filterOwner)) || !s.hasFilter(owners[2].(*filterOwner)) { + t.Error("shared-base removal did not preserve first-match alias semantics") + } + s.RemoveFilter(s.Filters()[0]) + if s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) || !s.finalizer.hasOwner(owners[2].(*filterOwner)) { + t.Error("removing second base alias affected independent owner") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(t, s) + if len(finals) != 1 || finals[0].owner != owners[2] || !finals[0].Final || finals[0].TransactionHash() != tx.Hash() { + t.Error("shared aliases canceled an unrelated queued final") + } +} + +type receiptsFixComparableFilter struct { + Filterer + label string +} + +func (f receiptsFixComparableFilter) Match(context.Context, Receipt) (bool, error) { + return f.label != "", nil +} + +func TestReceiptsFixComparableSharedBaseIsolation(t *testing.T) { + for _, pointer := range []bool{false, true} { + t.Run(fmt.Sprintf("pointer=%v", pointer), func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 804) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + base := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer) + var first, second Filterer = receiptsFixComparableFilter{base, "first"}, receiptsFixComparableFilter{base, "second"} + if pointer { + first = &ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + second = &ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + } + s := l.Subscribe(first, second).(*subscriber) + defer s.Unsubscribe() + owners := s.filterers() + if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + hardeningRead(t, s) + s.RemoveFilter(second) + if !s.hasFilter(owners[0].(*filterOwner)) || !s.finalizer.hasOwner(owners[0].(*filterOwner)) || s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) { + t.Fatal("base fallback overrode definitive comparable identity") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(t, s) + if len(finals) != 1 || finals[0].owner != owners[0] || !finals[0].Final { + t.Fatal("comparable cancellation lost surviving owner") + } + }) + } +} + +type receiptsFixNilSignalFilter struct{ ownershipValueFilter } + +func (receiptsFixNilSignalFilter) Exhausted() <-chan struct{} { return nil } + +func TestReceiptsFixCustomValueWithoutIdentity(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + value := receiptsFixNilSignalFilter{ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).(Filterer), []string{"no identity"}, func(context.Context, Receipt) (bool, error) { return true, nil }}} + other := FilterLogs(func([]*types.Log) bool { return true }).(Filterer) + s := l.Subscribe(value, other).(*subscriber) + defer s.Unsubscribe() + s.RemoveFilter(s.Filters()[0]) + if len(s.Filters()) != 2 { + t.Fatal("unidentifiable callback value removed another registration") + } + s.ClearFilters() + s.AddFilter(&value, other) + s.RemoveFilter(s.Filters()[0]) + if len(s.Filters()) != 1 || s.Filters()[0] != other { + t.Fatal("pointer identity did not isolate custom nil-signal removal") + } +} diff --git a/ethreceipts/pending_receipts_test.go b/ethreceipts/pending_receipts_test.go new file mode 100644 index 00000000..54141e23 --- /dev/null +++ b/ethreceipts/pending_receipts_test.go @@ -0,0 +1,357 @@ +package ethreceipts + +import ( + "context" + "errors" + "math/big" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningPendingOwners(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6) + b := hardeningBlock(100, tx) + var failing atomic.Bool + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if failing.Load() { + return nil, errors.New("temporary provider failure") + } + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(0).Finalize(true) + q2 := FilterTxnHash(tx.Hash()).ID(0) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, chainID: big.NewInt(1)}}) + if err == nil { + t.Fatal("expected transient fetch failure") + } + failing.Store(false) + s.retryMu.Lock() + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + } + s.retryMu.Unlock() + s.retryPendingReceipts(context.Background()) + owners := map[Filterer]bool{} + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if r.Final || r.BlockHash() != b.Hash() { + t.Errorf("wrong recovered receipt: %+v", r) + } + owners[r.Filter] = true + case <-time.After(100 * time.Millisecond): + t.Error("missing recovered owner") + } + } + if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { + t.Error("retry lost an owner") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.BlockHash() != b.Hash() { + t.Errorf("wrong recovered final: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing recovered final") + } + } +} + +func TestOwnershipPendingDropReleasesClaim(t *testing.T) { + for _, notFound := range []bool{false, true} { + t.Run(map[bool]string{false: "retry_limit", true: "not_found"}[notFound], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 108) + tx2, _, _ := hardeningTxn(t, 109) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx2.Hash() { + return hardeningReceipt(b2, tx2), nil + } + return nil, errors.New("temporary receipt failure") + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx1, blockHash: b1.Hash(), blockNum: b1.Number()}}); err == nil { + t.Fatal("expected initial pending fetch failure") + } + if len(s.pendingReceipts) != 1 || len(s.claims) != 1 { + t.Fatal("pending fetch lost its selection") + } + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + pending.attempts = maxReceiptRetryAttempts - 1 + } + if notFound { + p.receipt = func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx2.Hash() { + return hardeningReceipt(b2, tx2), nil + } + return nil, ethereum.NotFound + } + } + s.retryPendingReceipts(context.Background()) + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 { + t.Error("discarded pending work retained an undelivered claim") + } + hardeningProcess(t, l, s, b2) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("released pending claim selected wrong transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("discarded pending work blocked a new transaction") + } + }) + } +} + +func TestReceiptsFixRetryDeadlineAfterFetch(t *testing.T) { + for _, terminal := range []string{"retry", "remove", "clear", "rollback"} { + t.Run(terminal, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 800) + b := hardeningBlock(100, tx) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), b) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + seed := Receipt{transaction: tx, blockHash: b.Hash(), blockNum: b.Number()} + if !s.reserve(seed, owner) { + t.Fatal("initial reservation failed") + } + s.addPendingReceipt(seed, owner) + s.releaseReservation(seed, owner) + key := receiptOwner(seed, owner) + s.pendingReceipts[key].nextRetryAt = time.Time{} + pending := s.pendingReceipts[key] + parent := context.Background() + attempt, cancel := context.WithTimeout(parent, 200*time.Millisecond) + defer cancel() + s.deliveryMu.Lock() + done := make(chan struct{}) + go func() { s.retryPendingReceipts(attempt); close(done) }() + deadline := time.Now().Add(time.Second) + for { + _, cached, _ := l.pastReceipts.Get(parent, tx.Hash().Hex()) + if cached && len(l.fetchSem) == 0 { + break + } + if time.Now().After(deadline) { + s.deliveryMu.Unlock() + t.Fatal("receipt fetch did not complete before publication") + } + time.Sleep(time.Millisecond) + } + <-attempt.Done() + terminalDone := make(chan struct{}) + go func() { + defer close(terminalDone) + switch terminal { + case "remove": + s.RemoveFilter(q.(Filterer)) + case "clear": + s.ClearFilters() + case "rollback": + ref := l.invalidateBlock(parent, b) + s.rollbackBlock(ref) + } + }() + s.deliveryMu.Unlock() + <-done + <-terminalDone + hardeningNoReceipt(t, s) + if parent.Err() != nil { + t.Fatal("retry child canceled its listener parent") + } + if terminal == "retry" { + if s.pendingReceipts[key] != pending || len(s.claims) != 1 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Errorf("expired successful retry lost exact ownership: pending=%d claims=%d", len(s.pendingReceipts), len(s.claims)) + } + if s.pendingReceipts[key] == nil { + return + } + if pending.attempts != 1 || pending.nextRetryAt.After(time.Now().Add(time.Second)) { + t.Error("successful expired attempt was not promptly released for retry") + } + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(parent) + r := hardeningRead(t, s) + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q || r.Final || r.Reorged { + t.Error("later active retry lost mined identity") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + r = hardeningRead(t, s) + if !r.Final || r.Reorged || r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q { + t.Error("later active retry lost finality identity") + } + if calls.Load() != 1 { + t.Error("later retry discarded its successful cached receipt") + } + } else { + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Error("expired retry resurrected terminal owner/block state") + } + s.retryPendingReceipts(parent) + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestReceiptsFixPartialPendingCandidate(t *testing.T) { + for _, terminal := range []string{"retry", "rollback", "readoption", "remove", "clear"} { + t.Run(terminal, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 808) + replacement, _, _ := hardeningTxn(t, 809) + old, next := hardeningBlock(100, tx), hardeningBlock(101, replacement) + var ready atomic.Bool + var calls atomic.Int32 + l := hardeningListener(t, &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if !ready.Load() { + return nil, errors.New("provider temporarily unavailable") + } + return hardeningReceipt(old, tx), nil + }}, hardeningOptions(), old) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true). + QueryOnChain(func(context.Context) (*types.Receipt, error) { return &types.Receipt{TxHash: tx.Hash()}, nil }) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + if err := l.queryFilterOnChain(context.Background(), s, s.filterers()); err != nil { + t.Fatal(err) + } + if len(s.pendingReceipts) != 1 || s.claims[owner] != tx.Hash() { + t.Fatal("partial public callback did not enter the pending completion path") + } + var key receiptKey + var pending *pendingReceipt + for k, p := range s.pendingReceipts { + key, pending = k, p + } + if key.blockHash != (common.Hash{}) { + t.Fatal("partial callback already knew its block") + } + pending.nextRetryAt = time.Time{} + ready.Store(true) + attempt, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + s.deliveryMu.Lock() + done := make(chan struct{}) + go func() { s.retryPendingReceipts(attempt); close(done) }() + deadline := time.Now().Add(time.Second) + for { + _, cached, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if cached && len(l.fetchSem) == 0 { + break + } + if time.Now().After(deadline) { + s.deliveryMu.Unlock() + t.Fatal("partial pending fetch did not complete") + } + time.Sleep(time.Millisecond) + } + <-attempt.Done() + s.deliveryMu.Unlock() + <-done + hardeningNoReceipt(t, s) + if s.pendingReceipts[key] != pending || pending.attempts != 1 || calls.Load() != 2 { + t.Fatal("successful canceled completion did not retain its exact pending owner") + } + if terminal == "retry" { + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(context.Background()) + mined := hardeningRead(t, s) + if mined.Filter != q || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != old.Hash() || mined.generation != 0 || mined.Final || mined.Reorged { + t.Fatal("partial pending live retry lost its learned candidate") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if final.Filter != q || final.TransactionHash() != tx.Hash() || final.BlockHash() != old.Hash() || !final.Final || final.Reorged || calls.Load() != 2 { + t.Fatal("partial retry lost cached completion or canonical finality") + } + hardeningNoReceipt(t, s) + return + } + switch terminal { + case "rollback", "readoption": + ref := l.invalidateBlock(context.Background(), old) + if terminal == "readoption" { + // Even if authoritative same-hash acceptance wins before subscriber + // cleanup, the learned generation-0 candidate must stay invalid. + l.acceptBlock(l.monitor.GetBlock(old.Hash())) + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(context.Background()) + hardeningNoReceipt(t, s) + } + s.rollbackBlock(ref) + case "remove": + s.RemoveFilter(q.(Filterer)) + case "clear": + s.ClearFilters() + } + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Error("terminal invalidation retained a learned pending candidate or selection") + } + s.retryPendingReceipts(context.Background()) + hardeningNoReceipt(t, s) + if terminal == "remove" || terminal == "clear" { + s.AddFilter(q) + if s.owner(q.(Filterer)) == owner { + t.Fatal("new registration reused the canceled owner") + } + } + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: hardeningReceipt(next, replacement)}}); err != nil { + t.Fatal(err) + } + select { + case mined := <-s.TransactionReceipt(): + if mined.Filter != q || mined.TransactionHash() != replacement.Hash() || mined.BlockHash() != next.Hash() || mined.Final || mined.Reorged { + t.Error("fresh replacement lost its exact mined identity") + } + case <-time.After(100 * time.Millisecond): + t.Error("orphan pending claim blocked a fresh canonical replacement") + return + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if final.Filter != q || final.TransactionHash() != replacement.Hash() || final.BlockHash() != next.Hash() || !final.Final || final.Reorged { + t.Error("terminal invalidation published an orphan final") + } + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.Filters()) != 0 || calls.Load() != 2 { + t.Error("terminal candidate or completed owner was retained/re-fetched") + } + hardeningNoReceipt(t, s) + }) + } +} diff --git a/ethreceipts/receipt.go b/ethreceipts/receipt.go index 01ad8b96..94af72df 100644 --- a/ethreceipts/receipt.go +++ b/ethreceipts/receipt.go @@ -21,9 +21,12 @@ type Receipt struct { transaction *types.Transaction receipt *types.Receipt logs []*types.Log + blockNum *big.Int + blockHash common.Hash + generation uint64 + owner *filterOwner - // TODOXXX: this intermediate type is lame.. with new ethrpc we can remove - // NOTE: we only use this for From/To address resolution currently + // Cache transaction message decoding for From/To address resolution. message atomic.Value } @@ -61,7 +64,7 @@ func (r *Receipt) BlockNumber() *big.Int { if r.receipt != nil { return r.receipt.BlockNumber } else { - return nil + return r.blockNum } } @@ -69,7 +72,7 @@ func (r *Receipt) BlockHash() ethkit.Hash { if r.receipt != nil { return r.receipt.BlockHash } else { - return ethkit.Hash{} + return r.blockHash } } @@ -175,25 +178,22 @@ func (r *Receipt) To() common.Address { } func (r *Receipt) AsMessage() (*core.Message, error) { - msg, ok := r.message.Load().(*core.Message) - if !ok { - return nil, fmt.Errorf("ethreceipts: Receipt.message type-assertion fail, unexpected") + cached := r.message.Load() + if cached != nil { + msg, ok := cached.(*core.Message) + if !ok { + return nil, fmt.Errorf("ethreceipts: Receipt.message type-assertion fail, unexpected") + } + if msg != nil { + return msg, nil + } } - if msg != nil { - return msg, nil + if r.transaction == nil || r.chainID == nil { + return nil, fmt.Errorf("ethreceipts: transaction and chainID required to resolve message") } - // TODOXXX: avoid using AsMessage as its fairly expensive operation, especially - // to do it for every txn for every filter. - // TODO: in order to do this, we'll have to update ethrpc with a different - // implementation to just use raw types, aka, ethrpc/types.go with Block/Transaction/Receipt/Log .. txnMsg, err := ethtxn.AsMessage(r.transaction, r.chainID) if err != nil { - // NOTE: this should never happen, but lets log in case it does. In the - // future, we should just not use go-ethereum for these types. - // l.log.Warn(fmt.Sprintf("unexpected failure of txn (%s index %d) on block %d (total txns=%d) AsMessage(..): %s", - // txn.Hash(), i, block.NumberU64(), len(block.Transactions()), err, - // )) return nil, err } r.message.Store(txnMsg) diff --git a/ethreceipts/receipt_fetch_test.go b/ethreceipts/receipt_fetch_test.go new file mode 100644 index 00000000..59c9b634 --- /dev/null +++ b/ethreceipts/receipt_fetch_test.go @@ -0,0 +1,258 @@ +package ethreceipts + +import ( + "context" + "errors" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningQueryOnChainUsesCompleteReceipt(t *testing.T) { + tx, _, _ := hardeningTxn(t, 9) + b := hardeningBlock(90, tx) + complete := hardeningReceipt(b, tx) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + return nil, ethereum.NotFound + }} + l := hardeningListener(t, p, hardeningOptions()) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil })).(*subscriber) + defer s.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.Receipt() != complete || r.BlockHash() != b.Hash() { + t.Fatal("callback receipt replaced") + } + case <-time.After(100 * time.Millisecond): + t.Error("complete callback receipt was not delivered") + } + if calls.Load() != 0 { + t.Errorf("complete callback caused %d redundant RPCs", calls.Load()) + } +} + +func TestHardeningFetchPreservesSharedQuery(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }) + s := l.Subscribe(q) + defer s.Unsubscribe() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { defer wg.Done(); _, _, _ = l.FetchTransactionReceiptWithFilter(ctx, q) }() + } + wg.Wait() + opts := s.Filters()[0].Options() + if opts.LimitOne || opts.SearchCache { + t.Fatalf("Fetch rewrote active subscription: %+v", opts) + } +} + +// This builder becomes a Filterer only after the optional Finalize step. +type fetchFinalizeBuilder struct{ FilterQuery } + +func (b *fetchFinalizeBuilder) LimitOne(v bool) FilterQuery { + b.FilterQuery = b.FilterQuery.LimitOne(v) + return b +} + +func (b *fetchFinalizeBuilder) SearchCache(v bool) FilterQuery { + b.FilterQuery = b.FilterQuery.SearchCache(v) + return b +} + +func TestFetchFilterCompatibility(t *testing.T) { + for _, name := range []string{"builtin", "custom_pointer", "custom_value", "builder", "finalize_builder"} { + t.Run(name, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 3002) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), b) + var calls atomic.Int32 + match := func(context.Context, Receipt) (bool, error) { calls.Add(1); return true, nil } + base := FilterLogs(func([]*types.Log) bool { calls.Add(1); return true }).MaxWait(4).(Filterer) + var query FilterQuery = base + var public Filterer = base + builder := false + switch name { + case "custom_pointer", "custom_value": + // Only the custom Match accepts receipts, proving it survives the snapshot. + base = FilterLogs(func([]*types.Log) bool { return false }).MaxWait(4).(Filterer) + custom := ownershipValueFilter{base, []string{"fetch"}, match} + if name == "custom_pointer" { + public = &custom + } else { + public = custom + } + query = public + case "builder": + query = struct{ FilterQuery }{base} + builder = true + case "finalize_builder": + query = &fetchFinalizeBuilder{base} + builder = true + } + var other Subscription + if !builder { + other = l.Subscribe(public) + defer other.Unsubscribe() + } + hardeningStart(t, l) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + mined, waitFinal, err := l.FetchTransactionReceiptWithFilter(ctx, query, true) + if err != nil { + t.Fatal(err) + } + if mined == nil || mined.Final || mined.Reorged || mined.TransactionHash() != tx.Hash() || !sameFilter(mined.Filter, public) { + t.Fatalf("wrong mined receipt or public filter: %+v", mined) + } + if calls.Load() == 0 { + t.Fatal("source matching behavior was not used") + } + if !builder { + opts := public.Options() + if opts.LimitOne || opts.SearchCache || opts.Finalize || opts.MaxWait == nil || *opts.MaxWait != 4 { + t.Fatalf("fetch mutated shared filter options: %+v", opts) + } + } + if public.StartBlockNum() != 0 || public.LastMatchBlockNum() != 0 { + t.Fatal("fetch mutated public filter counters") + } + select { + case <-public.Exhausted(): + t.Fatal("fetch exhausted the public filter") + default: + } + l.mu.Lock() + var helper *subscriber + for _, s := range l.subscribers { + if s != other { + helper = s + } + } + l.mu.Unlock() + if helper == nil { + t.Fatal("fetch helper unsubscribed before finality") + } + if err := helper.finalizeReceipts(big.NewInt(102)); err != nil { + t.Fatal(err) + } + final, err := waitFinal(ctx) + if err != nil || final == nil || !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || !sameFilter(final.Filter, public) { + t.Fatalf("wrong final receipt or public filter: %+v, error: %v", final, err) + } + wantSubscribers := 0 + if other != nil { + wantSubscribers = 1 + } + if l.NumSubscribers() != wantSubscribers { + t.Error("fetch completion removed the wrong subscription") + } + }) + } +} + +type ownershipGateFilter struct { + Filterer + entered, release chan struct{} + calls atomic.Int32 +} + +func (f *ownershipGateFilter) Match(ctx context.Context, _ Receipt) (bool, error) { + if f.calls.Add(1) == 1 { + close(f.entered) + select { + case <-f.release: + case <-ctx.Done(): + return false, ctx.Err() + } + } + return false, nil +} + +func TestOwnershipStaleFetchPreservesCache(t *testing.T) { + for _, sameHash := range []bool{false, true} { + t.Run(map[bool]string{false: "remined_block", true: "readopted_generation"}[sameHash], func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 107) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + if sameHash { + canonical = old + } + complete := hardeningReceipt(canonical, tx) + var calls atomic.Int32 + var unavailable atomic.Bool + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if unavailable.Load() { + return nil, ethereum.NotFound + } + return complete, nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + gate := &ownershipGateFilter{Filterer: FilterLogs(func([]*types.Log) bool { return false }).(Filterer), entered: make(chan struct{}), release: make(chan struct{})} + s := l.Subscribe(q, gate).(*subscriber) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { + _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, blockHash: old.Hash(), blockNum: old.Number()}}) + done <- err + }() + <-gate.entered + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + hardeningRead(t, s) + if sameHash { + if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{canonical}, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + t.Fatal(err) + } + } else { + hardeningProcess(t, l, s, canonical) + } + hardeningRead(t, s) + cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Fatal("canonical receipt was not cached") + } + close(gate.release) + <-done + cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Error("late collection destroyed canonical cache") + } + unavailable.Store(true) + if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{old.Hash(), 0}); !errors.Is(err, ethereum.NotFound) { + t.Errorf("stale request error: %v", err) + } + cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Error("stale expected block/generation deleted current cache") + } + r, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || r != complete { + t.Errorf("fresh cache query depended on unavailable origin: %v", err) + } + if calls.Load() != 1 { + t.Errorf("stale work refetched a current canonical result: calls=%d", calls.Load()) + } + }) + } +} diff --git a/ethreceipts/reorg_test.go b/ethreceipts/reorg_test.go new file mode 100644 index 00000000..55183c97 --- /dev/null +++ b/ethreceipts/reorg_test.go @@ -0,0 +1,773 @@ +package ethreceipts + +import ( + "context" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningRollbackRemining(t *testing.T) { + tx, _, _ := hardeningTxn(t, 1) + old, newBlock := hardeningBlock(100, tx), hardeningBlock(102, tx) + var current atomic.Pointer[types.Receipt] + current.Store(hardeningReceipt(old, tx)) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { calls.Add(1); return current.Load(), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, old) + first := hardeningRead(t, s) + if first.Final || first.Reorged || first.BlockHash() != old.Hash() { + t.Fatalf("initial receipt: %+v", first) + } + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case rollback := <-s.TransactionReceipt(): + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.TransactionHash() != tx.Hash() { + t.Fatalf("rollback: %+v", rollback) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing rollback receipt") + } + if calls.Load() != 1 { + t.Errorf("rollback fetched orphan receipt: %d RPC calls", calls.Load()) + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + current.Store(hardeningReceipt(newBlock, tx)) + hardeningProcess(t, l, s, newBlock) + mined := hardeningRead(t, s) + if mined.BlockHash() != newBlock.Hash() || mined.Reorged || mined.Final { + t.Fatalf("re-mined receipt owns wrong block: %+v", mined) + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Reorged || final.BlockHash() != newBlock.Hash() || final.Filter != q { + t.Fatalf("canonical final receipt: %+v", final) + } +} + +func TestHardeningInflightRollbackRemining(t *testing.T) { + tx, _, _ := hardeningTxn(t, 11) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + select { + case <-release: + return hardeningReceipt(old, tx), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(canonical, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + snapshot := s.Filters() + oldDone := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}) + oldDone <- err + }() + <-entered + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case rollback := <-s.TransactionReceipt(): + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.Filter != q { + t.Errorf("wrong in-flight rollback: %+v", rollback) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing in-flight rollback") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + hardeningProcess(t, l, s, canonical) + select { + case mined := <-s.TransactionReceipt(): + if mined.BlockHash() != canonical.Hash() || mined.Reorged || mined.Final || mined.Filter != q { + t.Errorf("wrong canonical mined receipt: %+v", mined) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing canonical mined receipt") + } + close(release) + select { + case <-oldDone: + case <-time.After(time.Second): + t.Fatal("old fetch did not finish") + } + // A registration snapshot retained before rollback must also remain invalid. + if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + select { + case final := <-s.TransactionReceipt(): + if !final.Final || final.Reorged || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Errorf("wrong canonical final: %+v", final) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing canonical final") + } + cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || cached.BlockHash != canonical.Hash() { + t.Errorf("late orphan RPC poisoned receipt cache: receipt=%+v error=%v", cached, err) + } + if calls.Load() != 2 { + t.Errorf("unexpected fetch count: %d", calls.Load()) + } +} + +// This matcher deliberately needs the fetched receipt, which removed block data +// cannot supply. Rollback must use its previously delivered owner and receipt. +type hardeningStatusFilter struct{ Filterer } + +func (f *hardeningStatusFilter) Match(_ context.Context, r Receipt) (bool, error) { + return r.Status() == 1, nil +} + +func TestHardeningRollbackRetainsCustomOwner(t *testing.T) { + tx, _, _ := hardeningTxn(t, 14) + b := hardeningBlock(100, tx) + complete := hardeningReceipt(b, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return complete, nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := &hardeningStatusFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil }).(Filterer)} + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { + t.Fatal(err) + } + first := hardeningRead(t, s) + if first.Filter != q || first.Receipt() != complete { + t.Fatal("wrong custom filter owner or receipt") + } + removed := *b + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case r := <-s.TransactionReceipt(): + if !r.Reorged || r.Final || r.Filter != q || r.Receipt() != complete { + t.Errorf("rollback lost retained data/owner: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("custom complete-receipt matcher lost rollback") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} + +func TestHardeningRollbackInvalidatesPending(t *testing.T) { + tx, _, _ := hardeningTxn(t, 15) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + var failing atomic.Bool + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if failing.Load() { + return nil, errors.New("transient receipt error") + } + return hardeningReceipt(canonical, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, old) + s.retryMu.Lock() + pendingBefore := len(s.pendingReceipts) + s.retryMu.Unlock() + if pendingBefore != 1 { + t.Fatalf("expected one pending owner, got %d", pendingBefore) + } + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + s.retryMu.Lock() + pendingAfter := len(s.pendingReceipts) + s.retryMu.Unlock() + if pendingAfter != 0 { + t.Errorf("rollback retained %d orphan pending deliveries", pendingAfter) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Reorged || r.Filter != q || r.BlockHash() != old.Hash() { + t.Errorf("wrong pending rollback: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing pending rollback") + } + failing.Store(false) + hardeningProcess(t, l, s, canonical) + mined := hardeningRead(t, s) + if mined.BlockHash() != canonical.Hash() || mined.Reorged { + t.Fatal("retry ownership interfered with re-mining") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Fatal("wrong re-mined final owner") + } +} + +func TestHardeningSameHashReadoptionRejectsOldWork(t *testing.T) { + tx, _, _ := hardeningTxn(t, 19) + block := hardeningBlock(100, tx) + oldReceipt, canonicalReceipt := hardeningReceipt(block, tx), hardeningReceipt(block, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + select { + case <-release: + return oldReceipt, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return canonicalReceipt, nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + oldDone := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}) + oldDone <- err + }() + <-entered + removed := *block + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.BlockHash() != block.Hash() || rollback.Filter != q { + t.Fatalf("wrong rollback: %+v", rollback) + } + // A cache snapshot taken before removal cannot authoritatively re-adopt it. + hardeningProcess(t, l, s, block) + hardeningNoReceipt(t, s) + if calls.Load() != 1 { + t.Errorf("stale snapshot cleared removal marker: %d fetches", calls.Load()) + } + // The live block-processing entry point receives the canonical Added event. + if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { + t.Errorf("wrong readopted receipt: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("canonical same-hash re-adoption did not reopen delivery") + } + close(release) + <-oldDone + hardeningNoReceipt(t, s) + if calls.Load() != 2 { + t.Errorf("expected fresh readoption fetch, got %d", calls.Load()) + } + cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || cached != canonicalReceipt { + t.Errorf("old in-flight work replaced canonical cache: receipt=%p error=%v", cached, err) + } + // A callback with missing block data must acquire the fresh block generation + // when completing its receipt from cache, rather than inheriting generation zero. + partial := &types.Receipt{TxHash: tx.Hash()} + completion := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return partial, nil })).(*subscriber) + defer completion.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), completion, completion.Filters()); err != nil { + t.Fatal(err) + } + select { + case r := <-completion.TransactionReceipt(): + if r.Receipt() != canonicalReceipt || r.BlockHash() != block.Hash() { + t.Error("incomplete callback lost readopted receipt") + } + case <-time.After(100 * time.Millisecond): + t.Error("missing-data callback lost canonical generation") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { + t.Errorf("wrong readopted final: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing readopted final") + } + hardeningNoReceipt(t, s) +} + +func TestOwnershipLiveExhaustedRollbackReleasesOwner(t *testing.T) { + tx, _, _ := hardeningTxn(t, 110) + b := hardeningBlock(100, tx) + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }}, + canonical: make(map[uint64]*types.Block), + blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 100 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99)}); err != nil { + t.Fatal(err) + } + opts := hardeningOptions() + opts.NumBlocksToFinality = 10 + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true).MaxWait(1) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningStart(t, l) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor Run did not stop") + } + }) + wait := func(what string, condition func() bool) { + t.Helper() + deadline := time.Now().Add(12 * time.Second) + for !condition() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } + } + alternate := func(num, fork int64, parent common.Hash) *types.Block { + header := hardeningBlock(num).Header() + header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+fork)), parent + return types.NewBlockWithHeader(header) + } + p.advance(b.Block) + mined := hardeningRead(t, s) + if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Final || mined.Reorged { + t.Fatal("wrong initial mined owner") + } + wait("first live match", func() bool { return q.(Filterer).LastMatchBlockNum() == 100 }) + old101 := hardeningBlock(101) + p.advance(old101.Block) + wait("block 101", func() bool { return m.LatestBlockNum().Int64() == 101 }) + alt101 := alternate(101, 10000, b.Hash()) + alt102 := alternate(102, 10000, alt101.Hash()) + p.advance(alt101, alt102) + wait("live reorg counter reset", func() bool { + return q.(Filterer).StartBlockNum() == 102 && q.(Filterer).LastMatchBlockNum() == 0 + }) + alt103 := alternate(103, 10000, alt102.Hash()) + p.advance(alt103) + select { + case <-q.(Filterer).Exhausted(): + case <-time.After(time.Second): + t.Fatal("live MaxWait did not exhaust owner") + } + s.deliveryMu.Lock() + queued := len(s.finalizer.queue) == 1 && len(s.claims) == 1 + s.deliveryMu.Unlock() + if !queued || len(s.Filters()) != 0 { + t.Fatal("exhaustion lost queued finality ownership") + } + current := hardeningBlock(99).Block + for num := int64(100); num <= 104; num++ { + next := alternate(num, 20000, current.Hash()) + p.advance(next) + current = next + } + wait("rollback invalidation", func() bool { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + return m.LatestBlock().Hash() == current.Hash() && len(s.finalizer.queue) == 0 + }) + select { + case rollback := <-s.TransactionReceipt(): + if rollback.TransactionHash() != tx.Hash() || rollback.BlockHash() != b.Hash() || rollback.BlockNumber().Int64() != 100 || rollback.Filter != q || !rollback.Reorged || rollback.Final || rollback.Status() != 1 { + t.Error("wrong exhausted-owner rollback identity/state") + } + case <-time.After(100 * time.Millisecond): + t.Error("exhausted queued owner missed its delivered rollback") + } + checkRetired := func() { + t.Helper() + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.inFlight) != 0 || len(s.pendingReceipts) != 0 || len(s.finalizer.queue) != 0 || len(s.finalizer.txns) != 0 { + t.Errorf("inactive rollback owner retained state: claims=%d deliveries=%d", len(s.claims), len(s.deliveries)) + } + } + checkRetired() + for num := int64(105); num <= 112; num++ { + next := alternate(num, 20000, current.Hash()) + p.advance(next) + current = next + } + wait("finality advancement", func() bool { return m.LatestBlock().Hash() == current.Hash() }) + s.RemoveFilter(q.(Filterer)) + checkRetired() + hardeningNoReceipt(t, s) +} + +func TestOwnershipExhaustedRollbackKeepsOtherCandidates(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 111) + tx2, _, _ := hardeningTxn(t, 112) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + other := FilterTxnHash(tx2.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q, other).(*subscriber) + defer s.Unsubscribe() + for _, b := range []*ethmonitor.Block{b1, b2} { + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b, b.Transactions()[0])}}); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 3; i++ { + hardeningRead(t, s) + } + s.exhaustFilter(q.(Filterer)) + q.(*filter).closeExhausted() + removed := *b1 + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx1.Hash() || r.BlockHash() != b1.Hash() || r.Filter != q || !r.Reorged || r.Final { + t.Error("wrong exhausted retained rollback") + } + case <-time.After(100 * time.Millisecond): + t.Error("inactive owner missed rollback while another candidate remained") + } + if len(s.finalizer.queue) != 2 || len(s.Filters()) != 1 { + t.Fatal("rollback removed unrelated ownership") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + seen := make(map[Filterer]bool) + for i := 0; i < 2; i++ { + r := hardeningRead(t, s) + if r.TransactionHash() != tx2.Hash() || r.BlockHash() != b2.Hash() || !r.Final || r.Reorged || seen[r.Filter] { + t.Error("rollback lost or duplicated another owned final") + } + seen[r.Filter] = true + } + if !seen[q.(Filterer)] || !seen[other.(Filterer)] || len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.Filters()) != 0 { + t.Error("finished owners were not retired") + } + hardeningNoReceipt(t, s) +} + +func TestReceiptsFixRollbackPreservesMinedSelection(t *testing.T) { + tx, _, _ := hardeningTxn(t, 805) + other, _, _ := hardeningTxn(t, 806) + old, replacement, canonical := hardeningBlock(100, tx), hardeningBlock(101, other), hardeningBlock(102, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(old, tx)) + hardeningRead(t, s) + s.rollbackBlock(l.invalidateBlock(context.Background(), old)) + rollback := hardeningRead(t, s) + if s.claims[owner] != tx.Hash() || !rollback.Reorged || rollback.Filter != q || rollback.BlockHash() != old.Hash() { + t.Fatal("rollback released an already-mined selection") + } + process(hardeningReceipt(replacement, other)) + hardeningNoReceipt(t, s) + process(hardeningReceipt(canonical, tx)) + mined := hardeningRead(t, s) + if mined.Reorged || mined.Final || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != canonical.Hash() || mined.Filter != q { + t.Fatal("retained selection did not re-mine correctly") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Fatal("retained mined selection finalized an orphan") + } + hardeningNoReceipt(t, s) +} + +func TestReceiptsFixReadoptionAfterRetention(t *testing.T) { + for _, scenario := range []string{"canonical_retention", "obsolete_incarnation", "later_removal", "alternate_retention"} { + t.Run(scenario, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 807) + b := hardeningBlock(100, tx) + old, canonical := hardeningReceipt(b, tx), hardeningReceipt(b, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + var releaseOnce sync.Once + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + // Deliberately finish an old RPC after its request timeout, + // to verify invalidation of late provider completions. + <-release + return old, nil + } + return canonical, nil + }}, + canonical: make(map[uint64]*types.Block), blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 50 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99), b}); err != nil { + t.Fatal(err) + } + initialSnapshot := *m.LatestBlock() + opts := hardeningOptions() + opts.NumBlocksToFinality = 200 + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + stream := m.Subscribe("delayed receipt events") + defer stream.Unsubscribe() + oldDone := make(chan error, 1) + go func() { + _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{b.Hash(), 0}) + oldDone <- err + }() + <-entered + releaseOld := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(releaseOld) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor did not stop") + } + }) + consume := func() ethmonitor.Blocks { + t.Helper() + select { + case batch := <-stream.Blocks(): + return batch + case <-time.After(10 * time.Second): + t.Fatal("missing canonical monitor event") + return nil + } + } + process := func(batch ethmonitor.Blocks, cached bool) { + t.Helper() + var err error + if cached { + _, err = l.processCachedBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + } else { + _, err = l.processBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + } + if err != nil { + t.Fatal(err) + } + } + fork := func(from, through, offset int64, parent common.Hash) { + for num := from; num <= through; num++ { + header := hardeningBlock(num).Header() + header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+offset)), parent + next := types.NewBlockWithHeader(header) + p.advance(next) + parent = next.Hash() + } + } + waitHeight := func(height int64) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for m.LatestBlockNum().Int64() != height { + if time.Now().After(deadline) { + t.Fatal("monitor did not advance canonical height") + } + time.Sleep(time.Millisecond) + } + } + // The first real reorg removes H, invalidating the held generation-0 RPC. + fork(100, 101, 10000, hardeningBlock(99).Hash()) + process(consume(), false) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != b.Hash() || rollback.TransactionHash() != tx.Hash() || rollback.Filter != q { + t.Fatal("wrong original removal identity") + } + process(ethmonitor.Blocks{&initialSnapshot}, true) + hardeningNoReceipt(t, s) + // Rebuild the same hash H in a fresh, owned monitor incarnation. + p.advance(b.Block, hardeningBlock(101).Block, hardeningBlock(102).Block) + readoption := consume() + var added *ethmonitor.Block + for _, event := range readoption { + if event.Event == ethmonitor.Added && event.Hash() == b.Hash() { + added = event + } + } + if added == nil { + t.Fatal("monitor omitted readopted incarnation") + } + incarnation, valid := added.CanonicalState() + if incarnation == 0 || !valid { + t.Fatal("Added input was not the owned canonical monitor event") + } + process(ethmonitor.Blocks{added}, true) + hardeningNoReceipt(t, s) + negative := scenario == "later_removal" || scenario == "alternate_retention" + var removed *ethmonitor.Block + if negative { + fork(100, 103, 20000, hardeningBlock(99).Hash()) + for _, event := range consume() { + if event.Event == ethmonitor.Removed && event.Hash() == b.Hash() { + removed = event + } + } + if removed == nil { + t.Fatal("monitor omitted later removal") + } + } + if scenario == "canonical_retention" { + fork(103, 155, 0, hardeningBlock(102).Hash()) + waitHeight(155) + } else if scenario == "alternate_retention" { + fork(104, 155, 20000, common.BigToHash(big.NewInt(20103))) + waitHeight(155) + } + if scenario == "canonical_retention" || scenario == "alternate_retention" { + if m.GetBlock(b.Hash()) != nil || m.OldestBlockNum().Int64() != 106 { + t.Fatal("H was not evicted more than one block behind retention") + } + } + if scenario == "obsolete_incarnation" { + // A removed tracked snapshot cannot fall back to a newer retained + // incarnation of the same hash, even during authoritative processing. + if m.GetBlock(b.Hash()) == nil { + t.Fatal("newer same-hash incarnation is missing") + } + process(ethmonitor.Blocks{&initialSnapshot}, false) + hardeningNoReceipt(t, s) + if !l.blockStates[b.Hash()].removed { + t.Error("obsolete tracked incarnation used same-hash fallback") + } + } + process(readoption, false) + if negative { + hardeningNoReceipt(t, s) + if !l.blockStates[b.Hash()].removed { + t.Error("later-removed Added reopened canonical marker") + } + process(ethmonitor.Blocks{added, removed}, false) + for _, r := range receiptsFixCollect(t, s) { + if !r.Reorged || r.Final { + t.Error("same-batch removal published stale mined/final receipt") + } + } + if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false); !errors.Is(err, ethereum.NotFound) { + t.Errorf("later removal reopened hash lookup: %v", err) + } + } else { + select { + case mined := <-s.TransactionReceipt(): + if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Reorged || mined.Final || mined.generation != 1 { + t.Error("readopted incarnation lost exact mined ownership") + } + case <-time.After(100 * time.Millisecond): + t.Error("canonical readoption after retention was rejected") + } + result, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || result != canonical { + t.Errorf("canonical readoption hash lookup failed: %v", err) + } + } + releaseOld() + if err := <-oldDone; !errors.Is(err, ethereum.NotFound) { + t.Errorf("old generation RPC escaped invalidation: %v", err) + } + if !negative { + cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != canonical { + t.Error("old RPC replaced fresh canonical cache") + } + } + hardeningNoReceipt(t, s) + }) + } +} diff --git a/ethreceipts/stale_receipt_test.go b/ethreceipts/stale_receipt_test.go new file mode 100644 index 00000000..9fea2901 --- /dev/null +++ b/ethreceipts/stale_receipt_test.go @@ -0,0 +1,170 @@ +package ethreceipts + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestReceiptsStaleRPCRecoversCurrentCandidate(t *testing.T) { + for _, pendingFirst := range []bool{false, true} { + name := "initial_response" + if pendingFirst { + name = "pending_retry" + } + t.Run(name, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6000) + a, b := hardeningBlock(100, tx), hardeningBlock(102, tx) + var current atomic.Pointer[types.Receipt] + current.Store(hardeningReceipt(a, tx)) + var providerError atomic.Bool + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if providerError.Load() { + return nil, errors.New("temporary provider failure") + } + return current.Load(), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + ctx := context.Background() + hardeningProcess(t, l, s, a) + mined := hardeningRead(t, s) + if mined.BlockHash() != a.Hash() || mined.Reorged || mined.Final { + t.Fatalf("wrong initial mined receipt: %+v", mined) + } + removed := *a + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.BlockHash() != a.Hash() { + t.Fatalf("wrong rollback notification: %+v", rollback) + } + providerError.Store(pendingFirst) + hardeningProcess(t, l, s, b) + providerError.Store(false) + pendingCount := func() int { + s.retryMu.Lock() + defer s.retryMu.Unlock() + return len(s.pendingReceipts) + } + makeDue := func() { + s.retryMu.Lock() + defer s.retryMu.Unlock() + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Now().Add(-time.Second) + } + } + if pendingFirst { + if pendingCount() != 1 { + t.Fatal("provider failure did not queue B") + } + makeDue() + hardeningProcess(t, l, s, hardeningBlock(103)) + } + if pendingCount() != 1 { + t.Errorf("stale A response discarded current B candidate: pending=%d", pendingCount()) + } + if _, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); found { + t.Fatal("stale A receipt was cached") + } + hardeningNoReceipt(t, s) + + // The provider recovers; only an unrelated later block triggers retry. + canonical := hardeningReceipt(b, tx) + current.Store(canonical) + makeDue() + callsBefore := calls.Load() + hardeningProcess(t, l, s, hardeningBlock(104)) + if calls.Load() <= callsBefore { + t.Fatal("provider recovered but B was never retried") + } + mined = hardeningRead(t, s) + if mined.BlockHash() != b.Hash() || mined.TransactionHash() != tx.Hash() || mined.Reorged || mined.Final || mined.Filter != q { + t.Fatalf("wrong recovered mined receipt: %+v", mined) + } + hardeningProcess(t, l, s, hardeningBlock(105)) + final := hardeningRead(t, s) + if final.BlockHash() != b.Hash() || final.TransactionHash() != tx.Hash() || final.Reorged || !final.Final || final.Filter != q { + t.Fatalf("wrong recovered final receipt: %+v", final) + } + if pendingCount() != 0 { + t.Error("delivered B remained pending") + } + if cached, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); !found || cached != canonical { + t.Error("canonical B receipt was not cached") + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestReceiptsStaleRPCKeepsObsoleteWorkTerminal(t *testing.T) { + t.Run("expected_generation_removed_during_fetch", func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6001) + b := hardeningBlock(100, tx) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + close(entered) + select { + case <-release: + return hardeningReceipt(b, tx), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + }} + l := hardeningListener(t, p, hardeningOptions(), b) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := l.fetchTransactionReceipt(ctx, tx.Hash(), true, blockRef{b.Hash(), 0}) + done <- err + }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("RPC did not start") + } + l.invalidateBlock(ctx, b) + l.acceptBlock(b) + if !l.isCurrentBlock(b.Hash(), 1) { + t.Fatal("same-hash re-adoption did not advance the generation") + } + fresh := hardeningReceipt(b, tx) + l.pastReceipts.Set(ctx, tx.Hash().Hex(), fresh) + close(release) + if err := <-done; !errors.Is(err, ethereum.NotFound) { + t.Fatalf("obsolete expected generation became retryable: %v", err) + } + if cached, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); !found || cached != fresh { + t.Error("obsolete completion replaced the current cache entry") + } + }) + t.Run("hash_query_without_current_candidate", func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6002) + a := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(a, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), a) + ctx := context.Background() + l.invalidateBlock(ctx, a) + if _, err := l.fetchTransactionReceipt(ctx, tx.Hash(), true); !errors.Is(err, ethereum.NotFound) { + t.Fatalf("unscoped orphan query became retryable: %v", err) + } + if _, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); found { + t.Error("unscoped orphan query cached stale data") + } + }) +} diff --git a/ethreceipts/subscription.go b/ethreceipts/subscription.go index 0d59f0a5..ed55b382 100644 --- a/ethreceipts/subscription.go +++ b/ethreceipts/subscription.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/big" + "reflect" "sync" "time" @@ -33,10 +34,17 @@ var ( type Subscription interface { TransactionReceipt() <-chan Receipt Done() <-chan struct{} + // Unsubscribe closes the subscription. Repeated and concurrent calls are safe. Unsubscribe() Filters() []Filterer AddFilter(filters ...FilterQuery) + // RemoveFilter removes the first matching active registration, or its queued + // finality owner if inactive. Comparable filters retain Go equality. Otherwise + // callback-bearing values of the same type use a stable nonnil Exhausted signal + // as base identity. Values sharing a base are aliases; distinct callbacks need + // distinct bases or pointer identities for independent removal. Values without + // a stable signal should be registered as pointers for individual removal. RemoveFilter(filter Filterer) ClearFilters() } @@ -48,17 +56,27 @@ type subscriber struct { ch channel.Channel[Receipt] done chan struct{} unsubscribe func() - filters []Filterer + filters []*filterOwner // Protected by mu. finalizer *finalizer mu sync.Mutex - pendingReceipts map[common.Hash]*pendingReceipt + // retryMu protects pendingReceipts and the mutable fields of each pendingReceipt. + pendingReceipts map[receiptKey]*pendingReceipt retryMu sync.Mutex + + // deliveryMu serializes publication, rollback, finality and owner retirement, + // and protects deliveries, inFlight and claims. When both locks are needed, + // acquire deliveryMu before retryMu. RPC calls run outside these locks. + deliveryMu sync.Mutex + deliveries map[receiptKey]Receipt + inFlight map[receiptKey]struct{} + claims map[*filterOwner]common.Hash } type pendingReceipt struct { receipt Receipt - filterer Filterer + completed *Receipt // Learned candidate; receipt retains the original queue key. + owner *filterOwner attempts int nextRetryAt time.Time } @@ -80,225 +98,515 @@ func (s *subscriber) Unsubscribe() { s.unsubscribe() } +// Every registration owns a comparable token; the public filter may contain +// slices, maps or functions and may be reused in a later registration. +type filterOwner struct{ Filterer } + +func sameFilter(a, b Filterer) bool { + if reflect.ValueOf(a).Comparable() && reflect.ValueOf(b).Comparable() { + return a == b + } + if reflect.DeepEqual(a, b) { + return true + } + if reflect.TypeOf(a) != reflect.TypeOf(b) { + return false + } + identity := a.Exhausted() + return identity != nil && identity == b.Exhausted() +} + func (s *subscriber) Filters() []Filterer { s.mu.Lock() defer s.mu.Unlock() filters := make([]Filterer, len(s.filters)) - copy(filters, s.filters) + for i, owner := range s.filters { + filters[i] = owner.Filterer + } return filters } -func (s *subscriber) AddFilter(filterQueries ...FilterQuery) { - if len(filterQueries) == 0 { - return +func (s *subscriber) filterers() []Filterer { + s.mu.Lock() + defer s.mu.Unlock() + filters := make([]Filterer, len(s.filters)) + for i, owner := range s.filters { + filters[i] = owner } + return filters +} - filters := make([]Filterer, len(filterQueries)) - for i, query := range filterQueries { +func (s *subscriber) AddFilter(queries ...FilterQuery) { + if len(queries) == 0 { + return + } + owners := make([]*filterOwner, len(queries)) + filters := make([]Filterer, len(queries)) + for i, query := range queries { filterer, ok := query.(Filterer) if !ok { panic("ethreceipts: unexpected") } - filters[i] = filterer + owners[i] = &filterOwner{filterer} + filters[i] = owners[i] } - s.mu.Lock() - if len(s.filters)+len(filters) > maxFiltersPerListener { - // too many filters, ignore the extra filter. not ideal, but better than - // deadlocking - s.listener.log.Warn(fmt.Sprintf("ethreceipts: subscriber has too many filters (%d), ignoring extra", len(s.filters)+len(filters))) - // TODO: maybe return an error or force-unsubscribe instead? + if len(s.filters)+len(owners) > maxFiltersPerListener { + s.listener.log.Warn(fmt.Sprintf("ethreceipts: subscriber has too many filters (%d), ignoring extra", len(s.filters)+len(owners))) s.mu.Unlock() return } - s.filters = append(s.filters, filters...) + s.filters = append(s.filters, owners...) s.mu.Unlock() - - // TODO: maybe add non-blocking push structure like in relayer queue select { case s.listener.registerFiltersCh <- registerFilters{subscriber: s, filters: filters}: - // ok default: s.listener.log.Warn("ethreceipts: listener registerFiltersCh full, dropping filter register") } } -func (s *subscriber) RemoveFilter(filter Filterer) { +func (s *subscriber) owner(filter Filterer) *filterOwner { + if owner, ok := filter.(*filterOwner); ok { + return owner + } s.mu.Lock() defer s.mu.Unlock() + for _, owner := range s.filters { + if sameFilter(owner.Filterer, filter) { + return owner + } + } + return nil +} + +func (s *subscriber) RemoveFilter(filter Filterer) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + owner := s.owner(filter) + if owner == nil { + owner = s.finalizer.findOwner(filter) + } + if owner != nil { + s.retireOwner(owner, true) + } +} +func (s *subscriber) removeActive(owner *filterOwner) { + s.mu.Lock() + defer s.mu.Unlock() for i, f := range s.filters { - if f == filter { - s.filters = append(s.filters[:i], s.filters[i+1:]...) + if f == owner { + copy(s.filters[i:], s.filters[i+1:]) + s.filters[len(s.filters)-1] = nil + s.filters = s.filters[:len(s.filters)-1] return } } } +// Caller holds deliveryMu. Exhaustion retains queued finals, while explicit +// cancellation and automatic completion release the complete owner lifetime. +func (s *subscriber) retireOwner(owner *filterOwner, cancelFinality bool) { + s.removeActive(owner) + if cancelFinality { + s.finalizer.invalidateOwner(owner) + } + if cancelFinality || !s.finalizer.hasOwner(owner) { + delete(s.claims, owner) + for key := range s.deliveries { + if key.owner == owner { + delete(s.deliveries, key) + } + } + } + for key := range s.inFlight { + if key.owner == owner { + delete(s.inFlight, key) + } + } + s.retryMu.Lock() + for key := range s.pendingReceipts { + if key.owner == owner { + delete(s.pendingReceipts, key) + } + } + s.retryMu.Unlock() +} + +func (s *subscriber) exhaustFilter(filter Filterer) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if owner := s.owner(filter); owner != nil { + s.retireOwner(owner, false) + } +} + func (s *subscriber) ClearFilters() { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + s.finalizer.clear() + clear(s.claims) + clear(s.inFlight) + clear(s.deliveries) + s.retryMu.Lock() + clear(s.pendingReceipts) + s.retryMu.Unlock() +} + +// Reserve matches before fetching so concurrent registration and live processing +// cannot select different transactions for one LimitOne filter. +func (s *subscriber) reserve(receipt Receipt, owner *filterOwner) bool { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if !s.hasFilter(owner) || !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + return false + } + if s.claims == nil { + s.claims = make(map[*filterOwner]common.Hash) + } + if owner.Options().LimitOne { + if txn, claimed := s.claims[owner]; claimed && txn != receipt.TransactionHash() { + return false + } + s.claims[owner] = receipt.TransactionHash() + } + key := receiptOwner(receipt, owner) + if _, exists := s.deliveries[key]; exists { + return false + } + if s.inFlight == nil { + s.inFlight = make(map[receiptKey]struct{}) + } + if _, exists := s.inFlight[key]; exists { + return false + } + s.retryMu.Lock() + _, pending := s.pendingReceipts[key] + s.retryMu.Unlock() + if pending { + return false + } + s.inFlight[key] = struct{}{} + return true +} + +func (s *subscriber) releaseReservation(receipt Receipt, owner *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + delete(s.inFlight, receiptOwner(receipt, owner)) + s.releaseClaim(owner) +} + +// Caller holds deliveryMu. Keep selection only while real owner work remains. +func (s *subscriber) releaseClaim(owner *filterOwner) { + if s.finalizer.hasOwner(owner) { + return + } + for key := range s.inFlight { + if key.owner == owner { + return + } + } + for key := range s.deliveries { + if key.owner == owner { + return + } + } + s.retryMu.Lock() + defer s.retryMu.Unlock() + for key := range s.pendingReceipts { + if key.owner == owner { + return + } + } + delete(s.claims, owner) +} + +func (s *subscriber) hasFilter(owner *filterOwner) bool { s.mu.Lock() defer s.mu.Unlock() - s.filters = s.filters[:0] + for _, f := range s.filters { + if f == owner { + return true + } + } + return false } -// matchFiltersAndPublish matches the given receipts against the provided filterers, -// fetches any missing receipt data as needed, and notifies the subscriber of matches. -func (s *subscriber) matchFiltersAndPublish(ctx context.Context, filterers []Filterer, receipts []Receipt) ([]bool, error) { - oks := make([]bool, len(filterers)) +// RPC waits happen outside deliveryMu. Rollback, delivery and finalization share +// this lock so an invalidated in-flight receipt cannot be published afterwards. +func (s *subscriber) publish(ctx context.Context, receipt Receipt, owner *filterOwner) bool { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + return s.publishLocked(ctx, receipt, owner) +} + +// Caller holds deliveryMu so retry publication and pending ownership commit +// together with Remove/Clear and rollback invalidation. +func (s *subscriber) publishLocked(ctx context.Context, receipt Receipt, owner *filterOwner) bool { + if ctx.Err() != nil || !s.hasFilter(owner) { + return false + } + s.listener.receiptMu.Lock() + defer s.listener.receiptMu.Unlock() + if ctx.Err() != nil || !s.listener.currentBlock(receipt.BlockHash(), receipt.generation) { + return false + } + key := receiptOwner(receipt, owner) + if _, delivered := s.deliveries[key]; delivered { + return true + } + if owner.Options().LimitOne { + if txn, claimed := s.claims[owner]; claimed && txn != receipt.TransactionHash() { + return false + } + } + receipt.Filter = owner.Filterer + receipt.owner = owner + receipt.Final = s.listener.isBlockFinal(receipt.BlockNumber()) + if !receipt.Final && owner.Options().Finalize { + s.finalizer.enqueue(owner, receipt, receipt.BlockNumber()) + } + if s.deliveries == nil { + s.deliveries = make(map[receiptKey]Receipt) + } + s.deliveries[key] = receipt + s.ch.Send(receipt) + if owner.Options().LimitOne && (!owner.Options().Finalize || receipt.Final) { + s.retireOwner(owner, true) + } + return true +} + +func (s *subscriber) rollbackBlock(block blockRef) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + for _, receipt := range s.finalizer.invalidateBlock(block) { + // Queued finality still owns the delivered receipt after exhaustion, + // including when finalized-block history has already been pruned. + key := receiptOwner(receipt, receipt.owner) + if _, retained := s.deliveries[key]; !retained { + if s.deliveries == nil { + s.deliveries = make(map[receiptKey]Receipt) + } + s.deliveries[key] = receipt + } + } + s.retryMu.Lock() + invalidatedPending := make(map[*filterOwner]struct{}) + for key, pending := range s.pendingReceipts { + candidate := pending.receipt + if pending.completed != nil { + candidate = *pending.completed + } + if candidate.BlockHash() == block.hash && candidate.generation == block.generation { + invalidatedPending[key.owner] = struct{}{} + delete(s.pendingReceipts, key) + } + } + s.retryMu.Unlock() + // A custom matcher may require fields that lightweight block receipts lack. + // Previously published owners already matched, so use their retained data. + inactive := make(map[*filterOwner]struct{}) + for key, receipt := range s.deliveries { + if key.blockHash != block.hash || key.generation != block.generation { + continue + } + if !receipt.Reorged { + receipt.Final = false + receipt.Reorged = true + s.deliveries[key] = receipt + s.ch.Send(receipt) + } + if !s.hasFilter(key.owner) { + inactive[key.owner] = struct{}{} + } + } + // Notify every delivered receipt before retiring an exhausted lifetime. + for owner := range inactive { + if !s.finalizer.hasOwner(owner) { + s.retireOwner(owner, false) + } + } + for owner := range invalidatedPending { + s.releaseClaim(owner) + } +} - // Collect matches that need receipt fetching - type matchedReceipt struct { - receipt Receipt - filtererIdx int - filterer Filterer +func (s *subscriber) rollback(receipt Receipt, owner *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + key := receiptOwner(receipt, owner) + if !s.hasFilter(owner) { + return + } + if txn, claimed := s.claims[owner]; owner.Options().LimitOne && claimed && txn != receipt.TransactionHash() { + return + } + if retained, ok := s.deliveries[key]; ok { + if retained.Reorged { + return + } + receipt = retained + } + receipt.Filter = owner.Filterer + receipt.owner = owner + receipt.Final = false + receipt.Reorged = true + if s.deliveries == nil { + s.deliveries = make(map[receiptKey]Receipt) } - var toFetch []matchedReceipt + s.deliveries[key] = receipt + s.ch.Send(receipt) +} - // First pass: find all matches +func (s *subscriber) fetchReceipt(ctx context.Context, receipt Receipt) (Receipt, error) { + l := s.listener + l.receiptMu.Lock() + started := l.reorgRevision + l.receiptMu.Unlock() + expected := blockRef{receipt.BlockHash(), receipt.generation} + r, err := l.fetchTransactionReceipt(ctx, receipt.TransactionHash(), true, expected) + if err != nil { + return receipt, err + } + if expected.hash == (common.Hash{}) { + l.receiptMu.Lock() + valid := l.validFetchedBlock(r.BlockHash, blockRef{}, started) + receipt.generation = l.blockStates[r.BlockHash].generation + l.receiptMu.Unlock() + if !valid { + return receipt, ethereum.NotFound + } + } + receipt.receipt = r + receipt.logs = r.Logs + return receipt, nil +} + +func (s *subscriber) matchFiltersAndPublish(ctx context.Context, filterers []Filterer, receipts []Receipt) ([]bool, error) { + oks := make([]bool, len(filterers)) + type match struct { + receipt Receipt + owner *filterOwner + } + var matches []match + owners := make([]*filterOwner, len(filterers)) + for i, filter := range filterers { + owners[i] = s.owner(filter) + } for _, receipt := range receipts { - for i, filterer := range filterers { - matched, err := filterer.Match(ctx, receipt) + for i, owner := range owners { + if ctx.Err() != nil { + return oks, ctx.Err() + } + if owner == nil || !s.hasFilter(owner) { + continue + } + matched, err := owner.Match(ctx, receipt) if err != nil { return oks, superr.New(ErrFilterMatch, err) } - if !matched { - // skip, not a match continue } - - // its a match oks[i] = true - - if !receipt.Reorged { - toFetch = append(toFetch, matchedReceipt{ - receipt: receipt, - filtererIdx: i, - filterer: filterer, - }) + if receipt.Reorged { + s.rollback(receipt, owner) + continue } + matches = append(matches, match{receipt, owner}) } } - - if len(toFetch) == 0 { - return oks, nil - } - - // Fetch receipts concurrently sem := make(chan struct{}, maxConcurrentReceiptFetches) - g, gctx := errgroup.WithContext(ctx) - - for _, item := range toFetch { - item := item // capture loop variable + // One owner's fetch failure must not cancel another owner's delivery or retry. + var g errgroup.Group + for _, item := range matches { + if ctx.Err() != nil { + break + } + if !s.reserve(item.receipt, item.owner) { + continue + } g.Go(func() error { + defer s.releaseReservation(item.receipt, item.owner) select { case sem <- struct{}{}: defer func() { <-sem }() - case <-gctx.Done(): - return gctx.Err() + case <-ctx.Done(): + return ctx.Err() } - - // Fetch transaction receipt - r, err := s.listener.fetchTransactionReceipt(gctx, item.receipt.TransactionHash(), true) - if err != nil { - if errors.Is(err, ethereum.NotFound) { - // not found, don't retry - return superr.Wrap(fmt.Errorf("txn %s not found", item.receipt.TransactionHash()), err) + receipt := item.receipt + if receipt.receipt == nil || receipt.receipt.BlockNumber == nil || receipt.receipt.TxHash == (common.Hash{}) || receipt.receipt.BlockHash == (common.Hash{}) { + completed, err := s.fetchReceipt(ctx, receipt) + if err != nil { + if !errors.Is(err, ethereum.NotFound) && ctx.Err() == nil { + s.addPendingReceipt(receipt, item.owner) + } + return superr.Wrap(fmt.Errorf("failed to fetch txn %s receipt", receipt.TransactionHash()), err) } - - // might be a provider issue, add to pending receipts for retry - s.addPendingReceipt(item.receipt, item.filterer) - return superr.Wrap(fmt.Errorf("failed to fetch txn %s receipt due to node issue", item.receipt.TransactionHash()), err) - } - - // Update receipt with fetched data - item.receipt.receipt = r - item.receipt.logs = r.Logs - item.receipt.Filter = item.filterer - - // Finality enqueue if filter asked to Finalize, and receipt isn't already final - if !item.receipt.Final && item.filterer.Options().Finalize { - s.finalizer.enqueue(item.filterer.FilterID(), item.receipt, item.receipt.BlockNumber()) - } - - // LimitOne will auto unsubscribe now if were not also waiting for finalizer, - // and if the returned txn isn't one that has been reorged - // - // NOTE: when Finalize is set, we don't want to remove this filter until the txn finalizes, - // because its possible that it can reorg and we have to fetch it again after being re-mined. - // So we only remove the filter now if the filter finalizer isn't used, otherwise the - // finalizer will remove the LimitOne filter - toFinalize := item.filterer.Options().Finalize && !item.receipt.Final - if item.filterer.Options().LimitOne && !toFinalize { - s.RemoveFilter(item.receipt.Filter) - } - - // Check if receipt is already final, in case comes from cache when - // previously final was not toggled. - if s.listener.isBlockFinal(item.receipt.BlockNumber()) { - item.receipt.Final = true + receipt = completed } - - // Broadcast to subscribers (needs mutex as multiple goroutines may send) - s.ch.Send(item.receipt) - + s.publish(ctx, receipt, item.owner) return nil }) } - - // Wait for all fetches to complete - if err := g.Wait(); err != nil { - return oks, err + err := g.Wait() + if err == nil && ctx.Err() != nil { + err = ctx.Err() } - - return oks, nil + return oks, err } func (s *subscriber) finalizeReceipts(blockNum *big.Int) error { - // check subscriber finalizer - finalizer := s.finalizer - if finalizer.len() == 0 { - return nil - } - - finalTxns := finalizer.dequeue(blockNum) - if len(finalTxns) == 0 { - // no matching txns which have been finalized - return nil - } - - // dispatch to subscriber finalized receipts - for _, x := range finalTxns { - if x.receipt.Reorged { - // for removed receipts, just skip - continue + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + for _, txn := range s.finalizer.dequeue(blockNum) { + receipt := txn.receipt + owner := receipt.owner + s.listener.receiptMu.Lock() + if !receipt.Reorged && s.listener.currentBlock(receipt.BlockHash(), receipt.generation) { + receipt.Final = true + s.ch.Send(receipt) + if owner != nil && (owner.Cond().TxnHash != nil || owner.Options().LimitOne) { + s.retireOwner(owner, true) + } } - - // mark receipt as final, and send the receipt payload to the subscriber - x.receipt.Final = true - - // send to the subscriber - s.ch.Send(x.receipt) - - // Automatically remove filters for finalized txn hashes, as they won't come up again. - filter := x.receipt.Filter - if filter != nil && (filter.Cond().TxnHash != nil || filter.Options().LimitOne) { - s.RemoveFilter(filter) + s.listener.receiptMu.Unlock() + if !s.hasFilter(owner) && !s.finalizer.hasOwner(owner) { + s.retireOwner(owner, false) + } + } + // Retain mined data until its block is final, for receipt-free rollback. + for key, receipt := range s.deliveries { + if receipt.BlockNumber() != nil && s.listener.isBlockFinal(receipt.BlockNumber()) { + delete(s.deliveries, key) } } - return nil } -func (s *subscriber) addPendingReceipt(receipt Receipt, filterer Filterer) { +func (s *subscriber) addPendingReceipt(receipt Receipt, owner *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if !s.hasFilter(owner) { + return + } s.retryMu.Lock() defer s.retryMu.Unlock() txnHash := receipt.TransactionHash() + key := receiptOwner(receipt, owner) + if !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + return + } if s.pendingReceipts == nil { // lazy init - s.pendingReceipts = make(map[common.Hash]*pendingReceipt) + s.pendingReceipts = make(map[receiptKey]*pendingReceipt) } if len(s.pendingReceipts) >= maxPendingReceipts { @@ -310,14 +618,14 @@ func (s *subscriber) addPendingReceipt(receipt Receipt, filterer Filterer) { return } - if _, exists := s.pendingReceipts[txnHash]; exists { + if _, exists := s.pendingReceipts[key]; exists { // already pending, skip return } - s.pendingReceipts[txnHash] = &pendingReceipt{ + s.pendingReceipts[key] = &pendingReceipt{ receipt: receipt, - filterer: filterer, + owner: owner, attempts: 1, nextRetryAt: time.Now().Add(1 * time.Second), // first retry after 1s } @@ -359,6 +667,7 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { wg.Add(1) go func(p *pendingReceipt) { defer wg.Done() + defer s.releaseReservation(p.receipt, p.owner) select { case sem <- struct{}{}: @@ -366,100 +675,114 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { case <-ctx.Done(): // If context is cancelled, release the claim so the item can be retried later. s.retryMu.Lock() - if current, ok := s.pendingReceipts[p.receipt.TransactionHash()]; ok && current == p { + if current, ok := s.pendingReceipts[receiptOwner(p.receipt, p.owner)]; ok && current == p { current.nextRetryAt = time.Now().Add(100 * time.Millisecond) // small delay to avoid immediate retry } s.retryMu.Unlock() return } - // Attempt to fetch the receipt - txnHash := p.receipt.TransactionHash() - r, err := s.listener.fetchTransactionReceipt(ctx, txnHash, true) - - s.retryMu.Lock() - defer s.retryMu.Unlock() - - // Check if the item still exists and is the same one we claimed. - currentPending, exists := s.pendingReceipts[txnHash] - if !exists || currentPending != p { - s.listener.log.Debug("Pending receipt is stale or already processed, skipping retry", "txnHash", txnHash.String()) - return - } + s.retryPendingReceipt(ctx, p) + }(pending) + } - if err != nil { - if errors.Is(err, ethereum.NotFound) { - // Transaction genuinely doesn't exist - remove from queue - delete(s.pendingReceipts, txnHash) - s.listener.log.Debug("Receipt not found after retry, removing from queue", "txnHash", txnHash.String()) - return - } + wg.Wait() +} - // Provider error - update retry state directly on the pointer. - currentPending.attempts++ - if currentPending.attempts >= maxReceiptRetryAttempts { - delete(s.pendingReceipts, txnHash) - s.listener.log.Error( - "Failed to fetch receipt after max retries", - "txnHash", txnHash.String(), - "attempts", currentPending.attempts, - "error", err, - ) - // TODO: perhaps we should close the subscription here as we failed - // to deliver a receipt after many attempts? - return - } +// retryPendingReceipt fetches and commits one claimed pending receipt. The caller +// holds its semaphore slot and handles reservation cleanup. +func (s *subscriber) retryPendingReceipt(ctx context.Context, p *pendingReceipt) { + // Attempt to fetch the receipt + txnHash := p.receipt.TransactionHash() + key := receiptOwner(p.receipt, p.owner) + s.retryMu.Lock() + candidate := p.receipt + if p.completed != nil { + candidate = *p.completed + } + s.retryMu.Unlock() + receipt, err := s.fetchReceipt(ctx, candidate) - // Exponential backoff for next retry - backoff := time.Duration(1< maxWaitBetweenRetries { - backoff = maxWaitBetweenRetries - } - currentPending.nextRetryAt = time.Now().Add(backoff) - - s.listener.log.Debug( - "Receipt fetch failed, will retry", - "txnHash", txnHash.String(), - "attempt", currentPending.attempts, - "nextRetryIn", backoff, - ) - return - } + s.retryMu.Lock() - // Remove from pending list - delete(s.pendingReceipts, txnHash) + // Check if the item still exists and is the same one we claimed. + currentPending, exists := s.pendingReceipts[key] + if !exists || currentPending != p { + s.retryMu.Unlock() + s.listener.log.Debug("Pending receipt is stale or already processed, skipping retry", "txnHash", txnHash.String()) + return + } - // Update receipt with fetched data - p.receipt.receipt = r - p.receipt.logs = r.Logs - p.receipt.Filter = p.filterer + if err != nil { + defer s.retryMu.Unlock() + if errors.Is(err, ethereum.NotFound) { + // NotFound is terminal for missing receipts and obsolete block generations. + delete(s.pendingReceipts, key) + s.listener.log.Debug("Receipt not found after retry, removing from queue", "txnHash", txnHash.String()) + return + } - // Check finality - if s.listener.isBlockFinal(r.BlockNumber) { - p.receipt.Final = true - } + // Provider error - update retry state directly on the pointer. + currentPending.attempts++ + if currentPending.attempts >= maxReceiptRetryAttempts { + delete(s.pendingReceipts, key) + s.listener.log.Error( + "Failed to fetch receipt after max retries", + "txnHash", txnHash.String(), + "attempts", currentPending.attempts, + "error", err, + ) + return + } - // Handle finalization queue if needed - if !p.receipt.Final && p.filterer.Options().Finalize { - s.finalizer.enqueue(p.filterer.FilterID(), p.receipt, r.BlockNumber) - } + // Exponential backoff for next retry + backoff := time.Duration(1< maxWaitBetweenRetries { + backoff = maxWaitBetweenRetries + } + currentPending.nextRetryAt = time.Now().Add(backoff) - // Handle LimitOne filter removal - toFinalize := p.filterer.Options().Finalize && !p.receipt.Final - if p.filterer.Options().LimitOne && !toFinalize { - s.RemoveFilter(p.filterer) - } + s.listener.log.Debug( + "Receipt fetch failed, will retry", + "txnHash", txnHash.String(), + "attempt", currentPending.attempts, + "nextRetryIn", backoff, + ) + return + } - // Send to subscriber - s.ch.Send(p.receipt) + attempts := currentPending.attempts + s.retryMu.Unlock() - s.listener.log.Info( - "Successfully fetched receipt after retry", - "txnHash", txnHash.String(), - "attempts", currentPending.attempts, - ) - }(pending) + s.deliveryMu.Lock() + s.retryMu.Lock() + currentPending, exists = s.pendingReceipts[key] + s.retryMu.Unlock() + if !exists || currentPending != p { + s.deliveryMu.Unlock() + return + } + published := s.publishLocked(ctx, receipt, p.owner) + s.retryMu.Lock() + if current, exists := s.pendingReceipts[key]; exists && current == p { + if published || !s.hasFilter(p.owner) || !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + delete(s.pendingReceipts, key) + } else { + // A successful fetch can outlive its child deadline while waiting + // for delivery. Keep this exact valid owner available to a live retry. + current.completed = &receipt + current.nextRetryAt = time.Now().Add(100 * time.Millisecond) + } + } + s.retryMu.Unlock() + s.deliveryMu.Unlock() + if !published { + return } - wg.Wait() + s.listener.log.Info( + "Successfully fetched receipt after retry", + "txnHash", txnHash.String(), + "attempts", attempts, + ) } diff --git a/go.mod b/go.mod index a8a1b939..7f279041 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/google/gofuzz v1.2.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 - github.com/goware/breaker v0.2.0 + github.com/goware/breaker v0.3.2 github.com/goware/cachestore-mem v0.2.2 github.com/goware/cachestore-redis v0.2.1 github.com/goware/cachestore2 v0.12.3 diff --git a/go.sum b/go.sum index 4eeddb69..258c1801 100644 --- a/go.sum +++ b/go.sum @@ -96,8 +96,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/goware/breaker v0.2.0 h1:MJOaFjHwQ7h5/nCtjAUuOHhJeLCFF2GeUnXsBDMXq1k= -github.com/goware/breaker v0.2.0/go.mod h1:ijCEfXAa0j6w7IoHA4v6Sox2W6U9HUbI/t+5x0zGaug= +github.com/goware/breaker v0.3.2 h1:rVT/0UqQ70b3PN78Gj9dtZRDFJ1ZK8sm/fA8FkpOyCc= +github.com/goware/breaker v0.3.2/go.mod h1:CfgulI2zAaSLSnIeT8EN3mOJeWKSUbC26mGaveebT3E= github.com/goware/cachestore-mem v0.2.2 h1:toE6/1QMQQcQLJQpTIiTDAIHWLN4zvihoqZHq41cPns= github.com/goware/cachestore-mem v0.2.2/go.mod h1:KpXr+yVajbeN0s+CX/08hdSww2WhjtAL1pmNy69S03w= github.com/goware/cachestore-redis v0.2.1 h1:bMdkzGuy6Dsybq2chv3kRwu8UGoJz3aQBO9Cu41vEqQ= @@ -160,7 +160,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= From a43e30f055f4cd2397624bc7edce58c717e2c385 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 13:28:01 -0400 Subject: [PATCH 07/11] fix(ethmonitor): preserve empty-log provider compatibility --- ethmonitor/ethmonitor.go | 38 ++++++------- ethmonitor/logs_test.go | 120 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 21 deletions(-) create mode 100644 ethmonitor/logs_test.go diff --git a/ethmonitor/ethmonitor.go b/ethmonitor/ethmonitor.go index 37acf283..c4a20170 100644 --- a/ethmonitor/ethmonitor.go +++ b/ethmonitor/ethmonitor.go @@ -817,18 +817,10 @@ func (m *Monitor) addLogs(ctx context.Context, blocks Blocks) { logs, _, err := m.filterLogs(tctx, blockHash, m.logTopics(), block.Bloom()) if err == nil { - // check the logsBloom from the block to check if we should be expecting logs. logsBloom - // will be included for any indexed logs. - if len(logs) > 0 || block.Bloom() == (types.Bloom{}) { - // successful backfill - if logs == nil { - block.Logs = []types.Log{} - } else { - block.Logs = logs - } - block.OK = true - continue - } + // filterLogs validates emptiness against the bloom and query topics. + block.Logs = logs + block.OK = true + continue } // mark for backfilling @@ -853,6 +845,9 @@ func (m *Monitor) logTopics() [][]common.Hash { func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics [][]common.Hash, blockBloom types.Bloom) ([]types.Log, []byte, error) { var fetchedLogs []types.Log + // The block bloom covers all logs, so unrelated topics may set it even + // when a topic-filtered query correctly returns no logs. + expectLogs := len(topics) == 0 && blockBloom != (types.Bloom{}) getter := func(ctx context.Context, _ string) ([]byte, error) { if m.options.DebugLogging { @@ -871,7 +866,7 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics } // Validate before caching so a malformed response cannot block log // backfilling until cache expiry. - fetchedLogs, err = m.unmarshalLogs(logsPayload, blockBloom) + fetchedLogs, err = m.unmarshalLogs(logsPayload, expectLogs) if err != nil { return nil, err } @@ -894,7 +889,7 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics if fetchedLogs != nil { return fetchedLogs, resp, nil } - logs, err := m.unmarshalLogs(resp, blockBloom) + logs, err := m.unmarshalLogs(resp, expectLogs) if err != nil { // Recover entries cached by peers or older monitors that did not // validate logs before writing them. @@ -1417,19 +1412,20 @@ func (m *Monitor) unmarshalBlock(blockPayload []byte) (*types.Block, error) { return block, nil } -func (m *Monitor) unmarshalLogs(logsPayload []byte, blockBloom types.Bloom) ([]types.Log, error) { +func (m *Monitor) unmarshalLogs(logsPayload []byte, expectLogs bool) ([]types.Log, error) { var logs []types.Log err := json.Unmarshal(logsPayload, &logs) if err != nil { return nil, err } - // JSON null decodes without error, but a logs response must be an array. - if logs == nil { - return nil, fmt.Errorf("ethmonitor: filterLogs expected a JSON array of block logs") - } - // Apply the bloom guard to decoded responses, including cache hits. - if len(logs) == 0 && blockBloom != (types.Bloom{}) { + // Some chains return JSON null instead of [] when there are no logs. + // Preserve the original bloom guard for unfiltered queries: a nonzero + // bloom means logs must be present, even after a successful RPC response. + if len(logs) == 0 && expectLogs { return nil, fmt.Errorf("ethmonitor: filterLogs detected empty block-logs response but block bloom is set, ignoring node response") } + if logs == nil { + logs = []types.Log{} + } return logs, nil } diff --git a/ethmonitor/logs_test.go b/ethmonitor/logs_test.go new file mode 100644 index 00000000..725e03bd --- /dev/null +++ b/ethmonitor/logs_test.go @@ -0,0 +1,120 @@ +package ethmonitor + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/0xsequence/ethkit/ethrpc" + "github.com/0xsequence/ethkit/ethrpc/jsonrpc" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/stretchr/testify/require" +) + +func TestMonitorEmptyLogResponses(t *testing.T) { + cases := []struct { + name string + payload string + nonzeroBloom bool + filtered bool + status int + rpcError bool + emptyBody bool + ready bool + }{ + {name: "zero_bloom_array", payload: `[]`, ready: true}, + {name: "zero_bloom_null", payload: `null`, ready: true}, + {name: "zero_bloom_spaced_array", payload: `[ ]`, ready: true}, + {name: "nonzero_bloom_array", payload: `[]`, nonzeroBloom: true}, + {name: "nonzero_bloom_null", payload: `null`, nonzeroBloom: true}, + {name: "nonzero_bloom_spaced_array", payload: `[ ]`, nonzeroBloom: true}, + {name: "filtered_array", payload: `[]`, nonzeroBloom: true, filtered: true, ready: true}, + {name: "filtered_null", payload: `null`, nonzeroBloom: true, filtered: true, ready: true}, + {name: "filtered_spaced_array", payload: `[ ]`, nonzeroBloom: true, filtered: true, ready: true}, + {name: "invalid_object", payload: `{}`}, + {name: "invalid_log", payload: `[{}]`}, + {name: "http_error", payload: `null`, status: http.StatusServiceUnavailable}, + {name: "rpc_error", rpcError: true}, + {name: "empty_http_body", emptyBody: true}, + {name: "missing_rpc_result"}, + } + for _, concurrency := range []int{0, 1} { + for _, tc := range cases { + t.Run(fmt.Sprintf("prefetch=%d/%s", concurrency, tc.name), func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + var request jsonrpc.Message + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + if request.Method != "eth_getLogs" { + t.Errorf("unexpected RPC method %q", request.Method) + } + w.Header().Set("Content-Type", "application/json") + status := tc.status + if status == 0 { + status = http.StatusOK + } + w.WriteHeader(status) + if tc.emptyBody { + return + } + response := jsonrpc.Message{Version: "2.0", ID: request.ID, Result: json.RawMessage(tc.payload)} + if tc.rpcError { + response.Error = &jsonrpc.Error{Code: -32000, Message: "provider not ready"} + } + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Error(err) + } + })) + defer server.Close() + provider, err := ethrpc.NewProvider(server.URL) + require.NoError(t, err) + opts := DefaultOptions + opts.WithLogs = true + opts.PrefetchConcurrency = concurrency + if tc.filtered { + opts.LogTopics = []common.Hash{common.HexToHash("0xabcd")} + } + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID = big.NewInt(1) + hash := common.HexToHash("0x1000") + var bloom types.Bloom + if tc.nonzeroBloom { + bloom = types.BytesToBloom([]byte{1}) // Logs exist, but need not match LogTopics. + } + for attempt := 0; attempt < 2; attempt++ { + block := &Block{ + Block: types.NewBlockWithHeader(&types.Header{Number: big.NewInt(1000), BlockHash: hash, Bloom: bloom}), + Event: Added, + } + monitor.addLogs(context.Background(), Blocks{block}) + require.Equal(t, tc.ready, block.OK, "attempt %d", attempt+1) + if tc.ready { + require.NotNil(t, block.Logs, "accepted null must become an empty log slice") + require.Empty(t, block.Logs) + } + } + wantCalls := int32(2) + if concurrency > 0 && tc.ready { + wantCalls = 1 // Valid empty results are reused; failures remain retryable. + } + require.Equal(t, wantCalls, calls.Load()) + if monitor.cache != nil { + _, found, err := monitor.cache.Get(context.Background(), CacheKeyBlockLogs(monitor.chainID, hash, monitor.logTopics())) + require.NoError(t, err) + require.Equal(t, tc.ready, found, "only valid responses may remain cached") + } + }) + } + } +} From 62a096a60bfcab029e55fa61c892d84b5cbf6d81 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 16:22:14 -0400 Subject: [PATCH 08/11] fix(ethmonitor): retry origin after cache fetch failures --- ethmonitor/ethmonitor.go | 13 +++- ethmonitor/fetch_test.go | 147 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 ethmonitor/fetch_test.go diff --git a/ethmonitor/ethmonitor.go b/ethmonitor/ethmonitor.go index c4a20170..f7ecff31 100644 --- a/ethmonitor/ethmonitor.go +++ b/ethmonitor/ethmonitor.go @@ -998,7 +998,18 @@ func (m *Monitor) fetchNextBlock(ctx context.Context, bypassCache bool) (*types. key := CacheKeyBlockByNumber(m.chainID, nextBlockNumber) resp, err := m.cache.GetOrSetWithLockEx(ctx, key, getter, m.options.CacheExpiry) if err != nil { - return nil, resp, miss, err + if ctx.Err() != nil { + return nil, resp, miss, ctx.Err() + } + // A cache timeout or a shared prefetch error must not interrupt the + // serial fetch. Retry directly with the caller's context, preserving + // the normal wait for a block that the node does not serve yet. + miss = true + m.hitStreak.Store(0) + resp, err = getter(ctx, "") + if err != nil { + return nil, resp, miss, err + } } if fetchedBlock != nil { return fetchedBlock, resp, miss, nil diff --git a/ethmonitor/fetch_test.go b/ethmonitor/fetch_test.go new file mode 100644 index 00000000..b107f422 --- /dev/null +++ b/ethmonitor/fetch_test.go @@ -0,0 +1,147 @@ +package ethmonitor + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/big" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + memcache "github.com/goware/cachestore-mem" + cachestore "github.com/goware/cachestore2" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func fetchTestPayload(t *testing.T) json.RawMessage { + t.Helper() + payload, err := json.Marshal(&types.Header{ + Number: big.NewInt(1000), Difficulty: big.NewInt(0), GasLimit: 30_000_000, + }) + require.NoError(t, err) + return payload +} + +func TestMonitorFetchNextBlockCacheTimeout(t *testing.T) { + for _, concurrency := range []int{0, 1} { + t.Run(fmt.Sprintf("prefetch=%d", concurrency), func(t *testing.T) { + // Use the actual memory backend with a shorter timeout than its + // eight-second default. Waiting for a slow head must outlive it. + backend, err := memcache.NewBackend(512, cachestore.WithLockRetryTimeout(25*time.Millisecond)) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + callerDeadline, _ := ctx.Deadline() + var readyAt time.Time + payload := fetchTestPayload(t) + usedCallerDeadline := false + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + provider.EXPECT().RawBlockByNumber(gomock.Any(), big.NewInt(1000)).DoAndReturn( + func(ctx context.Context, _ *big.Int) (json.RawMessage, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if readyAt.IsZero() { + readyAt = time.Now().Add(100 * time.Millisecond) + } + deadline, _ := ctx.Deadline() + usedCallerDeadline = usedCallerDeadline || deadline.Equal(callerDeadline) + if time.Now().Before(readyAt) { + return nil, ethereum.NotFound + } + return payload, nil + }, + ).AnyTimes() + opts := DefaultOptions + opts.CacheBackend = backend + opts.PrefetchConcurrency = concurrency + opts.PollingInterval = time.Millisecond + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID, monitor.nextBlockNumber = big.NewInt(1), big.NewInt(1000) + + block, response, miss, err := monitor.fetchNextBlock(ctx, false) + require.NoError(t, err) + require.Equal(t, uint64(1000), block.NumberU64()) + require.Equal(t, []byte(payload), response) + require.True(t, miss) + require.True(t, usedCallerDeadline, "origin retry inherited the cache's shorter deadline") + }) + } +} + +// Memory-cache singleflight can return another caller's getter error without +// invoking this caller's getter. Inject that failure without relying on timing. +type fetchErrorCache struct { + cachestore.Store[[]byte] + err error + cancel context.CancelFunc +} + +func (c *fetchErrorCache) GetOrSetWithLockEx(context.Context, string, func(context.Context, string) ([]byte, error), time.Duration) ([]byte, error) { + if c.cancel != nil { + c.cancel() + } + return nil, fmt.Errorf("cache getter: %w", c.err) +} + +func TestMonitorFetchNextBlockCacheFailure(t *testing.T) { + for _, tc := range []struct { + name string + cacheErr error + cancel bool + invalidOrigin bool + }{ + {name: "shared_prefetch_not_found", cacheErr: ethereum.NotFound}, + {name: "shared_prefetch_timeout", cacheErr: context.DeadlineExceeded}, + {name: "backend_failure", cacheErr: errors.New("cache unavailable")}, + {name: "caller_canceled", cacheErr: ethereum.NotFound, cancel: true}, + {name: "invalid_origin", cacheErr: ethereum.NotFound, invalidOrigin: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + payload := fetchTestPayload(t) + if tc.invalidOrigin { + payload = json.RawMessage(`{"number":"0x3e8"}`) + } + if !tc.cancel { + provider.EXPECT().RawBlockByNumber(gomock.Any(), big.NewInt(1000)).Return(payload, nil) + } + opts := DefaultOptions + opts.PrefetchConcurrency = 1 + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID, monitor.nextBlockNumber = big.NewInt(1), big.NewInt(1000) + cache := &fetchErrorCache{Store: monitor.cache, err: tc.cacheErr} + if tc.cancel { + cache.cancel = cancel + } + monitor.cache = cache + monitor.hitStreak.Store(2) + + block, response, miss, err := monitor.fetchNextBlock(ctx, false) + switch { + case tc.cancel: + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, block) // The mock also rejects any origin call. + case tc.invalidOrigin: + require.Error(t, err) + require.NotErrorIs(t, err, tc.cacheErr, "origin validation was never attempted") + require.Nil(t, block) + default: + require.NoError(t, err) + require.Equal(t, uint64(1000), block.NumberU64()) + require.Equal(t, []byte(payload), response) + require.True(t, miss) + require.Zero(t, monitor.hitStreak.Load(), "cache failure left the catch-up signal active") + } + }) + } +} From 0dba4f8b6fa1c3dbb61c36f44d52b72f97326f0e Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 16:51:24 -0400 Subject: [PATCH 09/11] fix(ethmonitor): keep log publication alive during cache outages --- ethmonitor/cache_test.go | 131 +++++++++++++++++++++++++++ ethmonitor/ethmonitor.go | 34 +++++-- ethmonitor/fetch_test.go | 83 ++++++++++++++++- ethmonitor/logs_test.go | 2 + ethmonitor/prefetch.go | 5 +- ethmonitor/prefetch_internal_test.go | 4 +- 6 files changed, 247 insertions(+), 12 deletions(-) create mode 100644 ethmonitor/cache_test.go diff --git a/ethmonitor/cache_test.go b/ethmonitor/cache_test.go new file mode 100644 index 00000000..08be9152 --- /dev/null +++ b/ethmonitor/cache_test.go @@ -0,0 +1,131 @@ +package ethmonitor_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/big" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + memcache "github.com/goware/cachestore-mem" + cachestore "github.com/goware/cachestore2" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// Fail before invoking the getter, as an unreachable shared cache would. +type outageBackend struct { + cachestore.Backend + down atomic.Bool + failures atomic.Int64 +} + +func (b *outageBackend) GetOrSetWithLockEx(ctx context.Context, key string, getter func(context.Context, string) (any, error), ttl time.Duration) (any, error) { + if b.down.Load() { + b.failures.Add(1) + return nil, errors.New("cache unavailable") + } + return b.Backend.GetOrSetWithLockEx(ctx, key, getter, ttl) +} + +// Give each fake block a nonzero bloom and an actual log to verify that +// outage recovery preserves logs, rather than just marking blocks ready. +type cacheTestProvider struct { + *fakeProvider + log types.Log +} + +func (p *cacheTestProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json.RawMessage, error) { + payload, err := p.fakeProvider.RawBlockByNumber(ctx, num) + if err != nil { + return nil, err + } + bloom := types.CreateBloom(&types.Receipt{Logs: []*types.Log{&p.log}}) + return json.RawMessage(strings.Replace(string(payload), strings.Repeat("0", 512), common.Bytes2Hex(bloom[:]), 1)), nil +} + +func (p *cacheTestProvider) RawFilterLogs(ctx context.Context, q ethereum.FilterQuery) (json.RawMessage, error) { + if err := p.chain.wait(ctx); err != nil { + return nil, err + } + block, ok := p.chain.byHash(*q.BlockHash) + if !ok { + return nil, ethereum.NotFound + } + log := p.log + log.BlockHash, log.BlockNumber = block.hash, block.num + return json.Marshal([]types.Log{log}) +} + +func TestMonitorCacheOutageWithLogs(t *testing.T) { + for _, concurrency := range []int{0, 2} { + t.Run(fmt.Sprintf("prefetch=%d", concurrency), func(t *testing.T) { + chain := newFakeChain(1000, 40, 0) + chain.hold(1020) + provider := &cacheTestProvider{ + fakeProvider: &fakeProvider{MockRawInterface: mocks.NewMockRawInterface(gomock.NewController(t)), chain: chain}, + log: types.Log{Address: common.HexToAddress("0x1234"), Topics: []common.Hash{common.HexToHash("0xabcd")}, Data: []byte{1, 2, 3}}, + } + mem, err := memcache.NewBackend(512) + require.NoError(t, err) + backend := &outageBackend{Backend: mem} + backend.down.Store(true) + opts := ethmonitor.DefaultOptions + opts.PollingInterval = 5 * time.Millisecond + opts.Timeout = time.Second + opts.WithLogs = true + opts.BlockRetentionLimit = 5 // Queue capacity is 10; publish 20 blocks while down. + opts.StartBlockNumber = big.NewInt(1000) + opts.PrefetchConcurrency = concurrency + opts.CacheBackend = backend + monitor, err := ethmonitor.NewMonitor(provider, opts) + require.NoError(t, err) + sub := monitor.Subscribe() + defer sub.Unsubscribe() + done := runMonitorForTest(t, monitor) + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + next := uint64(1000) + for next <= 1039 { + select { + case blocks := <-sub.Blocks(): + for _, block := range blocks { + require.Equal(t, next, block.NumberU64()) + require.Equal(t, ethmonitor.Added, block.Event) + require.True(t, block.OK) + require.NotZero(t, block.Bloom()) + log := provider.log + log.BlockHash, log.BlockNumber = block.Hash(), next + require.Equal(t, []types.Log{log}, block.Logs) + next++ + if next == 1020 { + require.True(t, backend.down.Load()) + require.GreaterOrEqual(t, backend.failures.Load(), int64(40)) + backend.down.Store(false) + chain.release(1020) + } + } + case err := <-done: + t.Fatalf("Run returned during cache outage/recovery: %v (head=%v)", err, monitor.LatestBlockNum()) + case <-timer.C: + t.Fatalf("monitor stopped publishing at block %d", next-1) + } + } + // After recovery, ordinary reads fill the cache again without a restart. + key := ethmonitor.CacheKeyBlockLogs(big.NewInt(1), chain.hashAt(1039), nil) + _, found, err := backend.Get(context.Background(), key) + require.NoError(t, err) + require.True(t, found) + require.True(t, monitor.IsRunning()) + }) + } +} diff --git a/ethmonitor/ethmonitor.go b/ethmonitor/ethmonitor.go index f7ecff31..311b61b6 100644 --- a/ethmonitor/ethmonitor.go +++ b/ethmonitor/ethmonitor.go @@ -96,7 +96,8 @@ type Options struct { // WithLogs will include logs with the blocks if specified true. WithLogs bool - // LogTopics will filter only specific log topics to include. + // LogTopics selects values for the first topic of each log. Empty filtered + // results are accepted because a nonzero bloom does not prove a match. LogTopics []common.Hash // CacheBackend to use for caching block data @@ -113,6 +114,8 @@ type Options struct { // reads them as cache hits instead of paying node round-trips per block. // Useful on chains whose block rate outpaces a serial fetch. If no // CacheBackend is set, an in-memory cache is used. 0 disables prefetching. + // A prefetched block may be reorged out before it is read, producing an + // Added then Removed event; subscribers must already handle such reorgs. PrefetchConcurrency int // PrefetchWindow is how many blocks past the monitor's next block the @@ -154,8 +157,8 @@ type Monitor struct { // next block without a miss, capped to avoid overflow. hitStreak atomic.Int32 - // latestHead is the most recent chain head number seen, from the - // newHeads stream or, in polling mode, the prefetcher's head poll. + // latestHead bounds prefetch scheduling using the most recent newHeads + // event or, in polling mode, the prefetcher's head poll. latestHead atomic.Uint64 cache cachestore.Store[[]byte] @@ -404,6 +407,9 @@ func (m *Monitor) isCatchingUp() bool { func (m *Monitor) listenNewHead(ctx context.Context, wg *sync.WaitGroup) <-chan uint64 { ch := make(chan uint64) + // This stream-only head controls listener pacing and resets on reconnect. + // In polling mode it stays zero so each iteration waits for a poll tick; + // m.latestHead separately bounds prefetch scheduling in either mode. var latestHeadBlock atomic.Uint64 nextBlock := make(chan uint64) @@ -884,7 +890,15 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics key := CacheKeyBlockLogs(m.chainID, blockHash, topics) resp, err := m.cache.GetOrSetWithLockEx(ctx, key, getter, m.options.CacheExpiry) if err != nil { - return nil, resp, err + if ctx.Err() != nil { + return nil, resp, ctx.Err() + } + // Keep logs available when the cache fails, just as for block fetches. + // The direct result deliberately bypasses cache reads and writes. + resp, err = getter(ctx, "") + if err != nil { + return nil, resp, err + } } if fetchedLogs != nil { return fetchedLogs, resp, nil @@ -1004,7 +1018,7 @@ func (m *Monitor) fetchNextBlock(ctx context.Context, bypassCache bool) (*types. // A cache timeout or a shared prefetch error must not interrupt the // serial fetch. Retry directly with the caller's context, preserving // the normal wait for a block that the node does not serve yet. - miss = true + // Deliberately bypass cache writes too; only origin misses slow polling. m.hitStreak.Store(0) resp, err = getter(ctx, "") if err != nil { @@ -1168,7 +1182,15 @@ func (m *Monitor) fetchBlockByHash(ctx context.Context, hash common.Hash) (*type key := CacheKeyBlockByHash(m.chainID, hash) resp, err := m.cache.GetOrSetWithLockEx(ctx, key, getter, m.options.CacheExpiry) if err != nil { - return nil, nil, err + if ctx.Err() != nil { + return nil, nil, ctx.Err() + } + // Reorg ancestry must remain available during a cache outage. + // The direct result deliberately bypasses cache reads and writes. + resp, err = getter(ctx, "") + if err != nil { + return nil, nil, err + } } block, err := m.unmarshalBlock(resp) return block, resp, err diff --git a/ethmonitor/fetch_test.go b/ethmonitor/fetch_test.go index b107f422..96d2479e 100644 --- a/ethmonitor/fetch_test.go +++ b/ethmonitor/fetch_test.go @@ -11,6 +11,7 @@ import ( "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" "github.com/0xsequence/ethkit/go-ethereum/core/types" memcache "github.com/goware/cachestore-mem" cachestore "github.com/goware/cachestore2" @@ -139,9 +140,89 @@ func TestMonitorFetchNextBlockCacheFailure(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(1000), block.NumberU64()) require.Equal(t, []byte(payload), response) - require.True(t, miss) + require.False(t, miss, "an immediately available origin block must not slow polling") require.Zero(t, monitor.hitStreak.Load(), "cache failure left the catch-up signal active") } }) } } + +func TestMonitorCacheFallback(t *testing.T) { + for _, method := range []string{"logs", "block_by_hash"} { + for _, failure := range []string{"backend_failure", "caller_canceled", "invalid_origin"} { + t.Run(method+"/"+failure, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + deadline, _ := ctx.Deadline() + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + hash := common.HexToHash("0x1000") + log := types.Log{Address: common.HexToAddress("0x1234"), Topics: []common.Hash{common.HexToHash("0xabcd")}, Data: []byte{1}} + payload := fetchTestPayload(t) + if method == "logs" { + var err error + payload, err = json.Marshal([]types.Log{log}) + require.NoError(t, err) + } + if failure == "invalid_origin" { + payload = json.RawMessage(`{}`) + } + if failure != "caller_canceled" { + if method == "logs" { + provider.EXPECT().RawFilterLogs(gomock.Any(), ethereum.FilterQuery{BlockHash: &hash}).DoAndReturn( + func(ctx context.Context, _ ethereum.FilterQuery) (json.RawMessage, error) { + actualDeadline, _ := ctx.Deadline() + require.Equal(t, deadline, actualDeadline) + return payload, nil + }) + } else { + provider.EXPECT().RawBlockByHash(gomock.Any(), hash).DoAndReturn( + func(ctx context.Context, _ common.Hash) (json.RawMessage, error) { + actualDeadline, _ := ctx.Deadline() + require.Equal(t, deadline, actualDeadline) + return payload, nil + }) + } + } + opts := DefaultOptions + opts.PrefetchConcurrency = 1 + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID = big.NewInt(1) + cache := &fetchErrorCache{Store: monitor.cache, err: errors.New("cache unavailable")} + if failure == "caller_canceled" { + cache.cancel = cancel + } + monitor.cache = cache + var response []byte + if method == "logs" { + var logs []types.Log + logs, response, err = monitor.filterLogs(ctx, hash, nil, types.CreateBloom(&types.Receipt{Logs: []*types.Log{&log}})) + if failure == "backend_failure" { + require.Equal(t, []types.Log{log}, logs) + } else { + require.Empty(t, logs) + } + } else { + var block *types.Block + block, response, err = monitor.fetchBlockByHash(ctx, hash) + if failure == "backend_failure" { + require.NotNil(t, block) + require.Equal(t, uint64(1000), block.NumberU64()) + } else { + require.Nil(t, block) + } + } + switch failure { + case "backend_failure": + require.NoError(t, err) + require.Equal(t, []byte(payload), response) + case "caller_canceled": + require.ErrorIs(t, err, context.Canceled) // Mock rejects any origin call. + case "invalid_origin": + require.Error(t, err) + require.NotErrorIs(t, err, cache.err, "origin validation was never attempted") + } + }) + } + } +} diff --git a/ethmonitor/logs_test.go b/ethmonitor/logs_test.go index 725e03bd..305fe354 100644 --- a/ethmonitor/logs_test.go +++ b/ethmonitor/logs_test.go @@ -107,6 +107,8 @@ func TestMonitorEmptyLogResponses(t *testing.T) { wantCalls := int32(2) if concurrency > 0 && tc.ready { wantCalls = 1 // Valid empty results are reused; failures remain retryable. + } else if concurrency > 0 { + wantCalls = 4 // Each cache error gets one direct origin retry. } require.Equal(t, wantCalls, calls.Load()) if monitor.cache != nil { diff --git a/ethmonitor/prefetch.go b/ethmonitor/prefetch.go index 9b6c0565..5709105f 100644 --- a/ethmonitor/prefetch.go +++ b/ethmonitor/prefetch.go @@ -203,9 +203,8 @@ func (p *prefetcher) worker(ctx context.Context) { } } -// fetch loads one block, and its logs, into the cache. It makes a single -// attempt: on any failure the run loop fetches the block itself when it gets -// there. +// fetch loads one block, and its logs, into the cache. Failures leave the +// run loop able to fetch from origin when it gets there. func (p *prefetcher) fetch(ctx context.Context, job prefetchJob) { defer func() { if r := recover(); r != nil { diff --git a/ethmonitor/prefetch_internal_test.go b/ethmonitor/prefetch_internal_test.go index 736f3e6b..ebd43dfe 100644 --- a/ethmonitor/prefetch_internal_test.go +++ b/ethmonitor/prefetch_internal_test.go @@ -116,10 +116,10 @@ func TestMonitorRecoversInvalidPrefetchedLogs(t *testing.T) { // Complete the speculative fetch before starting the serial loop, // so the worker deterministically receives the invalid response. monitor.prefetch.fetch(context.Background(), prefetchJob{num: target}) - require.Equal(t, int64(1), originCalls.Load()) + require.Equal(t, int64(2), originCalls.Load(), "invalid response gets one direct origin retry") _, found, err := monitor.cache.Get(context.Background(), key) require.NoError(t, err) - require.False(t, found, "failed prefetch must not cache invalid logs") + require.False(t, found, "neither the invalid response nor the direct retry should be cached") } sub := monitor.Subscribe("TestMonitorRecoversInvalidPrefetchedLogs") From 8233c609d8ebf4bb81b2ec1fe496bc88ca2b3f63 Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 20:15:52 -0400 Subject: [PATCH 10/11] fix(ethreceipts): release orphaned LimitOne selection at finality Rollback keeps a LimitOne owner's claim on its delivered txn so the same txn can be re-mined. Nothing released that claim once the orphaned delivery was pruned at finality, so a filter whose txn never came back silently dropped every later match: forever with MaxWait 0, or until ErrFilterExhausted otherwise. Release the claim when its delivery is pruned at finality. releaseClaim still keeps it while the owner has queued finality, in-flight, delivered or pending work, so a txn re-mined before then stays selected. Co-Authored-By: Claude Opus 5.5 --- ethreceipts/reorg_test.go | 56 +++++++++++++++++++++++++++++++++++++ ethreceipts/subscription.go | 9 ++++++ 2 files changed, 65 insertions(+) diff --git a/ethreceipts/reorg_test.go b/ethreceipts/reorg_test.go index 55183c97..dd374a26 100644 --- a/ethreceipts/reorg_test.go +++ b/ethreceipts/reorg_test.go @@ -555,6 +555,62 @@ func TestReceiptsFixRollbackPreservesMinedSelection(t *testing.T) { hardeningNoReceipt(t, s) } +func TestReceiptsFixOrphanedSelectionReleasedAtFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 808) + other, _, _ := hardeningTxn(t, 809) + var chain []*ethmonitor.Block + for num := int64(100); num <= 110; num++ { + chain = append(chain, hardeningBlock(num)) + } + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions(), chain...) + header := hardeningBlock(109).Header() + header.BlockHash = common.HexToHash("0xdead") + orphan := ðmonitor.Block{Block: types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: []*types.Transaction{tx}}), Event: ethmonitor.Added, OK: true} + canonical := hardeningBlock(110, other) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(orphan, tx)) + if mined := hardeningRead(t, s); mined.Final || mined.BlockHash() != orphan.Hash() { + t.Fatal("orphan was delivered as final") + } + s.rollbackBlock(l.invalidateBlock(context.Background(), orphan)) + hardeningRead(t, s) + if err := s.finalizeReceipts(big.NewInt(110)); err != nil { + t.Fatal(err) + } + if s.claims[owner] != tx.Hash() { + t.Fatal("selection released while its txn could still be re-mined") + } + // The bootstrapped monitor cannot advance, so shrink the finality depth to + // put the orphan's height past it. + l.mu.Lock() + l.options.NumBlocksToFinality = 1 + l.mu.Unlock() + if err := s.finalizeReceipts(big.NewInt(110)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + if _, claimed := s.claims[owner]; claimed || len(s.deliveries) != 0 || len(s.Filters()) != 1 { + t.Fatal("finalized orphan kept its selection") + } + process(hardeningReceipt(canonical, other)) + mined := hardeningRead(t, s) + if mined.Reorged || mined.Final || mined.TransactionHash() != other.Hash() || mined.BlockHash() != canonical.Hash() || mined.Filter != q { + t.Fatal("released owner did not select the canonical txn") + } + if s.claims[owner] != other.Hash() { + t.Fatal("canonical txn did not take over the selection") + } +} + func TestReceiptsFixReadoptionAfterRetention(t *testing.T) { for _, scenario := range []string{"canonical_retention", "obsolete_incarnation", "later_removal", "alternate_retention"} { t.Run(scenario, func(t *testing.T) { diff --git a/ethreceipts/subscription.go b/ethreceipts/subscription.go index ed55b382..c36627f4 100644 --- a/ethreceipts/subscription.go +++ b/ethreceipts/subscription.go @@ -581,9 +581,18 @@ func (s *subscriber) finalizeReceipts(blockNum *big.Int) error { } } // Retain mined data until its block is final, for receipt-free rollback. + pruned := make(map[*filterOwner]struct{}) for key, receipt := range s.deliveries { if receipt.BlockNumber() != nil && s.listener.isBlockFinal(receipt.BlockNumber()) { delete(s.deliveries, key) + pruned[key.owner] = struct{}{} + } + } + // Rollback keeps a LimitOne selection so its orphaned txn can be re-mined. + // Once the orphan's height is final, free the owner to select another txn. + for owner := range pruned { + if _, claimed := s.claims[owner]; claimed { + s.releaseClaim(owner) } } return nil From 1fc3e787241fa945277a29cf370ab0ea4b5b99fc Mon Sep 17 00:00:00 2001 From: Peter Kieltyka Date: Tue, 6 Oct 2026 21:28:19 -0400 Subject: [PATCH 11/11] fix(ethreceipts): harden delivery and channel handling Upgrade goware/channel to v0.6.0 for safe concurrent Send/Close and backed-off queue alerts. Raise the subscriber warning threshold to 10. Release receiptMu before channel sends while deliveryMu continues to serialize publication, rollback, and finality. Recheck cancellation after block validation, accumulate matches across added blocks, and let custom filters manage their own MaxWait exhaustion. Preserve block generations needed by pending retries and late RPCs beyond monitor retention. Cover receipt recovery, orphan rejection, cancellation, blocked sends, batch matching, and custom exhaustion with regression tests. Bound the blocked-send test's startup and cleanup waits. Validation: full ethreceipts and ethmonitor suites with -race; focused regressions repeated 20 times; go build ./... and package vet passed. --- ethreceipts/ethreceipts.go | 22 +++- ethreceipts/lifecycle_test.go | 185 ++++++++++++++++++++++++++++++++++ ethreceipts/reorg_test.go | 152 ++++++++++++++++++++++++++++ ethreceipts/subscription.go | 19 ++-- go.mod | 2 +- go.sum | 4 +- 6 files changed, 368 insertions(+), 16 deletions(-) diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index 7c411654..273d2466 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -37,6 +37,11 @@ var DefaultOptions = Options{ const ( maxFiltersPerListener = 1000 + + // subscriberQueueWarning is how many unread receipts a subscriber can have + // queued before its channel warns and alerts. It is kept low so readers + // falling behind show up quickly. + subscriberQueueWarning = 10 ) type Options struct { @@ -91,7 +96,7 @@ type ReceiptsListener struct { ctxStop context.CancelFunc running int32 lifecycleMu sync.Mutex // protects Run/Stop cancellation handle and running transitions - receiptMu sync.Mutex + receiptMu sync.Mutex // never held across a channel send or while taking subscriber locks blockStates map[common.Hash]blockState reorgRevision uint64 // removal sequence used to reject hash queries started before invalidation mu sync.RWMutex @@ -233,7 +238,7 @@ func (l *ReceiptsListener) Subscribe(filterQueries ...FilterQuery) Subscription subscriber := &subscriber{ listener: l, - ch: channel.NewUnboundedChan[Receipt](2, 5000, channel.Options{ + ch: channel.NewUnboundedChan[Receipt](subscriberQueueWarning, 5000, channel.Options{ Logger: l.log, Alerter: l.alert, Label: "ethreceipts:subscriber", @@ -780,7 +785,8 @@ func (l *ReceiptsListener) listener(runCtx context.Context) error { if maxWait != 0 && (latestBlockNum-blockNum) >= maxWait { f := builtinFilter(filterer) if f == nil { - panic("ethreceipts: unexpected") + // A custom Filterer keeps its own counters and exhaustion signal. + continue } if (f.Options().LimitOne && f.LastMatchBlockNum() == 0) || !f.Options().LimitOne { @@ -909,7 +915,13 @@ func (l *ReceiptsListener) processBlockEvents(ctx context.Context, blocks ethmon if err != nil { l.log.Warn(fmt.Sprintf("error while processing filters: %s", err)) } - oks[i] = matched + // A match in any added block of the batch counts. Rolling back an + // earlier match is not a new one. + if !reorged { + for j, ok := range matched { + oks[i][j] = oks[i][j] || ok + } + } // check subscriber to finalize any receipts if !reorged && ctx.Err() == nil { @@ -1145,6 +1157,8 @@ func groupLogsByTransaction(logs []types.Log) map[string][]*types.Log { // A removed hash can later be canonical again. Generations invalidate work // started before rollback even when its transaction and block hash are unchanged. +// Pending retries and RPCs can outlive monitor retention, so block age alone +// cannot determine when it is safe to discard this state. type blockRef struct { hash common.Hash generation uint64 diff --git a/ethreceipts/lifecycle_test.go b/ethreceipts/lifecycle_test.go index 63907f7f..d54359fa 100644 --- a/ethreceipts/lifecycle_test.go +++ b/ethreceipts/lifecycle_test.go @@ -280,3 +280,188 @@ func TestHardeningMonitorClosureCancelsHeadWait(t *testing.T) { t.Error("monitor closure did not cancel waiting for first head") } } + +// blockingAlerter holds the subscriber channel's piping goroutine, as a slow +// alerting backend would, until released. +type blockingAlerter struct { + entered chan struct{} + once sync.Once + release chan struct{} +} + +func (a *blockingAlerter) Alert(context.Context, string, ...interface{}) { + a.once.Do(func() { close(a.entered) }) + <-a.release +} + +func TestReceiptsFixBlockedSendKeepsListenerUnlocked(t *testing.T) { + var txns []*types.Transaction + for nonce := uint64(820); nonce < 820+subscriberQueueWarning+2; nonce++ { + tx, _, _ := hardeningTxn(t, nonce) + txns = append(txns, tx) + } + b := hardeningBlock(100, txns...) + alerter := &blockingAlerter{entered: make(chan struct{}), release: make(chan struct{})} + opts := hardeningOptions() + opts.Alerter = alerter + l := hardeningListener(t, &hardeningProvider{}, opts) + q := FilterLogs(func([]*types.Log) bool { return true }) + s := l.Subscribe(q).(*subscriber) + owner := s.owner(q.(Filterer)) + published := make(chan struct{}) + go func() { + defer close(published) + // Nobody reads, so the receipt that passes subscriberQueueWarning raises + // an alert and the next one blocks in Send. + for _, tx := range txns { + s.publish(context.Background(), Receipt{receipt: hardeningReceipt(b, tx)}, owner) + } + }() + t.Cleanup(func() { + close(alerter.release) + select { + case <-published: + s.Unsubscribe() + case <-time.After(time.Second): + t.Error("publisher did not stop after releasing the alert") + } + }) + select { + case <-alerter.entered: + case <-published: + t.Fatal("publisher finished without raising a queue alert") + case <-time.After(time.Second): + t.Fatal("subscriber queue alert did not fire") + } + for deadline := time.Now().Add(200 * time.Millisecond); time.Now().Before(deadline); { + checked := make(chan struct{}) + go func() { + defer close(checked) + l.isCurrentBlock(b.Hash(), 0) + }() + select { + case <-checked: + case <-time.After(time.Second): + t.Fatal("receiptMu held across a blocked subscriber send") + } + } +} + +// publicationContext signals the first cancellation check without changing +// its result, so the test can cancel while the subsequent block check waits. +type publicationContext struct { + context.Context + once sync.Once + checked chan struct{} +} + +func (c *publicationContext) Err() error { + err := c.Context.Err() + c.once.Do(func() { close(c.checked) }) + return err +} + +func TestSubscriptionPublishCancellationDuringBlockCheck(t *testing.T) { + tx, _, _ := hardeningTxn(t, 9903) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions(), b) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + base, cancel := context.WithCancel(context.Background()) + defer cancel() + ctx := &publicationContext{Context: base, checked: make(chan struct{})} + done := make(chan bool, 1) + l.receiptMu.Lock() + go func() { + done <- s.publish(ctx, Receipt{receipt: hardeningReceipt(b, tx)}, owner) + }() + select { + case <-ctx.checked: + case <-time.After(time.Second): + l.receiptMu.Unlock() + t.Fatal("publication did not check cancellation") + } + cancel() + l.receiptMu.Unlock() + select { + case published := <-done: + if published { + t.Error("receipt published after cancellation during block validation") + } + case <-time.After(time.Second): + t.Fatal("publication did not finish") + } + if len(s.deliveries) != 0 || s.finalizer.hasOwner(owner) { + t.Error("canceled publication recorded delivery or queued finality") + } + hardeningNoReceipt(t, s) +} + +// customWaitFilter reports its own start block and never matches. +type customWaitFilter struct{ Filterer } + +func (customWaitFilter) StartBlockNum() uint64 { return 1 } +func (customWaitFilter) Match(context.Context, Receipt) (bool, error) { return false, nil } + +func TestReceiptsFixCustomFilterMaxWait(t *testing.T) { + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{}, + canonical: make(map[uint64]*types.Block), + blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 100 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99)}); err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, m, hardeningOptions()) + if err != nil { + t.Fatal(err) + } + nobody := common.HexToAddress("0x3333333333333333333333333333333333333333") + custom := customWaitFilter{FilterFrom(nobody).MaxWait(1).(Filterer)} + builtin := FilterFrom(nobody).MaxWait(1) + s := l.Subscribe(custom, builtin) + defer s.Unsubscribe() + hardeningStart(t, l) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor Run did not stop") + } + }) + // One block at a time, so the built-in filter sees blocks after its start. + for num := int64(100); num <= 102; num++ { + p.advance(hardeningBlock(num).Block) + for deadline := time.Now().Add(5 * time.Second); m.LatestBlockNum().Int64() != num; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("monitor did not reach block %d", num) + } + } + } + select { + case <-builtin.(Filterer).Exhausted(): + case <-time.After(5 * time.Second): + t.Fatal("built-in filter did not exhaust") + } + if filters := s.Filters(); len(filters) != 1 || filters[0] != custom { + t.Fatal("custom filter was not left to manage its own MaxWait") + } +} diff --git a/ethreceipts/reorg_test.go b/ethreceipts/reorg_test.go index dd374a26..f697aece 100644 --- a/ethreceipts/reorg_test.go +++ b/ethreceipts/reorg_test.go @@ -611,6 +611,158 @@ func TestReceiptsFixOrphanedSelectionReleasedAtFinality(t *testing.T) { } } +func TestReceiptsReadoptedPendingSurvivesChainAdvance(t *testing.T) { + ctx := context.Background() + tx, _, _ := hardeningTxn(t, 9901) + b := hardeningBlock(100, tx) + var failing atomic.Bool + var calls atomic.Int32 + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if failing.Load() { + return nil, errors.New("temporary receipt provider failure") + } + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), b) + l.invalidateBlock(ctx, b) + l.acceptBlock(b) + if !l.isCurrentBlock(b.Hash(), 1) { + t.Fatal("readopted block did not become current") + } + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + filters := [][]Filterer{s.filterers()} + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{b}, []*subscriber{s}, filters); err != nil { + t.Fatal(err) + } + if len(s.pendingReceipts) != 1 { + t.Fatal("missing pending canonical receipt") + } + // Hold the retry while canonical events advance beyond retention plus + // finality. Receipt backoff is time based and can outlive that block horizon. + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Now().Add(time.Hour) + } + for num := int64(101); num <= 203; num++ { + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{hardeningBlock(num)}, []*subscriber{s}, filters); err != nil { + t.Fatal(err) + } + } + failing.Store(false) + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + } + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{hardeningBlock(204)}, []*subscriber{s}, filters); err != nil { + t.Fatal(err) + } + r := hardeningRead(t, s) + if r.Reorged || r.BlockHash() != b.Hash() || r.Filter != q || r.Receipt() == nil { + t.Fatal("incorrect recovered canonical receipt") + } + if len(s.pendingReceipts) != 0 || calls.Load() != 2 { + t.Fatalf("canonical retry did not recover: pending=%d, origin calls=%d", len(s.pendingReceipts), calls.Load()) + } +} + +func TestReceiptsRejectLateOrphanAfterChainAdvance(t *testing.T) { + tx, _, _ := hardeningTxn(t, 9902) + header := hardeningBlock(100).Header() + header.BlockHash = common.BigToHash(big.NewInt(1_000_100)) + b := ðmonitor.Block{Block: types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: []*types.Transaction{tx}}), Event: ethmonitor.Added, OK: true} + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + close(entered) + select { + case <-release: + return hardeningReceipt(b, tx), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + }} + l := hardeningListener(t, p, hardeningOptions(), b) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + done := make(chan struct{}) + var receipt *types.Receipt + var fetchErr error + t.Cleanup(func() { + cancel() + unblock() + select { + case <-done: + case <-time.After(time.Second): + t.Error("receipt fetch did not stop") + } + }) + go func() { + defer close(done) + receipt, fetchErr = l.fetchTransactionReceipt(ctx, tx.Hash(), false) + }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("receipt RPC did not start") + } + removed := *b + removed.Event = ethmonitor.Removed + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{&removed}, nil, nil); err != nil { + t.Fatal(err) + } + // Replace the orphan and advance past the former pruning horizon while + // the pre-removal RPC is still in flight. + for num := int64(100); num <= 204; num++ { + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{hardeningBlock(num)}, nil, nil); err != nil { + t.Fatal(err) + } + } + unblock() + select { + case <-done: + if receipt != nil || !errors.Is(fetchErr, ethereum.NotFound) { + t.Fatalf("late orphan RPC was accepted: receipt=%v, error=%v", receipt != nil, fetchErr) + } + case <-ctx.Done(): + t.Fatal("receipt RPC did not return") + } + if _, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); found { + t.Error("late orphan receipt entered cache") + } +} + +func TestReceiptsFixBatchMatchesAnyAddedBlock(t *testing.T) { + tx, _, _ := hardeningTxn(t, 810) + mined, empty := hardeningBlock(100, tx), hardeningBlock(101) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(mined, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + matched, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{mined, empty}, []*subscriber{s}, [][]Filterer{s.filterers()}) + if err != nil { + t.Fatal(err) + } + if !matched[0][0] { + t.Error("match in an earlier block of the batch was lost") + } + hardeningRead(t, s) + removed := *mined + removed.Event = ethmonitor.Removed + matched, err = l.processBlocks(context.Background(), ethmonitor.Blocks{&removed}, []*subscriber{s}, [][]Filterer{s.filterers()}) + if err != nil { + t.Fatal(err) + } + if rollback := hardeningRead(t, s); !rollback.Reorged { + t.Fatal("missing rollback receipt") + } + if matched[0][0] { + t.Error("rollback of an earlier match counted as a new match") + } +} + func TestReceiptsFixReadoptionAfterRetention(t *testing.T) { for _, scenario := range []string{"canonical_retention", "obsolete_incarnation", "later_removal", "alternate_retention"} { t.Run(scenario, func(t *testing.T) { diff --git a/ethreceipts/subscription.go b/ethreceipts/subscription.go index c36627f4..b6efd413 100644 --- a/ethreceipts/subscription.go +++ b/ethreceipts/subscription.go @@ -64,9 +64,10 @@ type subscriber struct { pendingReceipts map[receiptKey]*pendingReceipt retryMu sync.Mutex - // deliveryMu serializes publication, rollback, finality and owner retirement, - // and protects deliveries, inFlight and claims. When both locks are needed, - // acquire deliveryMu before retryMu. RPC calls run outside these locks. + // deliveryMu serializes publication, rollback, finality, owner retirement + // and channel sends, and protects deliveries, inFlight and claims. Lock order + // is deliveryMu, retryMu, then the listener's receiptMu. RPC calls run + // outside these locks. deliveryMu sync.Mutex deliveries map[receiptKey]Receipt inFlight map[receiptKey]struct{} @@ -347,12 +348,14 @@ func (s *subscriber) publish(ctx context.Context, receipt Receipt, owner *filter // Caller holds deliveryMu so retry publication and pending ownership commit // together with Remove/Clear and rollback invalidation. func (s *subscriber) publishLocked(ctx context.Context, receipt Receipt, owner *filterOwner) bool { + // deliveryMu, not receiptMu, orders this check against rollback: + // invalidateBlock runs before rollbackBlock, which waits for deliveryMu and + // reports any delivery recorded here. if ctx.Err() != nil || !s.hasFilter(owner) { return false } - s.listener.receiptMu.Lock() - defer s.listener.receiptMu.Unlock() - if ctx.Err() != nil || !s.listener.currentBlock(receipt.BlockHash(), receipt.generation) { + // Block validation can wait for receiptMu; recheck cancellation afterward. + if !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) || ctx.Err() != nil { return false } key := receiptOwner(receipt, owner) @@ -567,15 +570,13 @@ func (s *subscriber) finalizeReceipts(blockNum *big.Int) error { for _, txn := range s.finalizer.dequeue(blockNum) { receipt := txn.receipt owner := receipt.owner - s.listener.receiptMu.Lock() - if !receipt.Reorged && s.listener.currentBlock(receipt.BlockHash(), receipt.generation) { + if !receipt.Reorged && s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { receipt.Final = true s.ch.Send(receipt) if owner != nil && (owner.Cond().TxnHash != nil || owner.Options().LimitOne) { s.retireOwner(owner, true) } } - s.listener.receiptMu.Unlock() if !s.hasFilter(owner) && !s.finalizer.hasOwner(owner) { s.retireOwner(owner, false) } diff --git a/go.mod b/go.mod index 7f279041..6c5bb73b 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/goware/cachestore-mem v0.2.2 github.com/goware/cachestore-redis v0.2.1 github.com/goware/cachestore2 v0.12.3 - github.com/goware/channel v0.5.0 + github.com/goware/channel v0.6.0 github.com/goware/pp v0.0.3 github.com/goware/superr v0.0.2 github.com/holiman/uint256 v1.3.2 diff --git a/go.sum b/go.sum index 258c1801..4696bd83 100644 --- a/go.sum +++ b/go.sum @@ -104,8 +104,8 @@ github.com/goware/cachestore-redis v0.2.1 h1:bMdkzGuy6Dsybq2chv3kRwu8UGoJz3aQBO9 github.com/goware/cachestore-redis v0.2.1/go.mod h1:+8rOAfL1qNLNiXHe8+WACPk+I9kaLOAfArj2Z7FDiWg= github.com/goware/cachestore2 v0.12.3 h1:V4VODChSAV29p8htHj8Lb36Hvv28CLrJsw49gx0h+ks= github.com/goware/cachestore2 v0.12.3/go.mod h1:PR+lXK8UXa/wjKB7mpIj6HtRhC7vbcRXx4b5F1Av/ik= -github.com/goware/channel v0.5.0 h1:cOllKceCH5Xhibs0v8jtPJ81ez3L7WpYri/OU+9IBfg= -github.com/goware/channel v0.5.0/go.mod h1:Eai0KCjphDZ44M/qT7G1ZE6lZfywiTFvwV3Xc6cDPdo= +github.com/goware/channel v0.6.0 h1:eUY5Un8OiUgink+giDPRW0Vd4crdoGOvsNostRckv8s= +github.com/goware/channel v0.6.0/go.mod h1:Eai0KCjphDZ44M/qT7G1ZE6lZfywiTFvwV3Xc6cDPdo= github.com/goware/pp v0.0.3 h1:2Yv0IFGOpVjCDayPYzrqskCe9qmGoKBIyu6Uy//LVUU= github.com/goware/pp v0.0.3/go.mod h1:shID9y83CUGdg/BfO0SrVhchPpIAcT3ArfLVkq3x7tQ= github.com/goware/singleflight v0.3.0 h1:b+OM844fuHzanOlE84WeI+G8YMksUY636v0bdcAfnHE=