diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11046a93..6bf49709 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - go-version: [1.24.x,1.25.x] + go-version: [1.25.x,1.26.x] os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} diff --git a/Makefile b/Makefile index 6621a01d..5697fe25 100644 --- a/Makefile +++ b/Makefile @@ -56,13 +56,14 @@ install: # Run baseline tests test: check-testchain-running go-test +# Packages share one testchain and deployment sender, so serialize package binaries. # Go test short-hand, and skip testing go-ethereum go-test: test-clean - GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -race -run=$(TEST) `go list ./... | grep -v go-ethereum` + GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -p=1 -race -run=$(TEST) `go list ./... | grep -v go-ethereum` # Go test short-hand, including go-ethereum go-test-all: test-clean - GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -run=$(TEST) ./... + GOGC=off go test $(TEST_FLAGS) $(MOD_VENDOR) -p=1 -run=$(TEST) ./... test-clean: GOGC=off go clean -testcache diff --git a/ethmonitor/bootstrap.go b/ethmonitor/bootstrap.go index c4c28f6d..d616c6e2 100644 --- a/ethmonitor/bootstrap.go +++ b/ethmonitor/bootstrap.go @@ -38,8 +38,9 @@ func (c *Chain) bootstrapBlocks(blocks Blocks) error { return nil } - if len(blocks) == 1 { + if len(blocks) == 1 && blocks[0].Event != Added { c.blocks = blocks.Copy() + c.blocks[0].canonicalState = nil return nil } @@ -51,7 +52,7 @@ func (c *Chain) bootstrapBlocks(blocks Blocks) error { for _, b := range blocks { if b.Event == Added { - err := c.push(b) + _, err := c.push(b) if err != nil { return fmt.Errorf("ethmonitor: bootstrap failed to build canonical chain: %w", err) } @@ -101,5 +102,6 @@ func (b *Block) UnmarshalJSON(data []byte) error { b.Event = s.Event b.Logs = s.Logs b.OK = s.OK + b.canonicalState = nil return nil } diff --git a/ethmonitor/cache_test.go b/ethmonitor/cache_test.go new file mode 100644 index 00000000..08be9152 --- /dev/null +++ b/ethmonitor/cache_test.go @@ -0,0 +1,131 @@ +package ethmonitor_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/big" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + memcache "github.com/goware/cachestore-mem" + cachestore "github.com/goware/cachestore2" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// Fail before invoking the getter, as an unreachable shared cache would. +type outageBackend struct { + cachestore.Backend + down atomic.Bool + failures atomic.Int64 +} + +func (b *outageBackend) GetOrSetWithLockEx(ctx context.Context, key string, getter func(context.Context, string) (any, error), ttl time.Duration) (any, error) { + if b.down.Load() { + b.failures.Add(1) + return nil, errors.New("cache unavailable") + } + return b.Backend.GetOrSetWithLockEx(ctx, key, getter, ttl) +} + +// Give each fake block a nonzero bloom and an actual log to verify that +// outage recovery preserves logs, rather than just marking blocks ready. +type cacheTestProvider struct { + *fakeProvider + log types.Log +} + +func (p *cacheTestProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json.RawMessage, error) { + payload, err := p.fakeProvider.RawBlockByNumber(ctx, num) + if err != nil { + return nil, err + } + bloom := types.CreateBloom(&types.Receipt{Logs: []*types.Log{&p.log}}) + return json.RawMessage(strings.Replace(string(payload), strings.Repeat("0", 512), common.Bytes2Hex(bloom[:]), 1)), nil +} + +func (p *cacheTestProvider) RawFilterLogs(ctx context.Context, q ethereum.FilterQuery) (json.RawMessage, error) { + if err := p.chain.wait(ctx); err != nil { + return nil, err + } + block, ok := p.chain.byHash(*q.BlockHash) + if !ok { + return nil, ethereum.NotFound + } + log := p.log + log.BlockHash, log.BlockNumber = block.hash, block.num + return json.Marshal([]types.Log{log}) +} + +func TestMonitorCacheOutageWithLogs(t *testing.T) { + for _, concurrency := range []int{0, 2} { + t.Run(fmt.Sprintf("prefetch=%d", concurrency), func(t *testing.T) { + chain := newFakeChain(1000, 40, 0) + chain.hold(1020) + provider := &cacheTestProvider{ + fakeProvider: &fakeProvider{MockRawInterface: mocks.NewMockRawInterface(gomock.NewController(t)), chain: chain}, + log: types.Log{Address: common.HexToAddress("0x1234"), Topics: []common.Hash{common.HexToHash("0xabcd")}, Data: []byte{1, 2, 3}}, + } + mem, err := memcache.NewBackend(512) + require.NoError(t, err) + backend := &outageBackend{Backend: mem} + backend.down.Store(true) + opts := ethmonitor.DefaultOptions + opts.PollingInterval = 5 * time.Millisecond + opts.Timeout = time.Second + opts.WithLogs = true + opts.BlockRetentionLimit = 5 // Queue capacity is 10; publish 20 blocks while down. + opts.StartBlockNumber = big.NewInt(1000) + opts.PrefetchConcurrency = concurrency + opts.CacheBackend = backend + monitor, err := ethmonitor.NewMonitor(provider, opts) + require.NoError(t, err) + sub := monitor.Subscribe() + defer sub.Unsubscribe() + done := runMonitorForTest(t, monitor) + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + next := uint64(1000) + for next <= 1039 { + select { + case blocks := <-sub.Blocks(): + for _, block := range blocks { + require.Equal(t, next, block.NumberU64()) + require.Equal(t, ethmonitor.Added, block.Event) + require.True(t, block.OK) + require.NotZero(t, block.Bloom()) + log := provider.log + log.BlockHash, log.BlockNumber = block.Hash(), next + require.Equal(t, []types.Log{log}, block.Logs) + next++ + if next == 1020 { + require.True(t, backend.down.Load()) + require.GreaterOrEqual(t, backend.failures.Load(), int64(40)) + backend.down.Store(false) + chain.release(1020) + } + } + case err := <-done: + t.Fatalf("Run returned during cache outage/recovery: %v (head=%v)", err, monitor.LatestBlockNum()) + case <-timer.C: + t.Fatalf("monitor stopped publishing at block %d", next-1) + } + } + // After recovery, ordinary reads fill the cache again without a restart. + key := ethmonitor.CacheKeyBlockLogs(big.NewInt(1), chain.hashAt(1039), nil) + _, found, err := backend.Get(context.Background(), key) + require.NoError(t, err) + require.True(t, found) + require.True(t, monitor.IsRunning()) + }) + } +} diff --git a/ethmonitor/canonical_state_test.go b/ethmonitor/canonical_state_test.go new file mode 100644 index 00000000..e6ca0e64 --- /dev/null +++ b/ethmonitor/canonical_state_test.go @@ -0,0 +1,179 @@ +package ethmonitor + +import ( + "encoding/json" + "fmt" + "math/big" + "testing" + + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/stretchr/testify/require" +) + +func canonicalTestBlock(num int64) *Block { + return &Block{ + Block: types.NewBlockWithHeader(&types.Header{ + Number: big.NewInt(num), + BlockHash: common.BigToHash(big.NewInt(num)), + ParentHash: common.BigToHash(big.NewInt(num - 1)), + Time: uint64(num), + }), + Event: Added, + OK: true, + } +} + +func pushCanonicalTestBlock(t *testing.T, chain *Chain, input *Block) *Block { + t.Helper() + chain.push(input) + block := chain.Head() + require.NotNil(t, block) + require.Equal(t, input.Hash(), block.Hash()) + return block +} + +// The interface keeps the before-fix proof executable without the new method. +func canonicalTestState(t *testing.T, block *Block) (uint64, bool) { + t.Helper() + state, ok := any(block).(interface{ CanonicalState() (uint64, bool) }) + if !ok { + return 0, false + } + return state.CanonicalState() +} + +func TestBlockCanonicalStateRetention(t *testing.T) { + for _, depth := range []int64{1, 3} { + t.Run(fmt.Sprintf("evictionDepth=%d", depth), func(t *testing.T) { + chain := newChain(10, false) + block := pushCanonicalTestBlock(t, chain, canonicalTestBlock(100)) + copy := chain.Blocks().Copy()[0] + incarnation, canonical := canonicalTestState(t, block) + require.Positive(t, incarnation) + require.True(t, canonical) + for num := int64(101); num <= 109+depth; num++ { + pushCanonicalTestBlock(t, chain, canonicalTestBlock(num)) + } + require.Nil(t, chain.GetBlock(block.Hash())) + for _, snapshot := range []*Block{block, copy} { + got, canonical := canonicalTestState(t, snapshot) + require.Equal(t, incarnation, got) + require.True(t, canonical, "retention eviction was mistaken for removal") + } + }) + } +} + +func TestBlockCanonicalStateReadoption(t *testing.T) { + chain := newChain(10, false) + input := canonicalTestBlock(100) + block := pushCanonicalTestBlock(t, chain, input) + shallow := *block + copy := chain.Blocks().Copy()[0] + incarnation, canonical := canonicalTestState(t, block) + require.Positive(t, incarnation) + require.True(t, canonical) + removed := *chain.pop() + removed.Event = Removed + for _, snapshot := range []*Block{block, &shallow, copy, &removed} { + got, canonical := canonicalTestState(t, snapshot) + require.Equal(t, incarnation, got) + require.False(t, canonical, "snapshot did not observe the actual removal") + } + + // Reusing either the original input or a removed snapshot must create a fresh + // owned incarnation without mutating old Added/Removed event copies. + for _, reused := range []*Block{input, copy} { + fresh := pushCanonicalTestBlock(t, chain, reused) + got, canonical := canonicalTestState(t, fresh) + require.Greater(t, got, incarnation) + require.True(t, canonical) + for _, snapshot := range []*Block{block, &shallow, copy, &removed} { + got, canonical := canonicalTestState(t, snapshot) + require.Equal(t, incarnation, got) + require.False(t, canonical, "fresh readoption revived an old event") + } + chain.pop() + } +} + +func TestBlockCanonicalStateBootstrap(t *testing.T) { + for _, count := range []int{1, 3} { + for _, serialized := range []bool{false, true} { + t.Run(fmt.Sprintf("blocks=%d/JSON=%v", count, serialized), func(t *testing.T) { + inputs := make(Blocks, count) + for i := range inputs { + inputs[i] = canonicalTestBlock(int64(100 + i)) + } + chain := newChain(10, true) + if serialized { + data, err := json.Marshal(inputs) + require.NoError(t, err) + require.NoError(t, chain.BootstrapFromBlocksJSON(data)) + } else { + require.NoError(t, chain.BootstrapFromBlocks(inputs)) + } + for _, block := range chain.Blocks().Copy() { + incarnation, canonical := canonicalTestState(t, block) + require.Positive(t, incarnation) + require.True(t, canonical) + data, err := json.Marshal(block) + require.NoError(t, err) + require.NotContains(t, string(data), "incarnation") + require.NoError(t, json.Unmarshal(data, block)) + incarnation, canonical = canonicalTestState(t, block) + require.Zero(t, incarnation, "serialized state preserved runtime ownership") + require.False(t, canonical) + } + }) + } + } +} + +func TestBlockCanonicalStateConcurrentRemoval(t *testing.T) { + chain := newChain(10, false) + input := canonicalTestBlock(100) + block := pushCanonicalTestBlock(t, chain, input) + incarnation, _ := canonicalTestState(t, block) + require.Positive(t, incarnation) + state := any(block).(interface{ CanonicalState() (uint64, bool) }) + start, done := make(chan struct{}), make(chan struct{}) + result := make(chan error, 1) + go func() { + close(start) + removed := false + for { + got, canonical := state.CanonicalState() + if got != incarnation || (removed && canonical) { + result <- fmt.Errorf("old incarnation changed or revived: id=%d canonical=%v", got, canonical) + return + } + removed = removed || !canonical + select { + case <-done: + result <- nil + return + default: + } + } + }() + <-start + for i := 0; i < 100; i++ { + chain.pop() + pushCanonicalTestBlock(t, chain, input) + } + close(done) + require.NoError(t, <-result) + got, canonical := state.CanonicalState() + require.Equal(t, incarnation, got) + require.False(t, canonical) +} + +func TestBlockCanonicalStateUntracked(t *testing.T) { + for _, block := range []*Block{nil, canonicalTestBlock(100)} { + incarnation, canonical := canonicalTestState(t, block) + require.Zero(t, incarnation) + require.False(t, canonical) + } +} diff --git a/ethmonitor/chain.go b/ethmonitor/chain.go index 8685a862..7233b5d0 100644 --- a/ethmonitor/chain.go +++ b/ethmonitor/chain.go @@ -7,6 +7,7 @@ import ( "math/big" "slices" "sync" + "sync/atomic" "github.com/0xsequence/ethkit/go-ethereum" "github.com/0xsequence/ethkit/go-ethereum/common" @@ -28,6 +29,7 @@ type Chain struct { mu sync.RWMutex averageBlockTime float64 // in seconds + lastIncarnation uint64 } func newChain(retentionLimit int, bootstrapMode bool) *Chain { @@ -59,7 +61,7 @@ func newChain(retentionLimit int, bootstrapMode bool) *Chain { // } // Push to the top of the stack -func (c *Chain) push(nextBlock *Block) error { +func (c *Chain) push(nextBlock *Block) (*Block, error) { c.mu.Lock() defer c.mu.Unlock() @@ -70,12 +72,12 @@ func (c *Chain) push(nextBlock *Block) error { // Assert pointing at prev block if nextBlock.ParentHash() != headBlock.Hash() { - return ErrUnexpectedParentHash + return nil, ErrUnexpectedParentHash } // Assert block numbers are in sequence if nextBlock.NumberU64() != headBlock.NumberU64()+1 { - return ErrUnexpectedBlockNumber + return nil, ErrUnexpectedBlockNumber } // Update average block time @@ -86,14 +88,20 @@ func (c *Chain) push(nextBlock *Block) error { } } + // Each adoption owns its state so reusing an input cannot revive old events. + c.lastIncarnation++ + block := *nextBlock + block.canonicalState = &blockCanonicalState{incarnation: c.lastIncarnation} + block.canonicalState.canonical.Store(true) + // Add to head of stack - c.blocks = append(c.blocks, nextBlock) + c.blocks = append(c.blocks, &block) if len(c.blocks) > c.retentionLimit { c.blocks[0] = nil c.blocks = c.blocks[1:] } - return nil + return &block, nil } // Pop from the top of the stack @@ -107,6 +115,9 @@ func (c *Chain) pop() *Block { n := len(c.blocks) - 1 block := c.blocks[n] + if block.canonicalState != nil { + block.canonicalState.canonical.Store(false) + } c.blocks[n] = nil c.blocks = c.blocks[:n] return block @@ -215,6 +226,9 @@ const ( Removed ) +// Block contains a monitored block and its event data. +// Construct values with keyed composite literals: private canonical state makes +// positional literals unsupported. type Block struct { *types.Block @@ -228,6 +242,26 @@ type Block struct { // OK flag which represents the block is ready for broadcasting OK bool + + canonicalState *blockCanonicalState +} + +type blockCanonicalState struct { + incarnation uint64 + canonical atomic.Bool +} + +// CanonicalState reports the monitor-assigned incarnation and whether it has +// remained canonical without a known removal. Zero means the block is untracked. +// Retention eviction preserves this state; in-memory copies share removal updates. +// Incarnations are local to a monitor chain. Serialized blocks are untracked +// until accepted by a monitor. +func (b *Block) CanonicalState() (incarnation uint64, canonical bool) { + if b == nil || b.canonicalState == nil { + return 0, false + } + state := b.canonicalState + return state.incarnation, state.canonical.Load() } type Blocks []*Block @@ -560,10 +594,11 @@ func (blocks Blocks) Copy() Blocks { } nb[i] = &Block{ - Block: b.Block, - Event: b.Event, - Logs: logs, - OK: b.OK, + Block: b.Block, + Event: b.Event, + Logs: logs, + OK: b.OK, + canonicalState: b.canonicalState, } } diff --git a/ethmonitor/ethmonitor.go b/ethmonitor/ethmonitor.go index 8f7bd916..311b61b6 100644 --- a/ethmonitor/ethmonitor.go +++ b/ethmonitor/ethmonitor.go @@ -19,6 +19,7 @@ import ( "github.com/0xsequence/ethkit/go-ethereum/core/types" "github.com/0xsequence/ethkit/util" "github.com/goware/breaker" + memcache "github.com/goware/cachestore-mem" cachestore "github.com/goware/cachestore2" "github.com/goware/channel" "github.com/goware/superr" @@ -41,6 +42,8 @@ var DefaultOptions = Options{ LogTopics: []common.Hash{}, // all logs DebugLogging: false, CacheExpiry: 600 * time.Second, + PrefetchConcurrency: 0, // Prefetching is disabled by default + PrefetchWindow: 0, // 4x PrefetchConcurrency set on init Alerter: util.NoopAlerter(), } @@ -93,7 +96,8 @@ type Options struct { // WithLogs will include logs with the blocks if specified true. WithLogs bool - // LogTopics will filter only specific log topics to include. + // LogTopics selects values for the first topic of each log. Empty filtered + // results are accepted because a nonzero bloom does not prove a match. LogTopics []common.Hash // CacheBackend to use for caching block data @@ -104,6 +108,20 @@ type Options struct { // CacheExpiry is how long to keep each record in cache CacheExpiry time.Duration + // PrefetchConcurrency is the number of workers fetching blocks (and their + // logs, when WithLogs is set) ahead of the monitor while it trails the chain + // head. Prefetched payloads land in the cache, so the monitor's serial loop + // reads them as cache hits instead of paying node round-trips per block. + // Useful on chains whose block rate outpaces a serial fetch. If no + // CacheBackend is set, an in-memory cache is used. 0 disables prefetching. + // A prefetched block may be reorged out before it is read, producing an + // Added then Removed event; subscribers must already handle such reorgs. + PrefetchConcurrency int + + // PrefetchWindow is how many blocks past the monitor's next block the + // prefetcher may fetch. Defaults to 4x PrefetchConcurrency. + PrefetchWindow int + // Alerter config via github.com/goware/alerter Alerter util.Alerter @@ -135,7 +153,16 @@ type Monitor struct { pollInterval atomic.Int64 isStreamingMode atomic.Bool - cache cachestore.Store[[]byte] + // hitStreak counts the run loop's consecutive fetches which found the + // next block without a miss, capped to avoid overflow. + hitStreak atomic.Int32 + + // latestHead bounds prefetch scheduling using the most recent newHeads + // event or, in polling mode, the prefetcher's head poll. + latestHead atomic.Uint64 + + cache cachestore.Store[[]byte] + prefetch *prefetcher publishCh chan Blocks publishQueue *queue @@ -172,6 +199,25 @@ func NewMonitor(provider ethrpc.RawInterface, options ...Options) (*Monitor, err // with slog, we can't modify the log level after the logger is created. } + if opts.PrefetchConcurrency < 0 { + opts.PrefetchConcurrency = 0 + } + if opts.PrefetchConcurrency > 0 && opts.PrefetchWindow <= 0 { + opts.PrefetchWindow = 4 * opts.PrefetchConcurrency + } + + // prefetching hands payloads to the monitor loop through the cache, + // so it needs one even when the caller didn't configure a backend. + if opts.PrefetchConcurrency > 0 && opts.CacheBackend == nil { + // room for a block-by-number and a logs entry per block in the + // window, plus headroom for by-hash lookups during reorgs. + backend, err := memcache.NewBackend(uint32(4*opts.PrefetchWindow + 256)) + if err != nil { + return nil, fmt.Errorf("ethmonitor: creating prefetch cache: %w", err) + } + opts.CacheBackend = backend + } + var cache cachestore.Store[[]byte] if opts.CacheBackend != nil { if opts.CacheExpiry == 0 { @@ -180,7 +226,7 @@ func NewMonitor(provider ethrpc.RawInterface, options ...Options) (*Monitor, err cache = cachestore.OpenStore[[]byte](opts.CacheBackend, cachestore.WithDefaultKeyExpiry(opts.CacheExpiry)) } - return &Monitor{ + m := &Monitor{ options: opts, log: opts.Logger, alert: opts.Alerter, @@ -191,7 +237,11 @@ func NewMonitor(provider ethrpc.RawInterface, options ...Options) (*Monitor, err publishCh: make(chan Blocks), publishQueue: newQueue(opts.BlockRetentionLimit * 2), subscribers: make([]*subscriber, 0), - }, nil + } + if opts.PrefetchConcurrency > 0 { + m.prefetch = newPrefetcher(m, opts.PrefetchConcurrency, opts.PrefetchWindow) + } + return m, nil } func (m *Monitor) lazyInit(ctx context.Context) error { @@ -221,10 +271,19 @@ func (m *Monitor) Run(ctx context.Context) error { return fmt.Errorf("ethmonitor: already running") } - m.ctx, m.ctxStop = context.WithCancel(ctx) + ctx, cancel := context.WithCancel(ctx) + m.ctx, m.ctxStop = ctx, cancel atomic.StoreInt32(&m.running, 1) defer atomic.StoreInt32(&m.running, 0) + var wg sync.WaitGroup + defer func() { + cancel() + wg.Wait() + }() + m.hitStreak.Store(0) + m.latestHead.Store(0) + m.isStreamingMode.Store(false) if err := m.lazyInit(ctx); err != nil { return err @@ -265,7 +324,9 @@ func (m *Monitor) Run(ctx context.Context) error { } // Broadcast published events to all subscribers + wg.Add(1) go func() { + defer wg.Done() defer func() { if r := recover(); r != nil { m.log.Error(fmt.Sprintf("ethmonitor: panic in publish loop: %v - stack: %s", r, string(debug.Stack()))) @@ -289,7 +350,7 @@ func (m *Monitor) Run(ctx context.Context) error { }() // Monitor the chain for canonical representation - err := m.monitor() + err := m.monitor(ctx, &wg) if m.options.UnsubscribeOnStop { m.UnsubscribeAll(err) } @@ -325,13 +386,36 @@ func (m *Monitor) IsStreamingEnabled() bool { func (m *Monitor) IsStreamingMode() bool { return m.isStreamingMode.Load() } -func (m *Monitor) listenNewHead() <-chan uint64 { + +// nextBlockNum returns the number of the next block the run loop will fetch, +// or false while the monitor has not settled on one yet. +func (m *Monitor) nextBlockNum() (uint64, bool) { + m.nextBlockNumberMu.Lock() + defer m.nextBlockNumberMu.Unlock() + if m.nextBlockNumber == nil || !m.nextBlockNumber.IsUint64() { + return 0, false + } + return m.nextBlockNumber.Uint64(), true +} + +// isCatchingUp reports whether the run loop found its next block on at least +// its last two fetches in a row, ie. the chain has been ahead of it. +func (m *Monitor) isCatchingUp() bool { + return m.hitStreak.Load() >= 2 +} + +func (m *Monitor) listenNewHead(ctx context.Context, wg *sync.WaitGroup) <-chan uint64 { ch := make(chan uint64) + // This stream-only head controls listener pacing and resets on reconnect. + // In polling mode it stays zero so each iteration waits for a poll tick; + // m.latestHead separately bounds prefetch scheduling in either mode. var latestHeadBlock atomic.Uint64 nextBlock := make(chan uint64) + wg.Add(2) go func() { + defer wg.Done() defer func() { if r := recover(); r != nil { m.log.Error(fmt.Sprintf("ethmonitor: panic in new head loop: %v - stack: %s", r, string(debug.Stack()))) @@ -376,18 +460,23 @@ func (m *Monitor) listenNewHead() <-chan uint64 { m.isStreamingMode.Store(true) newHeads := make(chan *types.Header) - sub, err := m.provider.SubscribeNewHeads(m.ctx, newHeads) + sub, err := m.provider.SubscribeNewHeads(ctx, newHeads) if err != nil { m.log.Warn(fmt.Sprintf("ethmonitor (chain %s): websocket connect failed: %v", m.chainID.String(), err)) m.alert.Alert(context.Background(), "ethmonitor (chain %s): websocket connect failed: %v", m.chainID.String(), err) - time.Sleep(2000 * time.Millisecond) + select { + case <-ctx.Done(): + close(nextBlock) + return + case <-time.After(2 * time.Second): + } streamingErrLastTime = time.Now() goto reconnect } for { select { - case <-m.ctx.Done(): + case <-ctx.Done(): // if we're done, we'll unsubscribe and close the nextBlock channel sub.Unsubscribe() close(nextBlock) @@ -404,6 +493,10 @@ func (m *Monitor) listenNewHead() <-chan uint64 { case newHead := <-newHeads: latestHeadBlock.Store(newHead.Number.Uint64()) + m.latestHead.Store(newHead.Number.Uint64()) + if m.prefetch != nil { + m.prefetch.notify() + } select { case nextBlock <- newHead.Number.Uint64(): default: @@ -434,7 +527,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { // Polling mode, where we poll for the latest block number select { - case <-m.ctx.Done(): + case <-ctx.Done(): // if we're done, we'll close the nextBlock channel close(nextBlock) retryStreamingTimer.Stop() @@ -443,7 +536,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { case <-time.After(time.Duration(m.pollInterval.Load())): select { case nextBlock <- 0: - case <-m.ctx.Done(): + case <-ctx.Done(): } } } @@ -452,6 +545,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { // The main loop which notifies the monitor to continue to the next block go func() { + defer wg.Done() defer func() { if r := recover(); r != nil { m.log.Error(fmt.Sprintf("ethmonitor: panic in next block loop: %v - stack: %s", r, string(debug.Stack()))) @@ -461,7 +555,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { for { select { - case <-m.ctx.Done(): + case <-ctx.Done(): return default: } @@ -479,20 +573,23 @@ func (m *Monitor) listenNewHead() <-chan uint64 { // waiting on the nextBlock channel select { case ch <- nextBlockNumber: - case <-m.ctx.Done(): + case <-ctx.Done(): return } continue } else { // wait for the next block select { - case <-nextBlock: - case <-m.ctx.Done(): + case _, ok := <-nextBlock: + if !ok { + return + } + case <-ctx.Done(): return } select { case ch <- latestBlockNum: - case <-m.ctx.Done(): + case <-ctx.Done(): return } } @@ -502,8 +599,7 @@ func (m *Monitor) listenNewHead() <-chan uint64 { return ch } -func (m *Monitor) monitor() error { - ctx := m.ctx +func (m *Monitor) monitor(ctx context.Context, wg *sync.WaitGroup) error { events := Blocks{} // minLoopInterval is time we monitor between cycles. It's a fast @@ -512,13 +608,22 @@ func (m *Monitor) monitor() error { minLoopInterval := 5 * time.Millisecond // listen for new heads either via streaming or polling - listenNewHead := m.listenNewHead() + listenNewHead := m.listenNewHead(ctx, wg) + + // fetch blocks ahead of the run loop into the cache while we trail the head + if m.prefetch != nil { + wg.Add(1) + go func() { + defer wg.Done() + m.prefetch.run(ctx) + }() + } // monitor run loop for { select { - case <-m.ctx.Done(): + case <-ctx.Done(): return nil case newHeadNum := <-listenNewHead: @@ -539,8 +644,17 @@ func (m *Monitor) monitor() error { } // fetch the next block, either via the stream or via a poll - nextBlock, nextBlockPayload, miss, err := m.fetchNextBlock(ctx) + nextBlock, nextBlockPayload, miss, err := m.fetchNextBlock(ctx, false) + + // A cached block may be a stale entry from an abandoned fork, written by + // this or a peer monitor even when local prefetching is disabled. Confirm + // a parent mismatch with the node before treating it as a reorg. + if err == nil && m.cache != nil && headBlock != nil && nextBlock.ParentHash() != headBlock.Hash() { + nextBlock, nextBlockPayload, miss, err = m.refetchNextBlock(ctx, nextBlock) + } + if err != nil { + m.hitStreak.Store(0) if errors.Is(err, context.DeadlineExceeded) { m.log.Info(fmt.Sprintf("ethmonitor: fetchNextBlock timed out: '%v', for blockNum:%v, retrying..", err, m.nextBlockNumber)) } else { @@ -556,8 +670,12 @@ func (m *Monitor) monitor() error { // we speed up the polling interval if miss { m.pollInterval.Store(int64(m.options.PollingInterval)) + m.hitStreak.Store(0) } else { m.pollInterval.Store(int64(clampDuration(minLoopInterval, time.Duration(m.pollInterval.Load())/4))) + if m.hitStreak.Load() < 1<<20 { + m.hitStreak.Add(1) + } } // build deterministic set of add/remove events which construct the canonical chain @@ -593,6 +711,11 @@ func (m *Monitor) monitor() error { // clear events sink events = Blocks{} + + // the monitor advanced, which opens up room in the prefetch window + if m.prefetch != nil { + m.prefetch.notify() + } } } } @@ -613,9 +736,11 @@ func (m *Monitor) buildCanonicalChain(ctx context.Context, nextBlock *types.Bloc if headBlock == nil || nextBlock.ParentHash() == headBlock.Hash() { // block-chaining it up - block := &Block{Event: Added, Block: nextBlock} - events = append(events, block) - return events, m.chain.push(block) + block, err := m.chain.push(&Block{Event: Added, Block: nextBlock}) + if err != nil { + return events, err + } + return append(events, block), nil } // next block doest match prevHash, therefore we must pop our previous block and recursively @@ -633,6 +758,11 @@ func (m *Monitor) buildCanonicalChain(ctx context.Context, nextBlock *types.Bloc } } + // blocks prefetched above the popped block may be from the abandoned fork + if m.prefetch != nil { + m.prefetch.reset(ctx, poppedBlock.NumberU64()) + } + if m.options.DebugLogging { m.log.Debug(fmt.Sprintf("ethmonitor: block reorg, reverting block #%d hash:%s prevHash:%s", poppedBlock.NumberU64(), poppedBlock.Hash().Hex(), poppedBlock.ParentHash().Hex())) } @@ -656,8 +786,7 @@ func (m *Monitor) buildCanonicalChain(ctx context.Context, nextBlock *types.Bloc return events, err } - block := &Block{Event: Added, Block: nextBlock} - err = m.chain.push(block) + block, err := m.chain.push(&Block{Event: Added, Block: nextBlock}) if err != nil { return events, err } @@ -691,26 +820,13 @@ func (m *Monitor) addLogs(ctx context.Context, blocks Blocks) { blockHash := block.Hash() - topics := [][]common.Hash{} - if len(m.options.LogTopics) > 0 { - topics = append(topics, m.options.LogTopics) - } - - logs, _, err := m.filterLogs(tctx, blockHash, topics, block.Bloom()) + logs, _, err := m.filterLogs(tctx, blockHash, m.logTopics(), block.Bloom()) if err == nil { - // check the logsBloom from the block to check if we should be expecting logs. logsBloom - // will be included for any indexed logs. - if len(logs) > 0 || block.Bloom() == (types.Bloom{}) { - // successful backfill - if logs == nil { - block.Logs = []types.Log{} - } else { - block.Logs = logs - } - block.OK = true - continue - } + // filterLogs validates emptiness against the bloom and query topics. + block.Logs = logs + block.OK = true + continue } // mark for backfilling @@ -723,7 +839,22 @@ func (m *Monitor) addLogs(ctx context.Context, blocks Blocks) { } } +// logTopics returns the topic filter for block log queries, which is also +// part of the logs cache key. +func (m *Monitor) logTopics() [][]common.Hash { + topics := [][]common.Hash{} + if len(m.options.LogTopics) > 0 { + topics = append(topics, m.options.LogTopics) + } + return topics +} + func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics [][]common.Hash, blockBloom types.Bloom) ([]types.Log, []byte, error) { + var fetchedLogs []types.Log + // The block bloom covers all logs, so unrelated topics may set it even + // when a topic-filtered query correctly returns no logs. + expectLogs := len(topics) == 0 && blockBloom != (types.Bloom{}) + getter := func(ctx context.Context, _ string) ([]byte, error) { if m.options.DebugLogging { m.log.Debug(fmt.Sprintf("ethmonitor: filterLogs is calling origin for block hash %s", blockHash)) @@ -739,12 +870,11 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics if err != nil { return nil, err } - if blockBloom != (types.Bloom{}) && (len(logsPayload) == 0 || (len(logsPayload) == 2 && logsPayload[0] == '[' && logsPayload[1] == ']')) { - // If we have no logs and the block bloom is set, then we need to return an error - // as the node is incorrectly telling us the block-logs response is '[]' but in fact - // the block log bloom filter tells us we should be expecting logs. We do this to - // ensure we do not incorrectly cache an empty block-logs response as valid. - return nil, fmt.Errorf("ethmonitor: filterLogs detected empty block-logs response but block bloom is set, ignoring node response") + // Validate before caching so a malformed response cannot block log + // backfilling until cache expiry. + fetchedLogs, err = m.unmarshalLogs(logsPayload, expectLogs) + if err != nil { + return nil, err } return logsPayload, nil } @@ -754,16 +884,33 @@ func (m *Monitor) filterLogs(ctx context.Context, blockHash common.Hash, topics if err != nil { return nil, resp, err } - logs, err := m.unmarshalLogs(resp) - return logs, resp, err + return fetchedLogs, resp, nil } key := CacheKeyBlockLogs(m.chainID, blockHash, topics) resp, err := m.cache.GetOrSetWithLockEx(ctx, key, getter, m.options.CacheExpiry) if err != nil { - return nil, resp, err + if ctx.Err() != nil { + return nil, resp, ctx.Err() + } + // Keep logs available when the cache fails, just as for block fetches. + // The direct result deliberately bypasses cache reads and writes. + resp, err = getter(ctx, "") + if err != nil { + return nil, resp, err + } + } + if fetchedLogs != nil { + return fetchedLogs, resp, nil + } + logs, err := m.unmarshalLogs(resp, expectLogs) + if err != nil { + // Recover entries cached by peers or older monitors that did not + // validate logs before writing them. + if deleteErr := m.cache.Delete(ctx, key); deleteErr != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting invalid logs cache for block hash %s due to: '%v'", blockHash.Hex(), deleteErr)) + } } - logs, err := m.unmarshalLogs(resp) return logs, resp, err } @@ -806,12 +953,20 @@ func (m *Monitor) backfillChainLogs(ctx context.Context, newBlocks Blocks) { } } -func (m *Monitor) fetchNextBlock(ctx context.Context) (*types.Block, []byte, bool, error) { +func (m *Monitor) fetchNextBlock(ctx context.Context, bypassCache bool) (*types.Block, []byte, bool, error) { miss := false + var fetchedBlock *types.Block + + var nextBlockNumber *big.Int + m.nextBlockNumberMu.Lock() + if m.nextBlockNumber != nil { + nextBlockNumber = big.NewInt(0).Set(m.nextBlockNumber) + } + m.nextBlockNumberMu.Unlock() getter := func(ctx context.Context, _ string) ([]byte, error) { if m.options.DebugLogging { - m.log.Debug(fmt.Sprintf("ethmonitor: fetchNextBlock is calling origin for number %s", m.nextBlockNumber)) + m.log.Debug(fmt.Sprintf("ethmonitor: fetchNextBlock is calling origin for number %s", nextBlockNumber)) } for { select { @@ -820,10 +975,11 @@ func (m *Monitor) fetchNextBlock(ctx context.Context) (*types.Block, []byte, boo default: } - nextBlockPayload, err := m.fetchRawBlockByNumber(ctx, m.nextBlockNumber) + nextBlockPayload, err := m.fetchRawBlockByNumber(ctx, nextBlockNumber) if err != nil { - m.log.Debug(fmt.Sprintf("ethmonitor: [retrying] failed to fetch next block # %d, due to: %v", m.nextBlockNumber, err)) + m.log.Debug(fmt.Sprintf("ethmonitor: [retrying] failed to fetch next block # %d, due to: %v", nextBlockNumber, err)) miss = true + m.hitStreak.Store(0) if m.IsStreamingMode() { // in streaming mode, we'll use a shorter time to pause before we refetch time.Sleep(200 * time.Millisecond) @@ -833,37 +989,68 @@ func (m *Monitor) fetchNextBlock(ctx context.Context) (*types.Block, []byte, boo continue } + // Validate before caching so a bad node response cannot poison this + // block number until cache expiry. Let the run loop retry decode errors. + fetchedBlock, err = m.unmarshalBlock(nextBlockPayload) + if err != nil { + return nil, err + } return nextBlockPayload, nil } } - var nextBlockNumber *big.Int - m.nextBlockNumberMu.Lock() - if m.nextBlockNumber != nil { - nextBlockNumber = big.NewInt(0).Set(m.nextBlockNumber) - } - m.nextBlockNumberMu.Unlock() - - // skip cache if isn't provided, or in case when nextBlockNumber is nil (latest) - if m.cache == nil || nextBlockNumber == nil { + // Reorg confirmation must bypass cache reads and any in-flight cache getter. + if bypassCache || m.cache == nil || nextBlockNumber == nil { resp, err := getter(ctx, "") if err != nil { return nil, resp, miss, err } - block, err := m.unmarshalBlock(resp) - return block, resp, miss, err + return fetchedBlock, resp, miss, nil } // fetch with distributed mutex key := CacheKeyBlockByNumber(m.chainID, nextBlockNumber) resp, err := m.cache.GetOrSetWithLockEx(ctx, key, getter, m.options.CacheExpiry) if err != nil { - return nil, resp, miss, err + if ctx.Err() != nil { + return nil, resp, miss, ctx.Err() + } + // A cache timeout or a shared prefetch error must not interrupt the + // serial fetch. Retry directly with the caller's context, preserving + // the normal wait for a block that the node does not serve yet. + // Deliberately bypass cache writes too; only origin misses slow polling. + m.hitStreak.Store(0) + resp, err = getter(ctx, "") + if err != nil { + return nil, resp, miss, err + } + } + if fetchedBlock != nil { + return fetchedBlock, resp, miss, nil } block, err := m.unmarshalBlock(resp) + if err != nil { + // A peer or an older monitor may have cached an invalid payload. + if deleteErr := m.cache.Delete(ctx, key); deleteErr != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting invalid block cache for block num %s due to: '%v'", nextBlockNumber, deleteErr)) + } + } return block, resp, miss, err } +// refetchNextBlock drops the cached by-number entry for block and confirms the +// next block directly with the node, even if a peer repopulates the cache. +func (m *Monitor) refetchNextBlock(ctx context.Context, block *types.Block) (*types.Block, []byte, bool, error) { + key := CacheKeyBlockByNumber(m.chainID, block.Number()) + if err := m.cache.Delete(ctx, key); err != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting block cache for block num %d due to: '%v'", block.NumberU64(), err)) + } + if m.options.DebugLogging { + m.log.Debug(fmt.Sprintf("ethmonitor: block #%d hash:%s does not extend head, refetching from origin", block.NumberU64(), block.Hash().Hex())) + } + return m.fetchNextBlock(ctx, true) +} + func CacheKeyBlockByNumber(chainID *big.Int, num *big.Int) string { return fmt.Sprintf("ethmonitor:%s:BlockNum:%s", chainID.String(), num.String()) } @@ -913,6 +1100,8 @@ func (m *Monitor) fetchRawBlockByNumber(ctx context.Context, num *big.Int) ([]by cancel() if err != nil { + // Clear the catch-up signal before retries, which may wait at the head. + m.hitStreak.Store(0) if errors.Is(err, ethereum.NotFound) { return nil, ethereum.NotFound } else { @@ -993,7 +1182,15 @@ func (m *Monitor) fetchBlockByHash(ctx context.Context, hash common.Hash) (*type key := CacheKeyBlockByHash(m.chainID, hash) resp, err := m.cache.GetOrSetWithLockEx(ctx, key, getter, m.options.CacheExpiry) if err != nil { - return nil, nil, err + if ctx.Err() != nil { + return nil, nil, ctx.Err() + } + // Reorg ancestry must remain available during a cache outage. + // The direct result deliberately bypasses cache reads and writes. + resp, err = getter(ctx, "") + if err != nil { + return nil, nil, err + } } block, err := m.unmarshalBlock(resp) return block, resp, err @@ -1025,7 +1222,7 @@ func (m *Monitor) publish(ctx context.Context, events Blocks) error { if ok { select { case m.publishCh <- pubEvents: - case <-m.ctx.Done(): + case <-ctx.Done(): } } @@ -1248,11 +1445,20 @@ func (m *Monitor) unmarshalBlock(blockPayload []byte) (*types.Block, error) { return block, nil } -func (m *Monitor) unmarshalLogs(logsPayload []byte) ([]types.Log, error) { +func (m *Monitor) unmarshalLogs(logsPayload []byte, expectLogs bool) ([]types.Log, error) { var logs []types.Log err := json.Unmarshal(logsPayload, &logs) if err != nil { return nil, err } + // Some chains return JSON null instead of [] when there are no logs. + // Preserve the original bloom guard for unfiltered queries: a nonzero + // bloom means logs must be present, even after a successful RPC response. + if len(logs) == 0 && expectLogs { + return nil, fmt.Errorf("ethmonitor: filterLogs detected empty block-logs response but block bloom is set, ignoring node response") + } + if logs == nil { + logs = []types.Log{} + } return logs, nil } diff --git a/ethmonitor/fetch_test.go b/ethmonitor/fetch_test.go new file mode 100644 index 00000000..96d2479e --- /dev/null +++ b/ethmonitor/fetch_test.go @@ -0,0 +1,228 @@ +package ethmonitor + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/big" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + memcache "github.com/goware/cachestore-mem" + cachestore "github.com/goware/cachestore2" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func fetchTestPayload(t *testing.T) json.RawMessage { + t.Helper() + payload, err := json.Marshal(&types.Header{ + Number: big.NewInt(1000), Difficulty: big.NewInt(0), GasLimit: 30_000_000, + }) + require.NoError(t, err) + return payload +} + +func TestMonitorFetchNextBlockCacheTimeout(t *testing.T) { + for _, concurrency := range []int{0, 1} { + t.Run(fmt.Sprintf("prefetch=%d", concurrency), func(t *testing.T) { + // Use the actual memory backend with a shorter timeout than its + // eight-second default. Waiting for a slow head must outlive it. + backend, err := memcache.NewBackend(512, cachestore.WithLockRetryTimeout(25*time.Millisecond)) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + callerDeadline, _ := ctx.Deadline() + var readyAt time.Time + payload := fetchTestPayload(t) + usedCallerDeadline := false + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + provider.EXPECT().RawBlockByNumber(gomock.Any(), big.NewInt(1000)).DoAndReturn( + func(ctx context.Context, _ *big.Int) (json.RawMessage, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if readyAt.IsZero() { + readyAt = time.Now().Add(100 * time.Millisecond) + } + deadline, _ := ctx.Deadline() + usedCallerDeadline = usedCallerDeadline || deadline.Equal(callerDeadline) + if time.Now().Before(readyAt) { + return nil, ethereum.NotFound + } + return payload, nil + }, + ).AnyTimes() + opts := DefaultOptions + opts.CacheBackend = backend + opts.PrefetchConcurrency = concurrency + opts.PollingInterval = time.Millisecond + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID, monitor.nextBlockNumber = big.NewInt(1), big.NewInt(1000) + + block, response, miss, err := monitor.fetchNextBlock(ctx, false) + require.NoError(t, err) + require.Equal(t, uint64(1000), block.NumberU64()) + require.Equal(t, []byte(payload), response) + require.True(t, miss) + require.True(t, usedCallerDeadline, "origin retry inherited the cache's shorter deadline") + }) + } +} + +// Memory-cache singleflight can return another caller's getter error without +// invoking this caller's getter. Inject that failure without relying on timing. +type fetchErrorCache struct { + cachestore.Store[[]byte] + err error + cancel context.CancelFunc +} + +func (c *fetchErrorCache) GetOrSetWithLockEx(context.Context, string, func(context.Context, string) ([]byte, error), time.Duration) ([]byte, error) { + if c.cancel != nil { + c.cancel() + } + return nil, fmt.Errorf("cache getter: %w", c.err) +} + +func TestMonitorFetchNextBlockCacheFailure(t *testing.T) { + for _, tc := range []struct { + name string + cacheErr error + cancel bool + invalidOrigin bool + }{ + {name: "shared_prefetch_not_found", cacheErr: ethereum.NotFound}, + {name: "shared_prefetch_timeout", cacheErr: context.DeadlineExceeded}, + {name: "backend_failure", cacheErr: errors.New("cache unavailable")}, + {name: "caller_canceled", cacheErr: ethereum.NotFound, cancel: true}, + {name: "invalid_origin", cacheErr: ethereum.NotFound, invalidOrigin: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + payload := fetchTestPayload(t) + if tc.invalidOrigin { + payload = json.RawMessage(`{"number":"0x3e8"}`) + } + if !tc.cancel { + provider.EXPECT().RawBlockByNumber(gomock.Any(), big.NewInt(1000)).Return(payload, nil) + } + opts := DefaultOptions + opts.PrefetchConcurrency = 1 + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID, monitor.nextBlockNumber = big.NewInt(1), big.NewInt(1000) + cache := &fetchErrorCache{Store: monitor.cache, err: tc.cacheErr} + if tc.cancel { + cache.cancel = cancel + } + monitor.cache = cache + monitor.hitStreak.Store(2) + + block, response, miss, err := monitor.fetchNextBlock(ctx, false) + switch { + case tc.cancel: + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, block) // The mock also rejects any origin call. + case tc.invalidOrigin: + require.Error(t, err) + require.NotErrorIs(t, err, tc.cacheErr, "origin validation was never attempted") + require.Nil(t, block) + default: + require.NoError(t, err) + require.Equal(t, uint64(1000), block.NumberU64()) + require.Equal(t, []byte(payload), response) + require.False(t, miss, "an immediately available origin block must not slow polling") + require.Zero(t, monitor.hitStreak.Load(), "cache failure left the catch-up signal active") + } + }) + } +} + +func TestMonitorCacheFallback(t *testing.T) { + for _, method := range []string{"logs", "block_by_hash"} { + for _, failure := range []string{"backend_failure", "caller_canceled", "invalid_origin"} { + t.Run(method+"/"+failure, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + deadline, _ := ctx.Deadline() + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + hash := common.HexToHash("0x1000") + log := types.Log{Address: common.HexToAddress("0x1234"), Topics: []common.Hash{common.HexToHash("0xabcd")}, Data: []byte{1}} + payload := fetchTestPayload(t) + if method == "logs" { + var err error + payload, err = json.Marshal([]types.Log{log}) + require.NoError(t, err) + } + if failure == "invalid_origin" { + payload = json.RawMessage(`{}`) + } + if failure != "caller_canceled" { + if method == "logs" { + provider.EXPECT().RawFilterLogs(gomock.Any(), ethereum.FilterQuery{BlockHash: &hash}).DoAndReturn( + func(ctx context.Context, _ ethereum.FilterQuery) (json.RawMessage, error) { + actualDeadline, _ := ctx.Deadline() + require.Equal(t, deadline, actualDeadline) + return payload, nil + }) + } else { + provider.EXPECT().RawBlockByHash(gomock.Any(), hash).DoAndReturn( + func(ctx context.Context, _ common.Hash) (json.RawMessage, error) { + actualDeadline, _ := ctx.Deadline() + require.Equal(t, deadline, actualDeadline) + return payload, nil + }) + } + } + opts := DefaultOptions + opts.PrefetchConcurrency = 1 + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID = big.NewInt(1) + cache := &fetchErrorCache{Store: monitor.cache, err: errors.New("cache unavailable")} + if failure == "caller_canceled" { + cache.cancel = cancel + } + monitor.cache = cache + var response []byte + if method == "logs" { + var logs []types.Log + logs, response, err = monitor.filterLogs(ctx, hash, nil, types.CreateBloom(&types.Receipt{Logs: []*types.Log{&log}})) + if failure == "backend_failure" { + require.Equal(t, []types.Log{log}, logs) + } else { + require.Empty(t, logs) + } + } else { + var block *types.Block + block, response, err = monitor.fetchBlockByHash(ctx, hash) + if failure == "backend_failure" { + require.NotNil(t, block) + require.Equal(t, uint64(1000), block.NumberU64()) + } else { + require.Nil(t, block) + } + } + switch failure { + case "backend_failure": + require.NoError(t, err) + require.Equal(t, []byte(payload), response) + case "caller_canceled": + require.ErrorIs(t, err, context.Canceled) // Mock rejects any origin call. + case "invalid_origin": + require.Error(t, err) + require.NotErrorIs(t, err, cache.err, "origin validation was never attempted") + } + }) + } + } +} diff --git a/ethmonitor/logs_test.go b/ethmonitor/logs_test.go new file mode 100644 index 00000000..305fe354 --- /dev/null +++ b/ethmonitor/logs_test.go @@ -0,0 +1,122 @@ +package ethmonitor + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/0xsequence/ethkit/ethrpc" + "github.com/0xsequence/ethkit/ethrpc/jsonrpc" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/stretchr/testify/require" +) + +func TestMonitorEmptyLogResponses(t *testing.T) { + cases := []struct { + name string + payload string + nonzeroBloom bool + filtered bool + status int + rpcError bool + emptyBody bool + ready bool + }{ + {name: "zero_bloom_array", payload: `[]`, ready: true}, + {name: "zero_bloom_null", payload: `null`, ready: true}, + {name: "zero_bloom_spaced_array", payload: `[ ]`, ready: true}, + {name: "nonzero_bloom_array", payload: `[]`, nonzeroBloom: true}, + {name: "nonzero_bloom_null", payload: `null`, nonzeroBloom: true}, + {name: "nonzero_bloom_spaced_array", payload: `[ ]`, nonzeroBloom: true}, + {name: "filtered_array", payload: `[]`, nonzeroBloom: true, filtered: true, ready: true}, + {name: "filtered_null", payload: `null`, nonzeroBloom: true, filtered: true, ready: true}, + {name: "filtered_spaced_array", payload: `[ ]`, nonzeroBloom: true, filtered: true, ready: true}, + {name: "invalid_object", payload: `{}`}, + {name: "invalid_log", payload: `[{}]`}, + {name: "http_error", payload: `null`, status: http.StatusServiceUnavailable}, + {name: "rpc_error", rpcError: true}, + {name: "empty_http_body", emptyBody: true}, + {name: "missing_rpc_result"}, + } + for _, concurrency := range []int{0, 1} { + for _, tc := range cases { + t.Run(fmt.Sprintf("prefetch=%d/%s", concurrency, tc.name), func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + var request jsonrpc.Message + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + if request.Method != "eth_getLogs" { + t.Errorf("unexpected RPC method %q", request.Method) + } + w.Header().Set("Content-Type", "application/json") + status := tc.status + if status == 0 { + status = http.StatusOK + } + w.WriteHeader(status) + if tc.emptyBody { + return + } + response := jsonrpc.Message{Version: "2.0", ID: request.ID, Result: json.RawMessage(tc.payload)} + if tc.rpcError { + response.Error = &jsonrpc.Error{Code: -32000, Message: "provider not ready"} + } + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Error(err) + } + })) + defer server.Close() + provider, err := ethrpc.NewProvider(server.URL) + require.NoError(t, err) + opts := DefaultOptions + opts.WithLogs = true + opts.PrefetchConcurrency = concurrency + if tc.filtered { + opts.LogTopics = []common.Hash{common.HexToHash("0xabcd")} + } + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID = big.NewInt(1) + hash := common.HexToHash("0x1000") + var bloom types.Bloom + if tc.nonzeroBloom { + bloom = types.BytesToBloom([]byte{1}) // Logs exist, but need not match LogTopics. + } + for attempt := 0; attempt < 2; attempt++ { + block := &Block{ + Block: types.NewBlockWithHeader(&types.Header{Number: big.NewInt(1000), BlockHash: hash, Bloom: bloom}), + Event: Added, + } + monitor.addLogs(context.Background(), Blocks{block}) + require.Equal(t, tc.ready, block.OK, "attempt %d", attempt+1) + if tc.ready { + require.NotNil(t, block.Logs, "accepted null must become an empty log slice") + require.Empty(t, block.Logs) + } + } + wantCalls := int32(2) + if concurrency > 0 && tc.ready { + wantCalls = 1 // Valid empty results are reused; failures remain retryable. + } else if concurrency > 0 { + wantCalls = 4 // Each cache error gets one direct origin retry. + } + require.Equal(t, wantCalls, calls.Load()) + if monitor.cache != nil { + _, found, err := monitor.cache.Get(context.Background(), CacheKeyBlockLogs(monitor.chainID, hash, monitor.logTopics())) + require.NoError(t, err) + require.Equal(t, tc.ready, found, "only valid responses may remain cached") + } + }) + } + } +} diff --git a/ethmonitor/prefetch.go b/ethmonitor/prefetch.go new file mode 100644 index 00000000..5709105f --- /dev/null +++ b/ethmonitor/prefetch.go @@ -0,0 +1,285 @@ +package ethmonitor + +import ( + "context" + "fmt" + "math/big" + "runtime/debug" + "sync" + "sync/atomic" + "time" + + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +// prefetchTick is how often the prefetcher reconsiders its window when +// nothing else wakes it up. +const prefetchTick = 25 * time.Millisecond + +// prefetcher fetches blocks, and their logs when the monitor runs WithLogs, +// ahead of the monitor's run loop while the monitor trails the chain head. +// +// It only fills the cache, using the same keys the run loop reads, so the run +// loop stays the single place the canonical chain is built and validated; +// prefetching only changes where a payload comes from. When the monitor is at +// the head there is nothing past its next block to fetch, and the prefetcher +// is idle. +type prefetcher struct { + m *Monitor + concurrency int + window uint64 + + jobs chan prefetchJob + wake chan struct{} + + // gen is bumped on every reset, so a job scheduled before a reorg can + // tell its payload may belong to the abandoned fork. + gen atomic.Uint64 + + mu sync.Mutex + cursor uint64 // next block number to schedule + highWater uint64 // highest block number scheduled since the last reset +} + +type prefetchJob struct { + num uint64 + gen uint64 +} + +func newPrefetcher(m *Monitor, concurrency, window int) *prefetcher { + return &prefetcher{ + m: m, + concurrency: concurrency, + window: uint64(window), + jobs: make(chan prefetchJob, concurrency), + wake: make(chan struct{}, 1), + } +} + +// notify asks the prefetcher to reconsider its window, without blocking. +func (p *prefetcher) notify() { + select { + case p.wake <- struct{}{}: + default: + } +} + +// run schedules prefetch jobs until ctx is done, and returns once all of its +// workers have exited. +func (p *prefetcher) run(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + p.m.log.Error(fmt.Sprintf("ethmonitor: panic in prefetch loop: %v - stack: %s", r, string(debug.Stack()))) + p.m.alert.Alert(context.Background(), "ethmonitor: panic in prefetch loop: %v", r) + } + }() + + // start from a clean slate, as Run may be called again after a failure + p.mu.Lock() + p.gen.Add(1) + p.cursor, p.highWater = 0, 0 + p.mu.Unlock() + + ctx, cancel := context.WithCancel(ctx) + var wg sync.WaitGroup + defer func() { + cancel() + wg.Wait() + }() + for i := 0; i < p.concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + p.worker(ctx) + }() + } + + ticker := time.NewTicker(prefetchTick) + defer ticker.Stop() + + var lastHeadPoll time.Time + + for { + select { + case <-ctx.Done(): + return + case <-p.wake: + case <-ticker.C: + } + + // in streaming mode the newHeads stream keeps latestHead current, + // in polling mode nothing does, so we ask the node for it. but only + // while the monitor is catching up: at the head there is nothing to + // prefetch, and on a slow chain we'd just double the polling load. + if !p.m.IsStreamingMode() && p.m.isCatchingUp() && time.Since(lastHeadPoll) >= p.m.options.PollingInterval { + lastHeadPoll = time.Now() + p.pollHead(ctx) + } + + p.schedule() + } +} + +func (p *prefetcher) pollHead(ctx context.Context) { + tctx, cancel := context.WithTimeout(ctx, 4*time.Second) + defer cancel() + + head, err := p.m.provider.BlockNumber(tctx) + if err != nil { + p.m.log.Debug(fmt.Sprintf("ethmonitor: prefetch failed to poll head block number: %v", err)) + return + } + p.m.latestHead.Store(head) +} + +// schedule hands the workers the block numbers in the window past the +// monitor's next block, as far as they have room for. +func (p *prefetcher) schedule() { + next, ok := p.m.nextBlockNum() + if !ok { + return + } + + // The run loop fetches `next` itself, so we only look past it. We also + // leave the newest head block to the run loop: it is the one most likely + // not yet served by the node, and the run loop already retries it. + head := p.m.latestHead.Load() + if head < next+2 { + return + } + hi := min(head-1, next+p.window) + + p.mu.Lock() + defer p.mu.Unlock() + + gen := p.gen.Load() + for n := max(p.cursor, next+1); n <= hi; n++ { + select { + case p.jobs <- prefetchJob{num: n, gen: gen}: + p.cursor = n + 1 + p.highWater = max(p.highWater, n) + default: + // workers are busy, carry on from the cursor next time + return + } + } +} + +// reset is called when the run loop pops the block fromNum off the canonical +// chain during a reorg. Any block prefetched above it may belong to the +// abandoned fork, so their by-number cache entries are dropped and scheduling +// resumes from fromNum. +// +// Entries a worker writes after the reset are dropped by the worker itself, +// and any stale entry that slips through is caught by the run loop, which +// confirms a block that does not extend its head with the node before +// treating it as a reorg. +func (p *prefetcher) reset(ctx context.Context, fromNum uint64) { + p.mu.Lock() + p.gen.Add(1) + hi := p.highWater + p.cursor = fromNum + p.highWater = fromNum + p.mu.Unlock() + + for n := fromNum + 1; n <= hi; n++ { + key := CacheKeyBlockByNumber(p.m.chainID, new(big.Int).SetUint64(n)) + if err := p.m.cache.Delete(ctx, key); err != nil { + p.m.log.Warn(fmt.Sprintf("ethmonitor: error deleting prefetched block cache for block num %d due to: '%v'", n, err)) + } + } +} + +func (p *prefetcher) worker(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case job := <-p.jobs: + p.fetch(ctx, job) + p.notify() + } + } +} + +// fetch loads one block, and its logs, into the cache. Failures leave the +// run loop able to fetch from origin when it gets there. +func (p *prefetcher) fetch(ctx context.Context, job prefetchJob) { + defer func() { + if r := recover(); r != nil { + p.m.log.Error(fmt.Sprintf("ethmonitor: panic in prefetch worker: %v - stack: %s", r, string(debug.Stack()))) + p.m.alert.Alert(context.Background(), "ethmonitor: panic in prefetch worker: %v", r) + } + }() + + m := p.m + if job.gen != p.gen.Load() { + return + } + + num := new(big.Int).SetUint64(job.num) + key := CacheKeyBlockByNumber(m.chainID, num) + var block *types.Block + + // NOTE: the getter must not retry, as it runs while holding the lock for the key + getter := func(ctx context.Context, _ string) ([]byte, error) { + if m.options.DebugLogging { + m.log.Debug(fmt.Sprintf("ethmonitor: prefetch is calling origin for number %d", job.num)) + } + tctx, cancel := context.WithTimeout(ctx, m.options.Timeout) + defer cancel() + + payload, err := m.provider.RawBlockByNumber(tctx, num) + if err != nil { + return nil, err + } + if len(payload) == 0 || string(payload) == "null" { + return nil, ethereum.NotFound + } + // Validate even without logs: the serial loop must never inherit an + // undecodable response from a successful cache write. + block, err = m.unmarshalBlock(payload) + if err != nil { + return nil, err + } + return payload, nil + } + + payload, err := m.cache.GetOrSetWithLockEx(ctx, key, getter, m.options.CacheExpiry) + if err != nil { + m.log.Debug(fmt.Sprintf("ethmonitor: prefetch of block %d failed: %v", job.num, err)) + return + } + + if job.gen != p.gen.Load() { + // a reorg reset ran while we were fetching, so this payload may be + // from the abandoned fork + if err := m.cache.Delete(ctx, key); err != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting prefetched block cache for block num %d due to: '%v'", job.num, err)) + } + return + } + + if block == nil { + block, err = m.unmarshalBlock(payload) + if err != nil { + m.log.Debug(fmt.Sprintf("ethmonitor: prefetch failed to decode block %d: %v", job.num, err)) + if deleteErr := m.cache.Delete(ctx, key); deleteErr != nil { + m.log.Warn(fmt.Sprintf("ethmonitor: error deleting invalid block cache for block num %d due to: '%v'", job.num, deleteErr)) + } + return + } + } + + if !m.options.WithLogs { + return + } + + // logs are keyed by block hash, so they stay valid across reorgs + tctx, cancel := context.WithTimeout(ctx, m.options.Timeout) + defer cancel() + if _, _, err := m.filterLogs(tctx, block.Hash(), m.logTopics(), block.Bloom()); err != nil { + m.log.Debug(fmt.Sprintf("ethmonitor: prefetch of logs for block %d failed: %v", job.num, err)) + } +} diff --git a/ethmonitor/prefetch_internal_test.go b/ethmonitor/prefetch_internal_test.go new file mode 100644 index 00000000..ebd43dfe --- /dev/null +++ b/ethmonitor/prefetch_internal_test.go @@ -0,0 +1,203 @@ +package ethmonitor + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestPrefetchRejectsInvalidBlockBeforeCaching(t *testing.T) { + for _, withLogs := range []bool{false, true} { + t.Run(fmt.Sprintf("withLogs=%v", withLogs), func(t *testing.T) { + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + provider.EXPECT().RawBlockByNumber(gomock.Any(), big.NewInt(1000)). + Return(json.RawMessage(`{"number":"0x3e8"}`), nil) + opts := DefaultOptions + opts.PrefetchConcurrency = 1 + opts.WithLogs = withLogs + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID = big.NewInt(1) + + monitor.prefetch.fetch(context.Background(), prefetchJob{num: 1000}) + key := CacheKeyBlockByNumber(monitor.chainID, big.NewInt(1000)) + _, found, err := monitor.cache.Get(context.Background(), key) + require.NoError(t, err) + require.False(t, found, "worker cached an undecodable block") + }) + } +} + +func TestMonitorRecoversInvalidPrefetchedLogs(t *testing.T) { + for _, payload := range []string{`{}`, `[{}]`, `null`, `[]`, `[ ]`} { + for _, cached := range []bool{false, true} { + t.Run(fmt.Sprintf("payload=%s/cached=%v", payload, cached), func(t *testing.T) { + const first, target, last = uint64(1000), uint64(1002), uint64(1004) + targetHash := common.BigToHash(new(big.Int).SetUint64(target)) + expectedLogs := []types.Log{{ + Address: common.HexToAddress("0x1234"), + Topics: []common.Hash{common.HexToHash("0xabcd")}, + Data: []byte{1, 2}, + TxHash: common.HexToHash("0x5678"), + BlockHash: targetHash, + BlockNumber: target, + }} + validPayload, err := json.Marshal(expectedLogs) + require.NoError(t, err) + + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + provider.EXPECT().ChainID(gomock.Any()).Return(big.NewInt(1), nil).AnyTimes() + provider.EXPECT().IsStreamingEnabled().Return(false).AnyTimes() + provider.EXPECT().BlockNumber(gomock.Any()).Return(last, nil).AnyTimes() + provider.EXPECT().RawBlockByNumber(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, number *big.Int) (json.RawMessage, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if number.Uint64() > last { + return nil, ethereum.NotFound + } + var bloom types.Bloom + if number.Uint64() == target { + bloom = types.BytesToBloom([]byte{1}) + } + header := &types.Header{ + Number: number, + ParentHash: common.BigToHash(new(big.Int).Sub(number, big.NewInt(1))), + Difficulty: big.NewInt(0), + GasLimit: 30_000_000, + Bloom: bloom, + } + header.SetHash(common.BigToHash(number)) + return json.Marshal(header) + }, + ).AnyTimes() + var originCalls atomic.Int64 + provider.EXPECT().RawFilterLogs(gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, query ethereum.FilterQuery) (json.RawMessage, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if *query.BlockHash != targetHash { + return json.RawMessage(`[]`), nil + } + if originCalls.Add(1) == 1 && !cached { + return json.RawMessage(payload), nil + } + return json.RawMessage(validPayload), nil + }, + ).AnyTimes() + + opts := DefaultOptions + opts.WithLogs = true + opts.PrefetchConcurrency = 1 + opts.StartBlockNumber = new(big.Int).SetUint64(first) + opts.PollingInterval = 10 * time.Millisecond + opts.CacheExpiry = time.Hour + // Leave CacheBackend unset to exercise automatic memory caching. + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.chainID = big.NewInt(1) + key := CacheKeyBlockLogs(monitor.chainID, targetHash, monitor.logTopics()) + if cached { + require.NoError(t, monitor.cache.SetEx(context.Background(), key, []byte(payload), time.Hour)) + } else { + // Complete the speculative fetch before starting the serial loop, + // so the worker deterministically receives the invalid response. + monitor.prefetch.fetch(context.Background(), prefetchJob{num: target}) + require.Equal(t, int64(2), originCalls.Load(), "invalid response gets one direct origin retry") + _, found, err := monitor.cache.Get(context.Background(), key) + require.NoError(t, err) + require.False(t, found, "neither the invalid response nor the direct retry should be cached") + } + + sub := monitor.Subscribe("TestMonitorRecoversInvalidPrefetchedLogs") + defer sub.Unsubscribe() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- monitor.Run(ctx) }() + defer func() { + cancel() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Error("monitor did not stop") + } + }() + + timer := time.NewTimer(3 * time.Second) + defer timer.Stop() + next := first + for next <= last { + select { + case blocks := <-sub.Blocks(): + for _, block := range blocks { + require.Equal(t, Added, block.Event) + require.True(t, block.OK) + require.Equal(t, next, block.NumberU64(), "blocks must publish in order") + if next == target { + require.Equal(t, expectedLogs, block.Logs) + } + next++ + } + case <-timer.C: + t.Fatalf("publication stalled at block %d; origin calls: %d", next, originCalls.Load()) + } + } + if !cached { + require.GreaterOrEqual(t, originCalls.Load(), int64(2), "invalid prefetch must allow an origin retry") + } else { + require.Positive(t, originCalls.Load(), "invalid cache entry must allow an origin retry") + } + stored, found, err := monitor.cache.Get(context.Background(), key) + require.NoError(t, err) + require.True(t, found) + require.JSONEq(t, string(validPayload), string(stored)) + }) + } + } +} + +func TestPrefetchPanicStopsWorkers(t *testing.T) { + provider := mocks.NewMockRawInterface(gomock.NewController(t)) + provider.EXPECT().BlockNumber(gomock.Any()).DoAndReturn(func(context.Context) (uint64, error) { + panic("simulated head poll panic") + }) + opts := DefaultOptions + opts.PrefetchConcurrency = 4 + monitor, err := NewMonitor(provider, opts) + require.NoError(t, err) + monitor.hitStreak.Store(2) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + monitor.prefetch.run(ctx) + }() + defer func() { + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Error("prefetch did not stop after cancellation") + } + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("prefetch panic did not cancel and join its workers") + } +} diff --git a/ethmonitor/prefetch_test.go b/ethmonitor/prefetch_test.go new file mode 100644 index 00000000..d1713576 --- /dev/null +++ b/ethmonitor/prefetch_test.go @@ -0,0 +1,972 @@ +package ethmonitor_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/big" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/ethmonitor/internal/mocks" + "github.com/0xsequence/ethkit/ethrpc" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/0xsequence/ethkit/go-ethereum/crypto" + memcache "github.com/goware/cachestore-mem" + cachestore "github.com/goware/cachestore2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// TestMonitorPrefetchKeepsUp reproduces a chain producing blocks faster than +// the monitor can fetch them one at a time, and checks prefetching keeps the +// monitor at the head, in both polling and streaming mode. +func TestMonitorPrefetchKeepsUp(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + // 100 blocks/s, with 15ms per node call: a serial fetch of a block and + // its logs manages ~30 blocks/s at best. + const blockInterval = 10 * time.Millisecond + const latency = 15 * time.Millisecond + const runFor = 2 * time.Second + + for _, streaming := range []bool{false, true} { + for _, concurrency := range []int{0, 4, 8} { + t.Run(fmt.Sprintf("streaming=%v/prefetch=%d", streaming, concurrency), func(t *testing.T) { + chain := newFakeChain(1000, 1, latency) + monitor := newTestMonitor(t, chain, streaming, concurrency, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go chain.produce(ctx, blockInterval) + go monitor.Run(ctx) + defer monitor.Stop() + + time.Sleep(runFor) + lag := chain.head() - monitor.LatestBlockNum().Uint64() + t.Logf("head:%d monitor:%d lag:%d blocks", chain.head(), monitor.LatestBlockNum().Uint64(), lag) + + if concurrency == 0 { + // the bug: without prefetching, the monitor falls behind + assert.Greater(t, lag, uint64(60)) + } else { + assert.Less(t, lag, uint64(15)) + } + }) + } + } +} + +// TestMonitorPrefetchReorg reorgs the chain while the prefetcher is ahead of +// the monitor, and checks the monitor ends on the new fork with a consistent +// stream of events. +func TestMonitorPrefetchReorg(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + // prefetched: the reorg starts at a block the prefetcher already cached + // from the abandoned fork, ahead of the monitor head. otherwise it starts + // below the monitor head, which has already published blocks from it. + // + // the window is kept small, as the monitor pauses 2s for every block it + // reverts, and stale prefetched blocks can deepen a reorg up to the window. + for _, concurrency := range []int{2, 4} { + for _, prefetched := range []bool{true, false} { + t.Run(fmt.Sprintf("prefetch=%d/prefetched=%v", concurrency, prefetched), func(t *testing.T) { + testMonitorPrefetchReorg(t, concurrency, prefetched) + }) + } + } +} + +func testMonitorPrefetchReorg(t *testing.T, concurrency int, prefetched bool) { + chain := newFakeChain(1000, 200, 15*time.Millisecond) + backend, err := memcache.NewBackend(1024) + require.NoError(t, err) + cache := cachestore.OpenStore[[]byte](backend) + monitor := newTestMonitor(t, chain, false, concurrency, 4, backend) + + sub := monitor.Subscribe("TestMonitorPrefetchReorg") + defer sub.Unsubscribe() + events := newEventLog() + go func() { + for blocks := range sub.Blocks() { + events.apply(t, blocks) + } + }() + + // To reorg at a block the prefetcher cached from the abandoned fork, the + // monitor must not reach that block first. Rather than racing it, hold + // block 1060 so the monitor stalls before it while the workers fill the + // window past it, then reorg from 1062 and let the monitor continue. + const heldBlock, stalePrefetched = 1060, 1062 + if prefetched { + chain.hold(heldBlock) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go monitor.Run(ctx) + defer monitor.Stop() + + var reorgAt uint64 + if prefetched { + require.Eventually(t, func() bool { + key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(stalePrefetched)) + _, ok, _ := cache.Get(context.Background(), key) + return ok + }, 10*time.Second, time.Millisecond, "block %d was never prefetched", stalePrefetched) + require.Less(t, monitor.LatestBlockNum().Uint64(), uint64(heldBlock)) + reorgAt = stalePrefetched + } else { + require.Eventually(t, func() bool { + return monitor.LatestBlockNum().Uint64() >= 1050 + }, 10*time.Second, time.Millisecond) + reorgAt = monitor.LatestBlockNum().Uint64() - 2 + } + chain.reorgFrom(reorgAt) + t.Logf("reorged chain from block %d, monitor head %d", reorgAt, monitor.LatestBlockNum().Uint64()) + chain.release(heldBlock) + + require.Eventually(t, func() bool { + head := monitor.LatestReadyBlock() + return head != nil && head.Hash() == chain.hashAt(chain.head()) + }, 60*time.Second, 10*time.Millisecond) + + // the retained chain must be the new canonical chain + for _, b := range monitor.Chain().Blocks() { + assert.Equal(t, chain.hashAt(b.NumberU64()), b.Hash(), "block %d", b.NumberU64()) + } + + // and so must the chain subscribers built from the events + require.Eventually(t, func() bool { + return events.head() == chain.hashAt(chain.head()) + }, 5*time.Second, 10*time.Millisecond) + for num, hash := range events.blocks() { + assert.Equal(t, chain.hashAt(num), hash, "subscriber block %d", num) + } + + // both cases publish abandoned-fork blocks before the reorg is seen: in + // the prefetched case the stale block extends the head, so it is accepted + // and must be reverted once the new fork shows up. + assert.Positive(t, events.removedCount(), "the reorg was never exercised") +} + +// TestMonitorPrefetchHeadAhead has the node announce heads it cannot serve +// yet, as when a websocket is ahead of the http node. The prefetcher's misses +// must not wedge the monitor. +func TestMonitorPrefetchHeadAhead(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + chain := newFakeChain(1000, 1, 5*time.Millisecond) + chain.announceAhead = 5 + monitor := newTestMonitor(t, chain, true, 4, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go chain.produce(ctx, 10*time.Millisecond) + go monitor.Run(ctx) + defer monitor.Stop() + + time.Sleep(1500 * time.Millisecond) + assert.Greater(t, monitor.LatestBlockNum().Uint64(), uint64(1050)) +} + +// TestMonitorPrefetchSlowChainNoExtraCalls checks the prefetcher stays out +// of the way of a monitor at the head of a slow chain: workers never ask for +// blocks the chain hasn't made, and in polling mode the head isn't polled. +func TestMonitorPrefetchSlowChainNoExtraCalls(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + type calls struct{ blockNumber, foundBlocks, filterLogs, farAheadBlocks int64 } + + run := func(streaming bool, concurrency int) calls { + chain := newFakeChain(1000, 1, 5*time.Millisecond) + monitor := newTestMonitor(t, chain, streaming, concurrency, 0) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go chain.produce(ctx, 200*time.Millisecond) + go monitor.Run(ctx) + defer monitor.Stop() + + time.Sleep(2 * time.Second) + return calls{ + blockNumber: chain.blockNumberCalls.Load(), + foundBlocks: chain.foundBlockByNumberCalls.Load(), + filterLogs: chain.filterLogsCalls.Load(), + farAheadBlocks: chain.farAheadBlockByNumberCalls.Load(), + } + } + + for _, streaming := range []bool{false, true} { + t.Run(fmt.Sprintf("streaming=%v", streaming), func(t *testing.T) { + off := run(streaming, 0) + on := run(streaming, 4) + t.Logf("prefetch=0: %+v", off) + t.Logf("prefetch=4: %+v", on) + + assert.Zero(t, off.blockNumber) + assert.Zero(t, off.farAheadBlocks) + assert.Zero(t, on.farAheadBlocks, "prefetch asked for a block beyond the next block at the head") + if streaming { + // the stream supplies the head + assert.Zero(t, on.blockNumber) + } else { + // at most the one poll at startup, before the monitor has found + // out it's at the head + assert.LessOrEqual(t, on.blockNumber, int64(1)) + } + + // Compare requests that found blocks. Ordinary polling retries for the + // next missing block depend on timer scheduling and cannot be compared + // across independent runs. Allow jitter in the number of produced blocks. + assert.InDelta(t, off.foundBlocks, on.foundBlocks, 3) + assert.InDelta(t, off.filterLogs, on.filterLogs, 3) + }) + } +} + +func TestMonitorPrefetchShutdownNoGoroutineLeak(t *testing.T) { + if testing.Short() { + t.Skip("Skipping in short mode") + } + + chain := newFakeChain(1000, 100, 5*time.Millisecond) + baseline := runtime.NumGoroutine() + + monitor := newTestMonitor(t, chain, false, 4, 0) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- monitor.Run(ctx) + }() + + time.Sleep(300 * time.Millisecond) + cancel() + + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Monitor.Run() didn't return within timeout") + } + + // Run joins its goroutines; allow runtime cleanup before comparing counts. + // NOTE: poll here rather than with assert.Eventually, whose condition runs + // in a goroutine of its own and so always counts one extra. + deadline := time.Now().Add(2 * time.Second) + for runtime.NumGoroutine() > baseline && time.Now().Before(deadline) { + runtime.GC() + time.Sleep(50 * time.Millisecond) + } + if n := runtime.NumGoroutine(); n > baseline { + buf := make([]byte, 1<<20) + t.Fatalf("%d goroutine(s) leaked:\n%s", n-baseline, buf[:runtime.Stack(buf, true)]) + } +} + +// A peer can complete an old-fork fetch between DEL and the confirmation read, +// and a cache deletion can fail. Neither may cause a false canonical removal. +func TestMonitorPrefetchRefetchBypassesCache(t *testing.T) { + for _, concurrency := range []int{0, 1} { + for _, deleteFails := range []bool{false, true} { + t.Run(fmt.Sprintf("prefetch=%d/deleteFails=%v", concurrency, deleteFails), func(t *testing.T) { + chain := newFakeChain(1000, 20, time.Millisecond) + oldNext, ok := chain.byNumber(big.NewInt(1001)) + require.True(t, ok) + chain.reorgFrom(1000) + + backend, err := memcache.NewBackend(512) + require.NoError(t, err) + key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(1001)) + racingBackend := &repopulatingBackend{ + Backend: backend, key: key, payload: oldNext.payload(), deleteFails: deleteFails, + } + require.NoError(t, racingBackend.SetEx(context.Background(), key, []byte(oldNext.payload()), time.Minute)) + monitor := newTestMonitor(t, chain, false, concurrency, 1, racingBackend) + sub := monitor.Subscribe("TestMonitorPrefetchRefetchBypassesCache") + defer sub.Unsubscribe() + runMonitorForTest(t, monitor) + + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + for { + select { + case blocks := <-sub.Blocks(): + for _, b := range blocks { + require.Equal(t, ethmonitor.Added, b.Event, "removed canonical block %d", b.NumberU64()) + require.Equal(t, chain.hashAt(b.NumberU64()), b.Hash()) + if b.NumberU64() >= 1001 { + require.Positive(t, racingBackend.deletes.Load(), "did not exercise the mismatch confirmation") + return + } + } + case <-timer.C: + t.Fatal("monitor did not advance through the stale cache entry") + } + } + }) + } + } +} + +// Cache confirmation must preserve genuine reorgs, including when prefetching +// is disabled. An uncached monitor still follows the direct-origin path. +func TestMonitorParentMismatchReorg(t *testing.T) { + cases := []struct { + concurrency int + cached bool + }{{0, false}, {0, true}, {1, true}} + for _, tc := range cases { + t.Run(fmt.Sprintf("prefetch=%d/cached=%v", tc.concurrency, tc.cached), func(t *testing.T) { + chain := newFakeChain(1000, 1, time.Millisecond) + oldHash := chain.hashAt(1000) + var monitor *ethmonitor.Monitor + var tracker *repopulatingBackend + if tc.cached { + backend, err := memcache.NewBackend(512) + require.NoError(t, err) + tracker = &repopulatingBackend{Backend: backend, key: ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(1001)), deleteFails: true} + monitor = newTestMonitor(t, chain, false, tc.concurrency, 1, tracker) + } else { + monitor = newTestMonitor(t, chain, false, tc.concurrency, 1) + require.Nil(t, monitor.Options().CacheBackend) + } + sub := monitor.Subscribe("TestMonitorParentMismatchReorg") + defer sub.Unsubscribe() + runMonitorForTest(t, monitor) + state := func(block *ethmonitor.Block) (uint64, bool) { + t.Helper() + witness, ok := any(block).(interface{ CanonicalState() (uint64, bool) }) + require.True(t, ok) + return witness.CanonicalState() + } + var initial *ethmonitor.Block + var oldIncarnation uint64 + select { + case blocks := <-sub.Blocks(): + require.Len(t, blocks, 1) + require.Equal(t, ethmonitor.Added, blocks[0].Event) + require.Equal(t, oldHash, blocks[0].Hash()) + initial = blocks[0] + var canonical bool + oldIncarnation, canonical = state(initial) + require.Positive(t, oldIncarnation) + require.True(t, canonical) + case <-time.After(5 * time.Second): + t.Fatal("missing initial canonical block") + } + chain.reorgFrom(1000) + chain.mu.Lock() + chain.appendLocked() + chain.mu.Unlock() + timer := time.NewTimer(8 * time.Second) + defer timer.Stop() + removed, added := 0, 0 + for { + select { + case blocks := <-sub.Blocks(): + for _, b := range blocks { + if b.Event == ethmonitor.Removed { + require.Equal(t, uint64(1000), b.NumberU64()) + require.Equal(t, oldHash, b.Hash()) + incarnation, canonical := state(b) + require.Equal(t, oldIncarnation, incarnation) + require.False(t, canonical) + _, canonical = state(initial) + require.False(t, canonical, "queued Added did not observe removal") + removed++ + } else { + require.Equal(t, chain.hashAt(b.NumberU64()), b.Hash()) + incarnation, canonical := state(b) + require.Greater(t, incarnation, oldIncarnation) + require.True(t, canonical) + added++ + } + if b.Event == ethmonitor.Added && b.NumberU64() == 1001 { + require.Equal(t, 1, removed) + require.Equal(t, 2, added) + if tracker != nil { + require.Positive(t, tracker.deletes.Load(), "cached parent mismatch was not confirmed") + } + return + } + } + case <-timer.C: + t.Fatal("real reorg did not recover to new canonical block 1001") + } + } + }) + } +} + +func TestMonitorPrefetchInvalidBlockRecovery(t *testing.T) { + for _, withLogs := range []bool{false, true} { + for _, cached := range []bool{false, true} { + t.Run(fmt.Sprintf("withLogs=%v/cached=%v", withLogs, cached), func(t *testing.T) { + chain := newFakeChain(1000, 30, time.Millisecond) + initial := newTestMonitor(t, chain, false, 4, 0) + opts := initial.Options() + opts.WithLogs = withLogs + provider := &invalidBlockProvider{fakeProvider: initial.Provider().(*fakeProvider), target: 1000} + if cached { + key := ethmonitor.CacheKeyBlockByNumber(big.NewInt(1), big.NewInt(1000)) + require.NoError(t, opts.CacheBackend.SetEx(context.Background(), key, []byte(`{"number":"0x3e8"}`), time.Minute)) + // The origin is healthy; only the existing cache entry is invalid. + provider.injected.Store(true) + } else { + // Exercise the automatic memory cache with one invalid origin response. + opts.CacheBackend = nil + } + monitor, err := ethmonitor.NewMonitor(provider, opts) + require.NoError(t, err) + runMonitorForTest(t, monitor) + require.Eventually(t, func() bool { + return monitor.LatestReadyBlock() != nil && monitor.LatestBlockNum().Uint64() == chain.head() + }, 2*time.Second, time.Millisecond) + require.True(t, provider.injected.Load()) + for _, block := range monitor.Chain().Blocks() { + require.Equal(t, chain.hashAt(block.NumberU64()), block.Hash()) + } + }) + } + } +} + +func TestMonitorPrefetchIdleAfterBacklog(t *testing.T) { + chain := newFakeChain(1000, 12, time.Millisecond) + monitor := newTestMonitor(t, chain, false, 4, 0) + provider := &idleHeadProvider{fakeProvider: monitor.Provider().(*fakeProvider), missed: make(chan struct{})} + monitor, err := ethmonitor.NewMonitor(provider, monitor.Options()) + require.NoError(t, err) + runMonitorForTest(t, monitor) + + // Wait for the serial loop's first miss after draining the backlog, rather + // than sampling during the last block's processing. + select { + case <-provider.missed: + case <-time.After(2 * time.Second): + t.Fatal("monitor did not drain the backlog") + } + before := chain.blockNumberCalls.Load() + time.Sleep(150 * time.Millisecond) + // Permit a head poll that was already selected when the miss occurred. + require.LessOrEqual(t, chain.blockNumberCalls.Load()-before, int64(1)) +} + +func TestMonitorPrefetchFatalExitStopsWorkers(t *testing.T) { + chain := newFakeChain(1000, 20, time.Millisecond) + backend, err := memcache.NewBackend(512) + require.NoError(t, err) + initial := newTestMonitor(t, chain, false, 4, 0, &invalidLogsBackend{backend}) + opts := initial.Options() + opts.BlockRetentionLimit = 2 // Four queued events; failed logs prevent dequeue. + monitor, err := ethmonitor.NewMonitor(initial.Provider().(ethrpc.RawInterface), opts) + require.NoError(t, err) + sub := monitor.Subscribe("TestMonitorPrefetchFatalExitStopsWorkers") + defer sub.Unsubscribe() + + for i := 0; i < 2; i++ { + done := runMonitorForTest(t, monitor) + select { + case err := <-done: + require.ErrorIs(t, err, ethmonitor.ErrFatal) + case <-time.After(2 * time.Second): + t.Fatal("expected a fatal publish error") + } + require.False(t, monitor.IsRunning()) + before := chain.blockNumberCalls.Load() + time.Sleep(150 * time.Millisecond) + require.Equal(t, before, chain.blockNumberCalls.Load(), "prefetch continued after Run returned") + } +} + +func TestMonitorPrefetchRestart(t *testing.T) { + for _, streaming := range []bool{false, true} { + t.Run(fmt.Sprintf("streaming=%v", streaming), func(t *testing.T) { + chain := newFakeChain(1000, 10, time.Millisecond) + monitor := newTestMonitor(t, chain, streaming, 4, 0) + ctx, cancel := context.WithCancel(context.Background()) + produced := make(chan struct{}) + go func() { + defer close(produced) + chain.produce(ctx, 20*time.Millisecond) + }() + defer func() { cancel(); <-produced }() + + for i := 0; i < 2; i++ { + target := chain.head() + done := runMonitorForTest(t, monitor) + require.Eventually(t, func() bool { + return monitor.LatestBlockNum().Uint64() >= target + }, 2*time.Second, time.Millisecond) + monitor.Stop() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("monitor did not stop") + } + // Make a gap that the next run must ingest. + require.Eventually(t, func() bool { + return chain.head() >= monitor.LatestBlockNum().Uint64()+3 + }, time.Second, time.Millisecond) + } + }) + } +} + +func runMonitorForTest(t *testing.T, monitor *ethmonitor.Monitor) <-chan error { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- monitor.Run(ctx) + close(done) + }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("monitor did not shut down") + } + }) + return done +} + +type repopulatingBackend struct { + cachestore.Backend + key string + payload []byte + deleteFails bool + once sync.Once + deletes atomic.Int64 +} + +func (b *repopulatingBackend) Delete(ctx context.Context, key string) error { + if key != b.key { + return b.Backend.Delete(ctx, key) + } + b.deletes.Add(1) + if b.deleteFails { + return errors.New("simulated cache deletion failure") + } + if err := b.Backend.Delete(ctx, key); err != nil { + return err + } + b.once.Do(func() { _ = b.Backend.SetEx(ctx, key, b.payload, time.Minute) }) + return nil +} + +type invalidBlockProvider struct { + *fakeProvider + target uint64 + injected atomic.Bool +} + +func (p *invalidBlockProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json.RawMessage, error) { + payload, err := p.fakeProvider.RawBlockByNumber(ctx, num) + if err == nil && num != nil && num.Uint64() == p.target && p.injected.CompareAndSwap(false, true) { + return json.RawMessage(fmt.Sprintf(`{"number":"0x%x"}`, p.target)), nil + } + return payload, err +} + +type idleHeadProvider struct { + *fakeProvider + missed chan struct{} + once sync.Once +} + +func (p *idleHeadProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json.RawMessage, error) { + payload, err := p.fakeProvider.RawBlockByNumber(ctx, num) + if errors.Is(err, ethereum.NotFound) { + p.once.Do(func() { close(p.missed) }) + } + return payload, err +} + +type invalidLogsBackend struct{ cachestore.Backend } + +func (b *invalidLogsBackend) GetOrSetWithLockEx(ctx context.Context, key string, getter func(context.Context, string) (any, error), ttl time.Duration) (any, error) { + if strings.Contains(key, ":Logs:") { + return []byte("invalid logs"), nil + } + return b.Backend.GetOrSetWithLockEx(ctx, key, getter, ttl) +} + +func newTestMonitor(t *testing.T, chain *fakeChain, streaming bool, prefetchConcurrency, prefetchWindow int, cacheBackend ...cachestore.Backend) *ethmonitor.Monitor { + t.Helper() + + provider := &fakeProvider{ + MockRawInterface: mocks.NewMockRawInterface(gomock.NewController(t)), + chain: chain, + streaming: streaming, + } + + opts := ethmonitor.DefaultOptions + opts.PollingInterval = 20 * time.Millisecond + opts.Timeout = 2 * time.Second + opts.WithLogs = true + opts.StartBlockNumber = new(big.Int).SetUint64(chain.base) + opts.PrefetchConcurrency = prefetchConcurrency + opts.PrefetchWindow = prefetchWindow + if len(cacheBackend) > 0 { + opts.CacheBackend = cacheBackend[0] + } + + monitor, err := ethmonitor.NewMonitor(provider, opts) + require.NoError(t, err) + return monitor +} + +// fakeChain is an in-memory chain served with a fixed latency per call. +type fakeChain struct { + base uint64 + latency time.Duration + + // announceAhead makes the chain announce heads this many blocks past the + // blocks it serves. + announceAhead uint64 + + // node calls served, by method + blockNumberCalls atomic.Int64 + blockByNumberCalls atomic.Int64 + foundBlockByNumberCalls atomic.Int64 + farAheadBlockByNumberCalls atomic.Int64 + filterLogsCalls atomic.Int64 + + mu sync.Mutex + canonical []common.Hash // canonical[i] is block base+i + blocks map[common.Hash]fakeBlock + fork int + heads []*fakeSubscription + held map[uint64]chan struct{} // block numbers not served until released +} + +type fakeBlock struct { + num uint64 + hash common.Hash + parent common.Hash +} + +func newFakeChain(base uint64, n int, latency time.Duration) *fakeChain { + c := &fakeChain{base: base, latency: latency, blocks: map[common.Hash]fakeBlock{}} + for i := 0; i < n; i++ { + c.appendLocked() + } + return c +} + +func (c *fakeChain) appendLocked() fakeBlock { + num := c.base + uint64(len(c.canonical)) + b := fakeBlock{ + num: num, + hash: crypto.Keccak256Hash([]byte(fmt.Sprintf("fork:%d/block:%d", c.fork, num))), + } + if len(c.canonical) > 0 { + b.parent = c.canonical[len(c.canonical)-1] + } + c.canonical = append(c.canonical, b.hash) + c.blocks[b.hash] = b + return b +} + +func (c *fakeChain) produce(ctx context.Context, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.mu.Lock() + b := c.appendLocked() + heads := append([]*fakeSubscription{}, c.heads...) + c.mu.Unlock() + + header := &types.Header{Number: new(big.Int).SetUint64(b.num + c.announceAhead)} + for _, sub := range heads { + sub.send(header) + } + } + } +} + +// reorgFrom replaces the canonical chain from block num onwards with a new +// fork of the same length. +func (c *fakeChain) reorgFrom(num uint64) { + c.mu.Lock() + defer c.mu.Unlock() + + n := len(c.canonical) + c.canonical = c.canonical[:num-c.base] + c.fork++ + for len(c.canonical) < n { + c.appendLocked() + } +} + +// hold stops the chain serving block num by number until release(num). +func (c *fakeChain) hold(num uint64) { + c.mu.Lock() + defer c.mu.Unlock() + if c.held == nil { + c.held = map[uint64]chan struct{}{} + } + c.held[num] = make(chan struct{}) +} + +func (c *fakeChain) release(num uint64) { + c.mu.Lock() + defer c.mu.Unlock() + if ch, ok := c.held[num]; ok { + close(ch) + delete(c.held, num) + } +} + +// waitIfHeld blocks while block num is held. +func (c *fakeChain) waitIfHeld(ctx context.Context, num *big.Int) error { + if num == nil { + return nil + } + c.mu.Lock() + ch := c.held[num.Uint64()] + c.mu.Unlock() + if ch == nil { + return nil + } + select { + case <-ch: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (c *fakeChain) head() uint64 { + c.mu.Lock() + defer c.mu.Unlock() + return c.base + uint64(len(c.canonical)) - 1 +} + +func (c *fakeChain) hashAt(num uint64) common.Hash { + c.mu.Lock() + defer c.mu.Unlock() + return c.canonical[num-c.base] +} + +func (c *fakeChain) byNumber(num *big.Int) (fakeBlock, bool) { + c.mu.Lock() + defer c.mu.Unlock() + i := uint64(len(c.canonical) - 1) + if num != nil { + if num.Uint64() < c.base || num.Uint64()-c.base > i { + return fakeBlock{}, false + } + i = num.Uint64() - c.base + } + return c.blocks[c.canonical[i]], true +} + +func (c *fakeChain) byHash(hash common.Hash) (fakeBlock, bool) { + c.mu.Lock() + defer c.mu.Unlock() + b, ok := c.blocks[hash] + return b, ok +} + +func (c *fakeChain) wait(ctx context.Context) error { + select { + case <-time.After(c.latency): + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (b fakeBlock) payload() json.RawMessage { + zero := common.Hash{}.Hex() + return json.RawMessage(fmt.Sprintf(`{`+ + `"number":"0x%x","hash":"%s","parentHash":"%s",`+ + `"sha3Uncles":"%s","miner":"0x0000000000000000000000000000000000000000",`+ + `"stateRoot":"%s","transactionsRoot":"%s","receiptsRoot":"%s",`+ + `"logsBloom":"0x%s","difficulty":"0x0","gasLimit":"0x1c9c380","gasUsed":"0x0",`+ + `"timestamp":"0x%x","extraData":"0x","transactions":[],"uncles":[]}`, + b.num, b.hash.Hex(), b.parent.Hex(), + zero, zero, zero, zero, + strings.Repeat("0", 512), + b.num, + )) +} + +// fakeProvider serves a fakeChain. Methods the monitor is not expected to +// call fall through to the gomock mock, which fails the test. +type fakeProvider struct { + *mocks.MockRawInterface + chain *fakeChain + streaming bool +} + +var _ ethrpc.RawInterface = &fakeProvider{} + +func (p *fakeProvider) ChainID(ctx context.Context) (*big.Int, error) { + return big.NewInt(1), nil +} + +func (p *fakeProvider) IsStreamingEnabled() bool { + return p.streaming +} + +func (p *fakeProvider) BlockNumber(ctx context.Context) (uint64, error) { + p.chain.blockNumberCalls.Add(1) + if err := p.chain.wait(ctx); err != nil { + return 0, err + } + return p.chain.head() + p.chain.announceAhead, nil +} + +func (p *fakeProvider) RawBlockByNumber(ctx context.Context, num *big.Int) (json.RawMessage, error) { + p.chain.blockByNumberCalls.Add(1) + if num != nil && num.Uint64() > p.chain.head()+1 { + p.chain.farAheadBlockByNumberCalls.Add(1) + } + if err := p.chain.wait(ctx); err != nil { + return nil, err + } + if err := p.chain.waitIfHeld(ctx, num); err != nil { + return nil, err + } + b, ok := p.chain.byNumber(num) + if !ok { + return nil, ethereum.NotFound + } + p.chain.foundBlockByNumberCalls.Add(1) + return b.payload(), nil +} + +func (p *fakeProvider) RawBlockByHash(ctx context.Context, hash common.Hash) (json.RawMessage, error) { + if err := p.chain.wait(ctx); err != nil { + return nil, err + } + b, ok := p.chain.byHash(hash) + if !ok { + return nil, ethereum.NotFound + } + return b.payload(), nil +} + +func (p *fakeProvider) RawFilterLogs(ctx context.Context, q ethereum.FilterQuery) (json.RawMessage, error) { + p.chain.filterLogsCalls.Add(1) + if err := p.chain.wait(ctx); err != nil { + return nil, err + } + return json.RawMessage(`[]`), nil +} + +func (p *fakeProvider) SubscribeNewHeads(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) { + sub := &fakeSubscription{ch: ch, err: make(chan error), done: make(chan struct{})} + p.chain.mu.Lock() + p.chain.heads = append(p.chain.heads, sub) + p.chain.mu.Unlock() + return sub, nil +} + +type fakeSubscription struct { + ch chan<- *types.Header + err chan error + done chan struct{} + once sync.Once +} + +func (s *fakeSubscription) send(header *types.Header) { + select { + case s.ch <- header: + case <-s.done: + } +} + +func (s *fakeSubscription) Unsubscribe() { + s.once.Do(func() { close(s.done) }) +} + +func (s *fakeSubscription) Err() <-chan error { + return s.err +} + +// eventLog rebuilds the chain a subscriber sees from the monitor's events, +// checking each event is consistent with what came before. +type eventLog struct { + mu sync.Mutex + chain []fakeBlock + removed int +} + +func newEventLog() *eventLog { + return &eventLog{} +} + +func (l *eventLog) apply(t *testing.T, blocks ethmonitor.Blocks) { + l.mu.Lock() + defer l.mu.Unlock() + + for _, b := range blocks { + switch b.Event { + case ethmonitor.Added: + if n := len(l.chain); n > 0 { + assert.Equal(t, l.chain[n-1].hash, b.ParentHash(), "added block %d does not extend the chain", b.NumberU64()) + } + l.chain = append(l.chain, fakeBlock{num: b.NumberU64(), hash: b.Hash(), parent: b.ParentHash()}) + case ethmonitor.Removed: + n := len(l.chain) + if assert.Greater(t, n, 0, "removed block %d from an empty chain", b.NumberU64()) { + assert.Equal(t, l.chain[n-1].hash, b.Hash(), "removed block %d is not the head", b.NumberU64()) + l.chain = l.chain[:n-1] + } + l.removed++ + } + } +} + +func (l *eventLog) removedCount() int { + l.mu.Lock() + defer l.mu.Unlock() + return l.removed +} + +func (l *eventLog) head() common.Hash { + l.mu.Lock() + defer l.mu.Unlock() + if len(l.chain) == 0 { + return common.Hash{} + } + return l.chain[len(l.chain)-1].hash +} + +func (l *eventLog) blocks() map[uint64]common.Hash { + l.mu.Lock() + defer l.mu.Unlock() + out := make(map[uint64]common.Hash, len(l.chain)) + for _, b := range l.chain { + out[b.num] = b.hash + } + return out +} diff --git a/ethreceipts/ethreceipts.go b/ethreceipts/ethreceipts.go index cd7854f0..273d2466 100644 --- a/ethreceipts/ethreceipts.go +++ b/ethreceipts/ethreceipts.go @@ -37,39 +37,35 @@ var DefaultOptions = Options{ const ( maxFiltersPerListener = 1000 + + // subscriberQueueWarning is how many unread receipts a subscriber can have + // queued before its channel warns and alerts. It is kept low so readers + // falling behind show up quickly. + subscriberQueueWarning = 10 ) type Options struct { - // .. + // MaxConcurrentFetchReceiptWorkers limits concurrent receipt RPC requests. MaxConcurrentFetchReceiptWorkers int - // .. + // MaxConcurrentFilterWorkers limits concurrent block and filter processing. MaxConcurrentFilterWorkers int - // MaxConcurrentSearchOnChainWorkers is the maximum amount of concurrent - // on-chain searches (this is per subscriber) + // MaxConcurrentSearchOnChainWorkers limits on-chain searches per subscriber. MaxConcurrentSearchOnChainWorkers int - // .. + // PastReceiptsCacheSize limits the number of cached receipts. PastReceiptsCacheSize int - // .. + // NumBlocksToFinality is the number of blocks after mining required for finality. + // Values <= 0 select the network's finality policy when Run starts. NumBlocksToFinality int - // FilterMaxWaitNumBlocks is the maximum amount of blocks a filter will wait between getting - // a receipt filter match, before the filter will unsubscribe itself and stop listening. - // This value may be overriden by setting FilterCond#MaxListenNumBlocks on per-filter basis. - // - // NOTE: - // * value of -1 will use NumBlocksToFinality*2 - // * value of 0 will set no limit, so filter will always listen [default] - // * value of N will set the N number of blocks without results before unsubscribing between iterations + // FilterMaxWaitNumBlocks sets the default block wait between filter matches. + // Zero disables the limit. Individual filters can override it with MaxWait. FilterMaxWaitNumBlocks int - // Cache backend ... - // CacheBackend cachestore.Backend - - // Alerter config via github.com/goware/alerter + // Alerter receives listener and subscriber alerts. Alerter util.Alerter } @@ -82,7 +78,7 @@ type ReceiptsListener struct { chainID *big.Int br *breaker.Breaker - // fetchSem is used to limit amount of concurrenct fetch requests + // fetchSem limits concurrent receipt RPC requests. fetchSem chan struct{} // pastReceipts is a cache of past requested receipts @@ -93,15 +89,17 @@ type ReceiptsListener struct { // for us if they end up turning up. notFoundTxnHashes cachestore.Store[uint64] - // ... subscribers []*subscriber registerFiltersCh chan registerFilters filterSem chan struct{} - ctx context.Context - ctxStop context.CancelFunc - running int32 - mu sync.RWMutex + ctxStop context.CancelFunc + running int32 + lifecycleMu sync.Mutex // protects Run/Stop cancellation handle and running transitions + receiptMu sync.Mutex // never held across a channel send or while taking subscriber locks + blockStates map[common.Hash]blockState + reorgRevision uint64 // removal sequence used to reject hash queries started before invalidation + mu sync.RWMutex } var ( @@ -139,12 +137,12 @@ func NewReceiptsListener(log *slog.Logger, provider ethrpc.Interface, monitor *e return nil, err } - notFoundTxnHashes, err := memcache.NewCacheWithSize[uint64](uint32(5000)) //, cachestore.WithDefaultKeyExpiry(2*time.Minute)) + notFoundTxnHashes, err := memcache.NewCacheWithSize[uint64](uint32(5000)) if err != nil { return nil, err } - // max ~12s total wait time before giving up + // Retry transient RPC failures with exponential backoff. br := breaker.New(log, 200*time.Millisecond, 1.2, 20) return &ReceiptsListener{ @@ -160,6 +158,7 @@ func NewReceiptsListener(log *slog.Logger, provider ethrpc.Interface, monitor *e subscribers: make([]*subscriber, 0), registerFiltersCh: make(chan registerFilters, maxFiltersPerListener), filterSem: make(chan struct{}, opts.MaxConcurrentFilterWorkers), + blockStates: make(map[common.Hash]blockState), }, nil } @@ -186,32 +185,43 @@ func (l *ReceiptsListener) lazyInit(ctx context.Context) error { l.options.NumBlocksToFinality = ethrpc.DefaultNumBlocksToFinality } + for _, sub := range l.subscribers { + sub.finalizer.setFinality(l.options.NumBlocksToFinality) + } return nil } func (l *ReceiptsListener) Run(ctx context.Context) error { + l.lifecycleMu.Lock() if l.IsRunning() { + l.lifecycleMu.Unlock() return fmt.Errorf("ethreceipts: already running") } - - l.ctx, l.ctxStop = context.WithCancel(ctx) - + runCtx, cancel := context.WithCancel(ctx) + l.ctxStop = cancel atomic.StoreInt32(&l.running, 1) - defer atomic.StoreInt32(&l.running, 0) - - if err := l.lazyInit(ctx); err != nil { - slog.Error("ethreceipts: lazyInit failed", slog.String("error", err.Error())) + l.lifecycleMu.Unlock() + defer func() { + cancel() + l.lifecycleMu.Lock() + l.ctxStop = nil + atomic.StoreInt32(&l.running, 0) + l.lifecycleMu.Unlock() + }() + if err := l.lazyInit(runCtx); err != nil { return err } - l.log.Info("ethreceipts: running") - - return l.listener() + return l.listener(runCtx) } - func (l *ReceiptsListener) Stop() { l.log.Info("ethreceipts: stop") - l.ctxStop() + l.lifecycleMu.Lock() + cancel := l.ctxStop + l.lifecycleMu.Unlock() + if cancel != nil { + cancel() + } } func (l *ReceiptsListener) IsRunning() bool { @@ -228,7 +238,7 @@ func (l *ReceiptsListener) Subscribe(filterQueries ...FilterQuery) Subscription subscriber := &subscriber{ listener: l, - ch: channel.NewUnboundedChan[Receipt](2, 5000, channel.Options{ + ch: channel.NewUnboundedChan[Receipt](subscriberQueueWarning, 5000, channel.Options{ Logger: l.log, Alerter: l.alert, Label: "ethreceipts:subscriber", @@ -237,11 +247,11 @@ func (l *ReceiptsListener) Subscribe(filterQueries ...FilterQuery) Subscription finalizer: &finalizer{ numBlocksToFinality: big.NewInt(int64(l.options.NumBlocksToFinality)), queue: []finalTxn{}, - txns: map[common.Hash]struct{}{}, + txns: map[receiptKey]struct{}{}, }, } - subscriber.unsubscribe = func() { + subscriber.unsubscribe = sync.OnceFunc(func() { close(subscriber.done) subscriber.ch.Close() subscriber.ch.Flush() @@ -255,7 +265,7 @@ func (l *ReceiptsListener) Subscribe(filterQueries ...FilterQuery) Subscription return } } - } + }) l.subscribers = append(l.subscribers, subscriber) @@ -302,14 +312,22 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFinality(ctx context.Conte func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context, filter FilterQuery, optFilterFinalize ...bool) (*Receipt, WaitReceiptFinalityFunc, error) { // Fetch method searches for just a single filter match. If you'd like to keep the filter // open to listen to many similar receipts, use .Subscribe(filter) directly instead. - query := filter.LimitOne(true).SearchCache(true) - if len(optFilterFinalize) > 0 && optFilterFinalize[0] { - query = query.Finalize(true) - } - - filterer, ok := query.(Filterer) + source, ok := filter.(Filterer) if !ok { - return nil, nil, fmt.Errorf("ethreceipts: unable to cast Filterer from FilterQuery") + // Builders may expose a Filterer only after their normal option chaining. + filter = filter.LimitOne(true).SearchCache(true) + if len(optFilterFinalize) > 0 && optFilterFinalize[0] { + filter = filter.Finalize(true) + } + source, ok = filter.(Filterer) + if !ok { + return nil, nil, fmt.Errorf("ethreceipts: unable to cast Filterer from FilterQuery") + } + } + filterer := snapshotFilter(source) + filterer.LimitOne(true).SearchCache(true) + if len(optFilterFinalize) > 0 && optFilterFinalize[0] { + filterer.Finalize(true) } condMaxWait := 0 @@ -319,23 +337,23 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context condTxnHash := "" if filterer.Cond().TxnHash != nil { condTxnHash = (*filterer.Cond().TxnHash).String() - query = query.QueryOnChainTxnHash(true) + filterer.QueryOnChainTxnHash(true) } - sub := l.Subscribe(query) - - // Use a WaitGroup to ensure the goroutine cleans up before the function returns - var wg sync.WaitGroup + sub := l.Subscribe(filterer) + workerDone := make(chan struct{}) exhausted := make(chan struct{}) mined := make(chan Receipt, 2) finalized := make(chan Receipt, 1) - found := uint32(0) finalityFunc := func(ctx context.Context) (*Receipt, error) { - // Wait for the goroutine to finish its cleanup before proceeding in finalityFunc, - // ensuring Unsubscribe has been called if the goroutine exited. - wg.Wait() + // Wait for the worker's cleanup, including Unsubscribe, before returning finality. + select { + case <-workerDone: + case <-ctx.Done(): + return nil, ctx.Err() + } select { case <-ctx.Done(): return nil, ctx.Err() @@ -355,12 +373,8 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context } } - // TODO/NOTE: perhaps in an extended node failure. could there be a scenario - // where filterer.Exhausted is never hit? and this subscription never unsubscribes..? - // don't think so, but we can double check. - wg.Add(1) go func() { - defer wg.Done() + defer close(workerDone) defer sub.Unsubscribe() defer close(mined) defer close(finalized) @@ -372,6 +386,8 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context } }() + found := false + filterExhausted := filterer.Exhausted() for { select { case <-ctx.Done(): @@ -381,19 +397,14 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // Subscription closed externally (less likely here, but good practice) return - case <-filterer.Exhausted(): - // Exhausted, check if we ever found a match. - if atomic.LoadUint32(&found) == 0 { - // Never found a match, signal exhaustion and exit. - close(exhausted) + case <-filterExhausted: + // Handle the closed exhaustion channel once. Already mined receipts + // keep waiting for finality until the subscription or context closes. + filterExhausted = nil + close(exhausted) + if !found { return } - // Found a match previously, but now exhausted. - // Allow loop to continue briefly to let finalizer potentially finish, - // but the finalized channel will eventually be closed if no final receipt comes. - // The finalityFunc will handle the exhausted state if needed. - // We signal exhaustion mainly for the initial return value check. - close(exhausted) case receipt, ok := <-sub.TransactionReceipt(): if !ok { @@ -401,7 +412,9 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context return } - atomic.StoreUint32(&found, 1) + found = true + // Helper state stays private; public receipts retain the resolved source filter. + receipt.Filter = source if receipt.Final { // Send to mined (in case caller only waits for mined) @@ -438,19 +451,17 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // Wait for the first mined receipt or an exit signal select { case <-ctx.Done(): - wg.Wait() // Ensure cleanup + <-workerDone return nil, nil, ctx.Err() case <-sub.Done(): - wg.Wait() // Ensure cleanup + <-workerDone return nil, nil, ErrSubscriptionClosed case <-exhausted: - // Exhausted before finding *any* receipt. - // finalityFunc will handle waiting and returning the exhaustion error. return nil, finalityFunc, superr.Wrap(ErrFilterExhausted, fmt.Errorf("txnHash=%s maxWait=%d", condTxnHash, condMaxWait)) case receipt, ok := <-mined: if !ok { // Mined channel closed without sending, implies goroutine exited early. - wg.Wait() // Ensure cleanup + <-workerDone // Check if exhaustion occurred select { case <-exhausted: @@ -459,8 +470,6 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context return nil, nil, ErrSubscriptionClosed } } - // Got the first mined receipt. Return it and the finality func. - // The finalityFunc will use wg.Wait() internally. return &receipt, finalityFunc, nil } } @@ -468,12 +477,17 @@ func (l *ReceiptsListener) FetchTransactionReceiptWithFilter(ctx context.Context // fetchTransactionReceipt from the rpc provider, up to some amount of concurrency. When forceFetch is passed, // it indicates that we have high conviction that the receipt should be available, as the monitor has found // this transaction hash. -func (l *ReceiptsListener) fetchTransactionReceipt(ctx context.Context, txnHash common.Hash, forceFetch bool) (*types.Receipt, error) { +func (l *ReceiptsListener) fetchTransactionReceipt(ctx context.Context, txnHash common.Hash, forceFetch bool, expectedBlock ...blockRef) (*types.Receipt, error) { + l.receiptMu.Lock() + started := l.reorgRevision + l.receiptMu.Unlock() timeStart := time.Now() for { select { case l.fetchSem <- struct{}{}: goto start + case <-ctx.Done(): + return nil, ctx.Err() case <-time.After(1 * time.Minute): elapsed := time.Since(timeStart) l.log.Warn(fmt.Sprintf("fetchTransactionReceipt(%s) waiting for fetch semaphore for %s", txnHash.String(), elapsed)) @@ -496,7 +510,27 @@ start: txnHashHex := txnHash.String() + expected := blockRef{} + if len(expectedBlock) > 0 { + expected = expectedBlock[0] + } + l.receiptMu.Lock() + if expected.hash != (common.Hash{}) && !l.currentBlock(expected.hash, expected.generation) { + l.receiptMu.Unlock() + errCh <- ethereum.NotFound + return + } receipt, ok, _ := l.pastReceipts.Get(ctx, txnHashHex) + if ok && l.blockStates[receipt.BlockHash].removed { + ok = false + l.pastReceipts.Delete(ctx, txnHashHex) + } + if ok && !l.validFetchedBlock(receipt.BlockHash, expected, started) { + l.receiptMu.Unlock() + errCh <- ethereum.NotFound + return + } + l.receiptMu.Unlock() if ok { resultCh <- receipt return @@ -546,9 +580,26 @@ start: return superr.Wrap(fmt.Errorf("failed to fetch receipt %s", txnHash), err) } - l.pastReceipts.Set(ctx, txnHashHex, receipt) - l.notFoundTxnHashes.Delete(ctx, txnHashHex) - + if receipt == nil || receipt.TxHash != txnHash || receipt.BlockNumber == nil { + return fmt.Errorf("ethreceipts: invalid receipt for txn %s", txnHash) + } + l.receiptMu.Lock() + valid := l.validFetchedBlock(receipt.BlockHash, expected, started) + // A stale provider response does not invalidate a current monitored candidate. + retryable := !valid && expected.hash != (common.Hash{}) && l.currentBlock(expected.hash, expected.generation) + if valid { + l.pastReceipts.Set(ctx, txnHashHex, receipt) + l.notFoundTxnHashes.Delete(ctx, txnHashHex) + } + l.receiptMu.Unlock() + if !valid { + if retryable { + errCh <- fmt.Errorf("ethreceipts: stale receipt for txn %s in block %s, expected current block %s", txnHash, receipt.BlockHash, expected.hash) + } else { + errCh <- ethereum.NotFound + } + return nil + } resultCh <- receipt return nil }) @@ -568,15 +619,25 @@ start: } } -func (l *ReceiptsListener) listener() error { +func (l *ReceiptsListener) listener(runCtx context.Context) error { + runCtx, cancel := context.WithCancel(runCtx) + defer cancel() monitor := l.monitor.Subscribe("ethreceipts") defer monitor.Unsubscribe() - latestBlockNum := l.LatestBlockNum().Uint64() + g, ctx := errgroup.WithContext(runCtx) + // Monitor closure must cancel RPC work even while both processing loops wait. + g.Go(func() error { + select { + case <-monitor.Done(): + cancel() + case <-ctx.Done(): + } + return nil + }) + latestBlockNum := l.latestBlockNum(ctx).Uint64() l.log.Debug(fmt.Sprintf("latestBlockNum %d", latestBlockNum)) - g, ctx := errgroup.WithContext(l.ctx) - // Listen on filter registration to search cached and on-chain receipts // At filter subscription registration time, we first search our local cache // of recent blocks retained by the monitor, and then we query on-chain for @@ -591,6 +652,7 @@ func (l *ReceiptsListener) listener() error { return nil case <-monitor.Done(): + cancel() l.log.Info("ethreceipts: receipt listener is stopped because monitor signaled its stopping") return nil @@ -623,7 +685,7 @@ func (l *ReceiptsListener) listener() error { // Search our local blocks cache from monitor retention list, and notify subscriber // of any matches found by publishing receipts. - matchedList, err := l.processBlocks(blocks, []*subscriber{reg.subscriber}, [][]Filterer{filters}) + matchedList, err := l.processCachedBlocks(ctx, blocks, []*subscriber{reg.subscriber}, [][]Filterer{filters}) if err != nil { l.log.Warn(fmt.Sprintf("ethreceipts: failed to process blocks during new filter registration: %v", err)) } @@ -649,6 +711,7 @@ func (l *ReceiptsListener) listener() error { return nil case <-monitor.Done(): + cancel() l.log.Info("ethreceipts: receipt listener is stopped because monitor signaled its stopping") return nil @@ -658,19 +721,15 @@ func (l *ReceiptsListener) listener() error { continue } - latestBlockNum = l.LatestBlockNum().Uint64() + latestBlockNum = l.latestBlockNum(ctx).Uint64() // pass blocks across filters of subscribers l.mu.Lock() - if len(l.subscribers) == 0 { - l.mu.Unlock() - continue - } subscribers := make([]*subscriber, len(l.subscribers)) copy(subscribers, l.subscribers) filters := make([][]Filterer, len(l.subscribers)) for i := 0; i < len(subscribers); i++ { - filters[i] = subscribers[i].Filters() + filters[i] = subscribers[i].filterers() } l.mu.Unlock() @@ -683,13 +742,7 @@ func (l *ReceiptsListener) listener() error { l.notFoundTxnHashes.Delete(ctx, txn.Hash().Hex()) } case ethmonitor.Removed: - // delete past receipts of removed blocks reorg = true - for _, txn := range block.Transactions() { - txnHashHex := txn.Hash().Hex() - l.pastReceipts.Delete(ctx, txnHashHex) - l.notFoundTxnHashes.Delete(ctx, txnHashHex) - } } } @@ -697,7 +750,7 @@ func (l *ReceiptsListener) listener() error { if reorg { for _, list := range filters { for _, filterer := range list { - if f, ok := filterer.(*filter); ok { + if f := builtinFilter(filterer); f != nil { f.setStartBlockNum(latestBlockNum) f.setLastMatchBlockNum(0) } @@ -706,7 +759,7 @@ func (l *ReceiptsListener) listener() error { } // Match blocks against subscribers[i] X filters[i][..] - matchedList, err := l.processBlocks(blocks, subscribers, filters) + matchedList, err := l.processBlocks(ctx, blocks, subscribers, filters) if err != nil { l.log.Warn(fmt.Sprintf("ethreceipts: failed to process blocks: %v", err)) } @@ -716,7 +769,7 @@ func (l *ReceiptsListener) listener() error { for y, matched := range list { filterer := filters[x][y] if matched || filterer.StartBlockNum() == 0 { - if f, ok := filterer.(*filter); ok { + if f := builtinFilter(filterer); f != nil { if f.StartBlockNum() == 0 { f.setStartBlockNum(latestBlockNum) } @@ -725,25 +778,24 @@ func (l *ReceiptsListener) listener() error { } } } else { - // NOTE: even if a filter is exhausted, the finalizer will still run - // for those transactions which were previously mined and marked by the finalizer. - // Therefore, the code below will not impact the functionality of the finalizer. + // Exhaustion stops matching but preserves queued finality. maxWait := l.getMaxWaitBlocks(filterer.Options().MaxWait) blockNum := max(filterer.StartBlockNum(), filterer.LastMatchBlockNum()) if maxWait != 0 && (latestBlockNum-blockNum) >= maxWait { - f, _ := filterer.(*filter) + f := builtinFilter(filterer) if f == nil { - panic("ethreceipts: unexpected") + // A custom Filterer keeps its own counters and exhaustion signal. + continue } if (f.Options().LimitOne && f.LastMatchBlockNum() == 0) || !f.Options().LimitOne { l.log.Debug(fmt.Sprintf("filter exhausted! last block matched:%d maxWait:%d filterID:%d", filterer.LastMatchBlockNum(), maxWait, filterer.FilterID())) subscriber := subscribers[x] - subscriber.RemoveFilter(filterer) + subscriber.exhaustFilter(filterer) - if f, ok := filterer.(*filter); ok { + if f := builtinFilter(filterer); f != nil { f.closeExhausted() } } @@ -755,40 +807,68 @@ func (l *ReceiptsListener) listener() error { } }) - // TODO/NOTE: perhaps in an extended node failure. could there be a scenario - // where filterer.Exhausted is never hit? and this subscription never unsubscribes..? - // TODO: we ultimately need to check the monitor and if we get no new blocks for a period - // of time, then we can assume node problems.. even more helpful woudl be if the monitor - // gave us an error count of node failures, and we'd listen on that, and if we hit a threshold - // and our block number doesn't change after a period of time, then we return an error - // that we're exhausted due to a node failure. + // MaxWait counts blocks; callers should use a context deadline to bound stalls. return g.Wait() } // processBlocks attempts to match blocks against subscriber[i] X filterers[i].. list of filters. There is // a corresponding list of filters[i] for each subscriber[i]. -func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { +func (l *ReceiptsListener) processBlocks(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { + return l.processBlockEvents(ctx, blocks, subscribers, filterers, true) +} + +// Cached snapshots may match receipts, but cannot authorize canonical re-adoption. +func (l *ReceiptsListener) processCachedBlocks(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer) ([][]bool, error) { + return l.processBlockEvents(ctx, blocks, subscribers, filterers, false) +} +func (l *ReceiptsListener) processBlockEvents(ctx context.Context, blocks ethmonitor.Blocks, subscribers []*subscriber, filterers [][]Filterer, canonicalEvents bool) ([][]bool, error) { // oks is the 'ok' match of the filterers [][]Filterer results oks := make([][]bool, len(filterers)) for i, f := range filterers { oks[i] = make([]bool, len(f)) } + // Invalidate every rollback before any retry, addition or finality advancement. + removed := make(map[common.Hash]blockRef) + for _, block := range blocks { + if block.Event == ethmonitor.Removed { + ref := l.invalidateBlock(ctx, block) + removed[block.Hash()] = ref + for _, sub := range subscribers { + sub.rollbackBlock(ref) + } + } + } if len(subscribers) == 0 || len(filterers) == 0 { + if canonicalEvents { + for _, block := range blocks { + if block.Event == ethmonitor.Added { + l.acceptBlock(block) + } + } + } return oks, nil } - // check each block against each subscriber X filter for _, block := range blocks { + if ctx.Err() != nil { + return oks, ctx.Err() + } // report if the txn was removed reorged := block.Event == ethmonitor.Removed + if canonicalEvents && !reorged { + l.acceptBlock(block) + } + generation := l.blockGeneration(block.Hash()) + if reorged { + generation = removed[block.Hash()].generation + } - // TODOXXX: feels wasteful to build all receipts for all subscribers every time, but its okay for now receipts := make([]Receipt, len(block.Transactions())) logs := groupLogsByTransaction(block.Logs) - // build unfiltered complete receipts for each txn which include the transaction and the logs + // Build transaction candidates with the logs from the monitored block. for i, txn := range block.Transactions() { txnLog, ok := logs[txn.Hash().Hex()] if !ok { @@ -800,6 +880,9 @@ func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers [ logs: txnLog, chainID: l.chainID, transaction: txn, + blockNum: block.Number(), + blockHash: block.Hash(), + generation: generation, } } @@ -807,8 +890,12 @@ func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers [ // and if there is a match notify the subscriber with the receipts. var wg sync.WaitGroup for i, sub := range subscribers { - l.filterSem <- struct{}{} - + select { + case l.filterSem <- struct{}{}: + case <-ctx.Done(): + wg.Wait() + return oks, ctx.Err() + } wg.Add(1) go func(i int, sub *subscriber) { defer func() { @@ -817,21 +904,30 @@ func (l *ReceiptsListener) processBlocks(blocks ethmonitor.Blocks, subscribers [ }() // retry pending receipts first - retryCtx, cancel := context.WithTimeout(l.ctx, 5*time.Second) - sub.retryPendingReceipts(retryCtx) // TODOXXXPETER: what is this pending receipts thing..? hmpf.. - cancel() + if !reorged { + retryCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + sub.retryPendingReceipts(retryCtx) + cancel() + } // filter matcher and notify subscriber of receipts if matched - matched, err := sub.matchFiltersAndPublish(l.ctx, filterers[i], receipts) + matched, err := sub.matchFiltersAndPublish(ctx, filterers[i], receipts) if err != nil { l.log.Warn(fmt.Sprintf("error while processing filters: %s", err)) } - oks[i] = matched + // A match in any added block of the batch counts. Rolling back an + // earlier match is not a new one. + if !reorged { + for j, ok := range matched { + oks[i][j] = oks[i][j] || ok + } + } // check subscriber to finalize any receipts - err = sub.finalizeReceipts(block.Number()) - if err != nil { - l.log.Error(fmt.Sprintf("finalizeReceipts failed: %v", err)) + if !reorged && ctx.Err() == nil { + if err = sub.finalizeReceipts(block.Number()); err != nil { + l.log.Error(fmt.Sprintf("finalizeReceipts failed: %v", err)) + } } }(i, sub) } @@ -885,6 +981,9 @@ func (l *ReceiptsListener) queryFilterOnChain(ctx context.Context, subscriber *s return gctx.Err() } + l.receiptMu.Lock() + started := l.reorgRevision + l.receiptMu.Unlock() var r *types.Receipt var err error @@ -909,15 +1008,24 @@ func (l *ReceiptsListener) queryFilterOnChain(ctx context.Context, subscriber *s return nil } + l.receiptMu.Lock() + valid := l.validFetchedBlock(r.BlockHash, blockRef{}, started) + generation := l.blockStates[r.BlockHash].generation + l.receiptMu.Unlock() + if !valid { + return nil + } // Found the receipt, update last match block num and continue - if f, ok := item.filterer.(*filter); ok { - f.setLastMatchBlockNum(r.BlockNumber.Uint64()) + if f := builtinFilter(item.filterer); f != nil { + if r.BlockNumber != nil { + f.setLastMatchBlockNum(r.BlockNumber.Uint64()) + } } receipt := Receipt{ - receipt: r, + receipt: r, + generation: generation, // NOTE: we do not include the transaction at this point, as we don't have it. - // transaction: txn, Final: l.isBlockFinal(r.BlockNumber), } @@ -954,7 +1062,7 @@ func (l *ReceiptsListener) getMaxWaitBlocks(maxWait *int) uint64 { } func (l *ReceiptsListener) isBlockFinal(blockNum *big.Int) bool { - latestBlockNum := l.LatestBlockNum() + latestBlockNum := l.monitor.LatestBlockNum() if latestBlockNum == nil || blockNum == nil { return false } @@ -963,10 +1071,12 @@ func (l *ReceiptsListener) isBlockFinal(blockNum *big.Int) bool { l.mu.RLock() defer l.mu.RUnlock() - return diff.Cmp(big.NewInt(int64(l.options.NumBlocksToFinality))) >= 0 + return l.options.NumBlocksToFinality > 0 && diff.Cmp(big.NewInt(int64(l.options.NumBlocksToFinality))) >= 0 } -func (l *ReceiptsListener) LatestBlockNum() *big.Int { +func (l *ReceiptsListener) LatestBlockNum() *big.Int { return l.latestBlockNum(context.Background()) } + +func (l *ReceiptsListener) latestBlockNum(ctx context.Context) *big.Int { // return immediately if the monitor has a latest block number latestBlockNum := l.monitor.LatestBlockNum() if latestBlockNum != nil && latestBlockNum.Cmp(big.NewInt(0)) > 0 { @@ -977,7 +1087,11 @@ func (l *ReceiptsListener) LatestBlockNum() *big.Int { maxWaitTime := 30 * time.Second period := 250 * time.Millisecond for { - time.Sleep(period) + select { + case <-time.After(period): + case <-ctx.Done(): + return big.NewInt(0) + } latestBlockNum := l.monitor.LatestBlockNum() if latestBlockNum != nil && latestBlockNum.Cmp(big.NewInt(0)) > 0 { @@ -994,24 +1108,23 @@ func (l *ReceiptsListener) LatestBlockNum() *big.Int { func getChainID(ctx context.Context, provider ethrpc.Interface) (*big.Int, error) { var chainID *big.Int - - // provide plenty of time for breaker to succeed err := breaker.Do(ctx, func() error { - ctx, cancel := context.WithTimeout(ctx, 4*time.Second) + requestCtx, cancel := context.WithTimeout(ctx, 4*time.Second) defer cancel() - id, err := provider.ChainID(ctx) + id, err := provider.ChainID(requestCtx) if err != nil { return err } chainID = id return nil - }, nil, 1*time.Second, 2, 10) - + }, nil, time.Second, 2, 10) if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } return nil, err } - return chainID, nil } @@ -1025,20 +1138,6 @@ func collectOk[T any](in []T, oks []bool, okCond bool) []T { return out } -// func txnLogs(blockLogs []types.Log, txnHash ethkit.Hash) []*types.Log { -// txnLogs := []*types.Log{} -// for i, log := range blockLogs { -// if log.TxHash == txnHash { -// log := log // copy -// txnLogs = append(txnLogs, &log) -// if i+1 >= len(blockLogs) || blockLogs[i+1].TxHash != txnHash { -// break -// } -// } -// } -// return txnLogs -// } - func groupLogsByTransaction(logs []types.Log) map[string][]*types.Log { var out = make(map[string][]*types.Log) for _, log := range logs { @@ -1056,12 +1155,82 @@ func groupLogsByTransaction(logs []types.Log) map[string][]*types.Log { return out } -// func blockLogsCount(numTxns int, logs []types.Log) uint { -// var max uint = uint(numTxns) -// for _, log := range logs { -// if log.TxIndex+1 > max { -// max = log.TxIndex + 1 -// } -// } -// return max -// } +// A removed hash can later be canonical again. Generations invalidate work +// started before rollback even when its transaction and block hash are unchanged. +// Pending retries and RPCs can outlive monitor retention, so block age alone +// cannot determine when it is safe to discard this state. +type blockRef struct { + hash common.Hash + generation uint64 +} +type blockState struct { + generation uint64 + removed bool + invalidatedAt uint64 +} + +func (l *ReceiptsListener) currentBlock(hash common.Hash, generation uint64) bool { + state := l.blockStates[hash] + return !state.removed && state.generation == generation +} +func (l *ReceiptsListener) isCurrentBlock(hash common.Hash, generation uint64) bool { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + return l.currentBlock(hash, generation) +} +func (l *ReceiptsListener) blockGeneration(hash common.Hash) uint64 { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + return l.blockStates[hash].generation +} + +// Caller holds receiptMu. Hash queries do not know their block before fetching, +// so reject results from blocks invalidated since the RPC started as well. +func (l *ReceiptsListener) validFetchedBlock(hash common.Hash, expected blockRef, started uint64) bool { + state := l.blockStates[hash] + if state.removed || state.invalidatedAt > started { + return false + } + return expected.hash == (common.Hash{}) || (hash == expected.hash && state.generation == expected.generation) +} +func (l *ReceiptsListener) acceptBlock(block *ethmonitor.Block) { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + hash := block.Hash() + state := l.blockStates[hash] + if state.removed { + incarnation, canonical := block.CanonicalState() + if incarnation == 0 { + // Legacy/manual blocks have no incarnation evidence; only a + // positive retained lookup can authorize their re-adoption. + retained := l.monitor.GetBlock(hash) + canonical = retained != nil && retained.Event == ethmonitor.Added + } + if canonical { + state.removed = false + l.blockStates[hash] = state + } + } +} +func (l *ReceiptsListener) invalidateBlock(ctx context.Context, block *ethmonitor.Block) blockRef { + l.receiptMu.Lock() + defer l.receiptMu.Unlock() + state := l.blockStates[block.Hash()] + if !state.removed { + l.reorgRevision++ + state.generation++ + state.invalidatedAt = l.reorgRevision + state.removed = true + l.blockStates[block.Hash()] = state + } + removed := blockRef{block.Hash(), state.generation - 1} + for _, txn := range block.Transactions() { + hash := txn.Hash().Hex() + cached, found, _ := l.pastReceipts.Get(ctx, hash) + if found && cached.BlockHash == block.Hash() { + l.pastReceipts.Delete(ctx, hash) + } + l.notFoundTxnHashes.Delete(ctx, hash) + } + return removed +} diff --git a/ethreceipts/ethreceipts_test.go b/ethreceipts/ethreceipts_test.go index 9c508a4e..d2bfc3ae 100644 --- a/ethreceipts/ethreceipts_test.go +++ b/ethreceipts/ethreceipts_test.go @@ -20,10 +20,12 @@ import ( "github.com/0xsequence/ethkit/ethrpc" "github.com/0xsequence/ethkit/ethtest" "github.com/0xsequence/ethkit/ethtxn" + "github.com/0xsequence/ethkit/ethwallet" "github.com/0xsequence/ethkit/go-ethereum/common" "github.com/0xsequence/ethkit/go-ethereum/core/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" ) var ( @@ -72,6 +74,109 @@ func newProvider(t *testing.T) *ethrpc.Provider { return provider } +// Service errors cancel receipt waits and are checked after the worker exits. +func runReceiptTestService(t *testing.T, ctx context.Context, cancel context.CancelFunc, name string, run func(context.Context) error) { + t.Helper() + done := make(chan error, 1) + go func() { + err := run(ctx) + if ctx.Err() != nil && (err == nil || errors.Is(err, ctx.Err())) { + done <- nil + return + } + if err == nil { + err = fmt.Errorf("%s stopped unexpectedly", name) + } else { + err = fmt.Errorf("%s: %w", name, err) + } + done <- err + cancel() + }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(10 * time.Second): + t.Errorf("%s did not stop after cancellation", name) + } + }) +} + +func restoreReceiptTestProvider(t *testing.T, ctx context.Context, cancel context.CancelFunc, provider *ethrpc.Provider) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + select { + case <-ctx.Done(): + case <-time.After(20 * time.Second): + provider.SetHTTPClient(newFlakyHTTPClient(0.0)) + } + }() + t.Cleanup(func() { + cancel() + <-done + }) +} + +func waitForFlakyProviderReceipts(t *testing.T, ctx context.Context, provider *ethrpc.Provider, listener *ethreceipts.ReceiptsListener, sender *ethwallet.Wallet, recipient common.Address) { + t.Helper() + nonce, err := sender.GetNonce(ctx) + require.NoError(t, err) + + txns := make([]*types.Transaction, 50) + for i := range txns { + txns[i], err = sender.NewTransaction(ctx, ðtxn.TransactionRequest{ + To: &recipient, + ETHValue: ethtest.ETHValue(0.01), + GasLimit: 120_000, + Nonce: big.NewInt(int64(nonce + uint64(i))), + }) + require.NoError(t, err) + } + + g, workerCtx := errgroup.WithContext(ctx) + for _, txn := range txns { + ready := make(chan struct{}) + g.Go(func() error { + sub := listener.Subscribe(ethreceipts.FilterTxnHash(txn.Hash())) + defer sub.Unsubscribe() + close(ready) + + select { + case <-workerCtx.Done(): + return workerCtx.Err() + case <-sub.Done(): + return fmt.Errorf("subscription closed before receipt for %s", txn.Hash()) + case receipt, ok := <-sub.TransactionReceipt(): + if !ok { + return fmt.Errorf("receipt channel closed for %s", txn.Hash()) + } + if receipt.TransactionHash() != txn.Hash() || receipt.Status() != types.ReceiptStatusSuccessful || receipt.Reorged { + return fmt.Errorf("unexpected receipt for %s: hash=%s status=%d reorged=%t", txn.Hash(), receipt.TransactionHash(), receipt.Status(), receipt.Reorged) + } + return nil + case <-time.After(300 * time.Second): + return fmt.Errorf("timeout waiting for receipt for %s", txn.Hash()) + } + }) + g.Go(func() error { + select { + case <-workerCtx.Done(): + return workerCtx.Err() + case <-ready: + } + _, _, err := ethtxn.SendTransaction(workerCtx, provider, txn) + if err != nil { + return fmt.Errorf("send transaction %s: %w", txn.Hash(), err) + } + return nil + }) + } + require.NoError(t, g.Wait()) +} + func newFlakyHTTPClient(failureRate float32) *http.Client { return &http.Client{ Timeout: 10 * time.Second, @@ -118,12 +223,7 @@ func TestFetchTransactionReceiptBasic(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) require.Zero(t, monitor.NumSubscribers()) @@ -134,12 +234,7 @@ func TestFetchTransactionReceiptBasic(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) assert.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // // Setup test wallet @@ -174,54 +269,29 @@ func TestFetchTransactionReceiptBasic(t *testing.T) { txnHashes = append(txnHashes, txn.Hash()) } - // dispatch txns in the background - go func() { + workers, workerCtx := errgroup.WithContext(ctx) + workers.Go(func() error { for _, txn := range txns { - _, _, err = wallet.SendTransaction(ctx, txn) - require.NoError(t, err) - // time.Sleep(500 * time.Millisecond) + if _, _, err := wallet.SendTransaction(workerCtx, txn); err != nil { + return fmt.Errorf("send transaction %s: %w", txn.Hash(), err) + } } - }() - - // ensure all txns made it - // delay processing if we want to make sure SearchCache works - // time.Sleep(2 * time.Second) - // for _, txnHash := range txnHashes { - // receipt, err := provider.TransactionReceipt(context.Background(), txnHash) - // require.NoError(t, err) - // require.True(t, receipt.Status == 1) - // } - - // Let's listen for all the txns - var wg sync.WaitGroup - for i, txnHash := range txnHashes { - wg.Add(1) - go func(i int, txnHash common.Hash) { - defer wg.Done() + return nil + }) - receipt, err := receiptsListener.FetchTransactionReceipt(ctx, txnHash, 7) - require.NoError(t, err) - require.NotNil(t, receipt) - require.True(t, receipt.Status() == types.ReceiptStatusSuccessful) - require.False(t, receipt.Final) - t.Logf("=> MINED %d :: %s", i, receipt.TransactionHash().String()) - - // receipt, waitFinality, err := receiptsListener.FetchTransactionReceiptWithFinality(ctx, txnHash, 7) - // require.NoError(t, err) - // require.NotNil(t, receipt) - // require.True(t, receipt.Status() == types.ReceiptStatusSuccessful) - // require.False(t, receipt.Final) - // t.Logf("=> MINED %d :: %s", i, receipt.TransactionHash().String()) - // - // finalReceipt, err := waitFinality(context.Background()) - // require.NoError(t, err) - // require.NotNil(t, finalReceipt) - // require.True(t, finalReceipt.Status() == types.ReceiptStatusSuccessful) - // require.True(t, finalReceipt.Final) - // t.Logf("=> FINAL %d :: %s", i, receipt.TransactionHash().String()) - }(i, txnHash) + for _, txnHash := range txnHashes { + workers.Go(func() error { + receipt, err := receiptsListener.FetchTransactionReceipt(workerCtx, txnHash, 7) + if err != nil { + return fmt.Errorf("fetch receipt %s: %w", txnHash, err) + } + if receipt == nil || receipt.TransactionHash() != txnHash || receipt.Status() != types.ReceiptStatusSuccessful || receipt.Final { + return fmt.Errorf("unexpected mined receipt for %s: %v", txnHash, receipt) + } + return nil + }) } - wg.Wait() + require.NoError(t, workers.Wait()) time.Sleep(2 * time.Second) @@ -292,12 +362,7 @@ func TestFetchTransactionReceiptBlast(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 @@ -306,12 +371,7 @@ func TestFetchTransactionReceiptBlast(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) assert.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // // Setup wallets @@ -345,176 +405,193 @@ func TestFetchTransactionReceiptBlast(t *testing.T) { txns[5].Hash(), txns[2].Hash(), txns[8].Hash(), txns[3].Hash(), } - var count uint64 - - var wg sync.WaitGroup - for i, txnHash := range txnHashes { - wg.Add(1) - go func(i int, txnHash common.Hash) { - defer wg.Done() - - receipt, receiptFinality, err := receiptsListener.FetchTransactionReceiptWithFinality(ctx, txnHash) - assert.NoError(t, err) - assert.NotNil(t, receipt) - assert.True(t, receipt.Status() == types.ReceiptStatusSuccessful) - - finalReceipt, err := receiptFinality(context.Background()) - require.NoError(t, err) - require.True(t, finalReceipt.Status() == types.ReceiptStatusSuccessful) - - t.Logf("=> %d :: %s", i, receipt.TransactionHash().String()) - - atomic.AddUint64(&count, 1) - }(i, txnHash) + workers, workerCtx := errgroup.WithContext(ctx) + for _, txnHash := range txnHashes { + workers.Go(func() error { + receipt, waitFinality, err := receiptsListener.FetchTransactionReceiptWithFinality(workerCtx, txnHash) + if err != nil { + return fmt.Errorf("fetch receipt %s: %w", txnHash, err) + } + if receipt == nil || receipt.TransactionHash() != txnHash || receipt.Status() != types.ReceiptStatusSuccessful { + return fmt.Errorf("unexpected receipt for %s: %v", txnHash, receipt) + } + finalReceipt, err := waitFinality(workerCtx) + if err != nil { + return fmt.Errorf("final receipt %s: %w", txnHash, err) + } + if finalReceipt == nil || finalReceipt.TransactionHash() != txnHash || finalReceipt.Status() != types.ReceiptStatusSuccessful || !finalReceipt.Final { + return fmt.Errorf("unexpected final receipt for %s: %v", txnHash, finalReceipt) + } + return nil + }) } - wg.Wait() - - require.Equal(t, int(count), len(txnHashes)) + require.NoError(t, workers.Wait()) } func TestReceiptsListenerFilters(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() - // - // Setup ReceiptsListener - // provider := testchain.Provider - monitorOptions := ethmonitor.DefaultOptions - // monitorOptions.Logger = log monitorOptions.WithLogs = true monitorOptions.BlockRetentionLimit = 1000 monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) - assert.NoError(t, err) - - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + require.NoError(t, err) + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 listenerOptions.FilterMaxWaitNumBlocks = 4 - receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) - assert.NoError(t, err) - - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() - - // - // Setup wallets - // - - // create and fund a few wallets to send from - fromWallets, _ := testchain.DummyWallets(3, 100) - testchain.FundAddresses(ethtest.WalletAddresses(fromWallets), 10) - - // create a few wallets to send to - toWallets, _ := testchain.DummyWallets(3, 200) - - // prepare and sign bunch of txns - values := []*big.Int{} - for range fromWallets { - values = append(values, ethtest.ETHValue(0.1)) - } + require.NoError(t, err) + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) + fromWallets, err := testchain.DummyWallets(3, 100) + require.NoError(t, err) + require.NoError(t, testchain.FundAddresses(ethtest.WalletAddresses(fromWallets), 10)) + toWallets, err := testchain.DummyWallets(3, 200) + require.NoError(t, err) + values := []*big.Int{ethtest.ETHValue(0.1), ethtest.ETHValue(0.1), ethtest.ETHValue(0.1)} _, txns, err := ethtest.PrepareBlastSendTransactions(ctx, fromWallets, ethtest.WalletAddresses(toWallets), values) - assert.NoError(t, err) + require.NoError(t, err) - // send the txns -- these will be async, so we can just blast synchronously - // and not have to do it in a goroutine + fromFilter := ethreceipts.FilterFrom(fromWallets[1].Address()).LimitOne(true).ID(1111).MaxWait(0) + toFilter := ethreceipts.FilterTo(toWallets[1].Address()).ID(1112).MaxWait(0) + removedFilter := ethreceipts.FilterTxnHash(txns[2].Hash()).ID(2222) + addedFilter := ethreceipts.FilterTxnHash(txns[4].Hash()).ID(4444) + sub := receiptsListener.Subscribe(fromFilter, toFilter, removedFilter) + defer sub.Unsubscribe() + + sub2 := receiptsListener.Subscribe() + secondFilter := ethreceipts.FilterTxnHash(txns[3].Hash()) + sub2.AddFilter(secondFilter) + defer sub2.Unsubscribe() + + thirdFilter := ethreceipts.FilterTxnHash(txns[2].Hash()).ID(3333) + expiringFilter := ethreceipts.FilterFrom(ethkit.Address{4, 2, 4, 2}).ID(8888).MaxWait(4) + sub3 := receiptsListener.Subscribe(thirdFilter, expiringFilter) + defer sub3.Unsubscribe() + + // All live filters are installed before any transaction can be mined. + require.Eventually(t, func() bool { + return receiptsListener.IsRunning() && monitor.NumSubscribers() == 1 + }, 5*time.Second, 10*time.Millisecond) for _, txn := range txns { _, _, err := ethtxn.SendTransaction(ctx, provider, txn) - assert.NoError(t, err) + require.NoError(t, err) } - // - // Subscribe to a filter on the receipt listener - // - fmt.Println("listening for txns..") - - sub := receiptsListener.Subscribe( - ethreceipts.FilterFrom(fromWallets[1].Address()).LimitOne(true), - ethreceipts.FilterTo(toWallets[1].Address()), - ethreceipts.FilterTxnHash(txns[2].Hash()).ID(2222), //.Finalize(true) is set by default for FilterTxnHash - ) - - sub2 := receiptsListener.Subscribe() - sub2.AddFilter(ethreceipts.FilterTxnHash(txns[3].Hash())) - - sub3 := receiptsListener.Subscribe( - ethreceipts.FilterTxnHash(txns[2].Hash()).ID(3333), - - // will end up not being found and timeout after MaxWait - ethreceipts.FilterFrom(ethkit.Address{4, 2, 4, 2}).MaxWait(4), - ) - - go func() { - time.Sleep(5 * time.Second) - fmt.Println("==> delaying to find", txns[4].Hash().String()) - sub.AddFilter(ethreceipts.FilterTxnHash(txns[4].Hash()).ID(4444)) - }() - - go func() { - for r := range sub2.TransactionReceipt() { - fmt.Println("sub2, got receipt", r.TransactionHash(), "final?", r.Final) + type delivery struct { + subscription string + hash common.Hash + filterID uint64 + final bool + } + expected := make(map[delivery]ethreceipts.FilterQuery) + expect := func(subscription string, txnIndex int, filter ethreceipts.FilterQuery, states ...bool) { + for _, final := range states { + key := delivery{subscription, txns[txnIndex].Hash(), filter.(ethreceipts.Filterer).FilterID(), final} + expected[key] = filter } - }() - - go func() { - for r := range sub3.TransactionReceipt() { - fmt.Println("sub3, got receipt", r.TransactionHash(), "final?", r.Final, "id?", r.FilterID()) //, "maxWait hit?", r.Filter.IsExpired()) + } + // LimitOne selects the first matching nonce; the recipient filter selects all three senders. + expect("sub", 3, fromFilter, false) + for _, i := range []int{1, 4, 7} { + expect("sub", i, toFilter, false) + } + expect("sub", 2, removedFilter, false) + expect("sub", 4, addedFilter, false, true) + expect("sub2", 3, secondFilter, false, true) + expect("sub3", 2, thirdFilter, false, true) + + seen := make(map[delivery]bool) + added := false + removed := false + record := func(subscription string, receipt ethreceipts.Receipt) { + key := delivery{subscription, receipt.TransactionHash(), receipt.FilterID(), receipt.Final} + filter, ok := expected[key] + require.True(t, ok, "unexpected delivery: %+v", key) + require.False(t, seen[key], "duplicate delivery: %+v", key) + require.Same(t, filter, receipt.Filter, "wrong filter owner: %+v", key) + require.Equal(t, uint64(types.ReceiptStatusSuccessful), receipt.Status()) + require.False(t, receipt.Reorged) + require.NotZero(t, receipt.BlockHash()) + seen[key] = true + + if subscription == "sub" && receipt.FilterID() == 2222 { + // Remove only this owner after its mined event, before it can finalize. + sub.RemoveFilter(receipt.Filter) + require.NotContains(t, sub.Filters(), receipt.Filter) + removed = true } - }() + if subscription == "sub" && receipt.FilterID() == 1112 && receipt.TransactionHash() == txns[4].Hash() { + // This transaction is now in the monitor cache; registration must find it there. + sub.AddFilter(addedFilter) + require.Contains(t, sub.Filters(), addedFilter) + added = true + } + } -loop: - for { + exhausted := expiringFilter.(ethreceipts.Filterer).Exhausted() + expired := false + for len(seen) != len(expected) || !expired { select { - case <-ctx.Done(): - fmt.Println("ctx done") - break loop - + missing := make([]delivery, 0) + for key := range expected { + if !seen[key] { + missing = append(missing, key) + } + } + t.Fatalf("missing receipt deliveries: %v; expired=%t; context=%v", missing, expired, ctx.Err()) case <-sub.Done(): - fmt.Println("sub done") - break loop - + t.Fatal("subscription closed before expected deliveries") + case <-sub2.Done(): + t.Fatal("second subscription closed before expected deliveries") + case <-sub3.Done(): + t.Fatal("third subscription closed before expected deliveries") case receipt, ok := <-sub.TransactionReceipt(): - if !ok { - continue - } - - fmt.Println("=> sub, got receipt", receipt.TransactionHash(), "final?", receipt.Final, "id?", receipt.FilterID(), "status?", receipt.Status()) - - // txn := receipt.Transaction - // txnMsg := receipt.Message - - fmt.Println("=> filter matched!", receipt.From(), receipt.TransactionHash()) - fmt.Println("=> receipt status?", receipt.Status()) - - fmt.Println("==> len filters", len(sub.Filters())) - if receipt.TransactionHash() == txns[2].Hash() { - sub.RemoveFilter(receipt.Filter) - } - fmt.Println("==> len filters", len(sub.Filters())) - - fmt.Println("") - - // expecting to be finished with listening for events after a few seconds - case <-time.After(15 * time.Second): - sub.Unsubscribe() - + require.True(t, ok, "receipt channel closed before expected deliveries") + record("sub", receipt) + case receipt, ok := <-sub2.TransactionReceipt(): + require.True(t, ok, "second receipt channel closed before expected deliveries") + record("sub2", receipt) + case receipt, ok := <-sub3.TransactionReceipt(): + require.True(t, ok, "third receipt channel closed before expected deliveries") + record("sub3", receipt) + case <-exhausted: + expired = true + exhausted = nil } } + + require.True(t, added, "cached transaction filter was not added") + require.True(t, removed, "mined transaction filter was not removed") + require.Eventually(t, func() bool { + return len(sub.Filters()) == 1 && len(sub2.Filters()) == 0 && len(sub3.Filters()) == 0 + }, 5*time.Second, 10*time.Millisecond) + require.Same(t, toFilter, sub.Filters()[0]) + + // The unbounded subscription queue forwards asynchronously; observe it long enough + // to reject queued duplicates or finalization of the removed owner. + quiet := time.NewTimer(2 * time.Second) + defer quiet.Stop() + select { + case <-ctx.Done(): + t.Fatalf("context ended while checking for extra deliveries: %v", ctx.Err()) + case receipt, ok := <-sub.TransactionReceipt(): + require.True(t, ok, "receipt channel closed unexpectedly") + record("sub", receipt) + case receipt, ok := <-sub2.TransactionReceipt(): + require.True(t, ok, "second receipt channel closed unexpectedly") + record("sub2", receipt) + case receipt, ok := <-sub3.TransactionReceipt(): + require.True(t, ok, "third receipt channel closed unexpectedly") + record("sub3", receipt) + case <-quiet.C: + } } func TestReceiptsListenerERC20(t *testing.T) { @@ -544,12 +621,7 @@ func TestReceiptsListenerERC20(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 @@ -558,12 +630,7 @@ func TestReceiptsListenerERC20(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, provider, monitor, listenerOptions) assert.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // // Subscribe to a filter on the receipt listener @@ -602,26 +669,20 @@ func TestReceiptsListenerERC20(t *testing.T) { num := int64(2000) erc20Receipts := make([]*types.Receipt, 0) - var erc20ReceiptsMu sync.Mutex receipt := erc20Mock.Mint(t, wallet, num) erc20Receipts = append(erc20Receipts, receipt) erc20Mock.GetBalance(t, wallet.Address(), num) - go func() { - total := int64(0) - for i := 0; i < 5; i++ { - n := int64(40 + i) - total += n - - erc20ReceiptsMu.Lock() - receipt := erc20Mock.Transfer(t, wallet, wallet2.Address(), n) - erc20Receipts = append(erc20Receipts, receipt) - erc20ReceiptsMu.Unlock() + total := int64(0) + for i := 0; i < 5; i++ { + n := int64(40 + i) + total += n - erc20Mock.GetBalance(t, wallet2.Address(), total) - } - }() + receipt := erc20Mock.Transfer(t, wallet, wallet2.Address(), n) + erc20Receipts = append(erc20Receipts, receipt) + erc20Mock.GetBalance(t, wallet2.Address(), total) + } // // Listener loop @@ -667,20 +728,29 @@ loop: } } - // NOTE: expecting receipts twice. Once on mine, once on finalize. - for _, mr := range matchedReceipts { - found := false - for _, r := range erc20Receipts { - if mr.TransactionHash() == r.TxHash { - found = true - } - } - assert.True(t, found, "looking for matched receipt %s", mr.TransactionHash().String()) + // Each submitted transaction must have one mined and one final event, even + // when MaxWait expires before the queued finality threshold is reached. + expectedHashes := make(map[common.Hash]bool, len(erc20Receipts)) + for _, receipt := range erc20Receipts { + expectedHashes[receipt.TxHash] = true } - - t.Logf("matchedCount: %d", matchedCount) - t.Logf("erc20Receipts: %d", len(erc20Receipts)) - require.Equal(t, matchedCount, len(erc20Receipts)*2) + type receiptState struct { + hash common.Hash + final bool + } + counts := make(map[receiptState]int) + for _, receipt := range matchedReceipts { + require.True(t, expectedHashes[receipt.TransactionHash()], "unexpected transaction %s", receipt.TransactionHash()) + require.Equal(t, uint64(9999), receipt.FilterID()) + require.Equal(t, uint64(types.ReceiptStatusSuccessful), receipt.Status()) + require.False(t, receipt.Reorged) + counts[receiptState{receipt.TransactionHash(), receipt.Final}]++ + } + for hash := range expectedHashes { + require.Equal(t, 1, counts[receiptState{hash, false}], "mined delivery for %s", hash) + require.Equal(t, 1, counts[receiptState{hash, true}], "final delivery for %s", hash) + } + require.Equal(t, len(erc20Receipts)*2, matchedCount) } func TestFiltersAddDeadlock(t *testing.T) { @@ -696,12 +766,7 @@ func TestFiltersAddDeadlock(t *testing.T) { monitor, err := ethmonitor.NewMonitor(provider, monitorOptions) assert.NoError(t, err) - go func() { - err := monitor.Run(ctx) - if err != nil { - t.Error(err) - } - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.NumBlocksToFinality = 10 @@ -780,10 +845,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(goodProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -792,81 +854,18 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, goodProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) t.Run("Wait for txn receipts with a flaky monitor and a healthy provider", func(t *testing.T) { @@ -885,10 +884,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(flakyProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -897,10 +893,7 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, goodProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) @@ -909,81 +902,15 @@ func TestFlakyProvider(t *testing.T) { t.Logf("Setting provider to flaky state") flakyProvider.SetHTTPClient(newFlakyHTTPClient(1.0)) - go func() { - // After 20 seconds, restore the provider to more reliable state - time.Sleep(20 * time.Second) - t.Logf("Restoring provider to healthy state") - flakyProvider.SetHTTPClient(newFlakyHTTPClient(0.0)) - }() + restoreReceiptTestProvider(t, ctx, cancel, flakyProvider) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - // Add a bunch of subscribers that will each send a txn and wait for it - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) t.Run("Wait for txn receipts with a healthy monitor and a flaky provider", func(t *testing.T) { @@ -1002,10 +929,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(goodProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -1014,10 +938,7 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, flakyProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) @@ -1026,81 +947,15 @@ func TestFlakyProvider(t *testing.T) { t.Logf("Setting provider to flaky state") flakyProvider.SetHTTPClient(newFlakyHTTPClient(1.0)) - go func() { - // After 20 seconds, restore the provider to more reliable state - time.Sleep(20 * time.Second) - t.Logf("Restoring provider to healthy state") - flakyProvider.SetHTTPClient(newFlakyHTTPClient(0.0)) - }() + restoreReceiptTestProvider(t, ctx, cancel, flakyProvider) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - // Add a bunch of subscribers that will each send a txn and wait for it - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) t.Run("Wait for txn receipts with a flaky monitor and a flaky provider", func(t *testing.T) { @@ -1119,10 +974,7 @@ func TestFlakyProvider(t *testing.T) { monitor, err := ethmonitor.NewMonitor(flakyProvider, monitorOptions) require.NoError(t, err) - go func() { - err := monitor.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "monitor", monitor.Run) listenerOptions := ethreceipts.DefaultOptions listenerOptions.FilterMaxWaitNumBlocks = 1 @@ -1131,10 +983,7 @@ func TestFlakyProvider(t *testing.T) { receiptsListener, err := ethreceipts.NewReceiptsListener(log, flakyProvider, monitor, listenerOptions) require.NoError(t, err) - go func() { - err := receiptsListener.Run(ctx) - require.NoError(t, err) - }() + runReceiptTestService(t, ctx, cancel, "receipts listener", receiptsListener.Run) // Wait for services to be ready time.Sleep(2 * time.Second) @@ -1143,80 +992,14 @@ func TestFlakyProvider(t *testing.T) { t.Logf("Setting provider to flaky state") flakyProvider.SetHTTPClient(newFlakyHTTPClient(1.0)) - go func() { - // After 20 seconds, restore the provider to more reliable state - time.Sleep(20 * time.Second) - t.Logf("Restoring provider to healthy state") - flakyProvider.SetHTTPClient(newFlakyHTTPClient(0.0)) - }() + restoreReceiptTestProvider(t, ctx, cancel, flakyProvider) - walletA, _ := testchain.DummyWallet(1) + walletA, err := testchain.DummyWallet(1) + require.NoError(t, err) testchain.MustFundAddress(walletA.Address()) - walletB, _ := testchain.DummyWallet(uint64(rand.Int63n(1000))) - walletBAddress := walletB.Address() - - nonce, err := walletA.GetNonce(ctx) + walletB, err := testchain.DummyWallet(2) require.NoError(t, err) - - var wg sync.WaitGroup - - // Add a bunch of subscribers that will each send a txn and wait for it - for i := 0; i < 50; i++ { - txr := ðtxn.TransactionRequest{ - To: &walletBAddress, - ETHValue: ethtest.ETHValue(0.01), - GasLimit: 120_000, - Nonce: big.NewInt(int64(nonce + uint64(i))), - } - - signedTxn, err := walletA.NewTransaction(ctx, txr) - require.NoError(t, err) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - // Delay sending the transaction to ensure the subscriber is ready - time.Sleep(2 * time.Second) - - // Using the trusted provider here to ensure txn is sent - ethtxn.SendTransaction(ctx, goodProvider, signedTxn) - require.NoError(t, err) - }(signedTxn) - - wg.Add(1) - go func(signedTxn *types.Transaction) { - defer wg.Done() - - receiptsFilter := ethreceipts.FilterTxnHash( - signedTxn.Hash(), - ) - - sub := receiptsListener.Subscribe( - receiptsFilter, - ) - defer sub.Unsubscribe() - - start := time.Now() - select { - case <-ctx.Done(): - t.Fatalf("Context done: %v", ctx.Err()) - case <-sub.Done(): - t.Fatal("Subscription closed unexpectedly") - case receipt := <-sub.TransactionReceipt(): - activeSubs := receiptsListener.NumSubscribers() - t.Logf("Filter matched txn %s after %s, active subs: %d", signedTxn.Hash().String(), time.Since(start), activeSubs) - require.Equal(t, signedTxn.Hash(), receipt.TransactionHash()) - require.Equal(t, uint64(1), receipt.Status()) - case <-time.After(300 * time.Second): - t.Fatal("Timeout waiting for filter to match txn") - } - }(signedTxn) - - } - - t.Logf("Waiting for all goroutines to complete...") - wg.Wait() + waitForFlakyProviderReceipts(t, ctx, goodProvider, receiptsListener, walletA, walletB.Address()) }) } diff --git a/ethreceipts/filterer.go b/ethreceipts/filterer.go index fd48c20d..28046694 100644 --- a/ethreceipts/filterer.go +++ b/ethreceipts/filterer.go @@ -120,22 +120,21 @@ type FilterQuery interface { QueryOnChain(func(context.Context) (*types.Receipt, error)) FilterQuery MaxWait(int) FilterQuery - // DEPRECATED: please use QueryOnChainTxnHash instead, which is the same thing, renamed to be more clear - // in addition, see new QueryChain(fn) as additional feature. + // Deprecated: use QueryOnChainTxnHash instead. SearchOnChain(bool) FilterQuery } type FilterOptions struct { - // .. + // ID is a caller-supplied label, not a unique registration identity. ID uint64 - // .. + // Finalize requests a final receipt after the mined receipt. Finalize bool - // . + // LimitOne stops matching after one mined receipt and preserves queued finality. LimitOne bool - // .. + // SearchCache searches the monitor's retained blocks when the filter is registered. SearchCache bool // QueryOnChainTxnHash will query the chain for the txn hash at the start @@ -144,17 +143,16 @@ type FilterOptions struct { // mined before the filter was created. QueryOnChainTxnHash bool - // .. + // QueryOnChain optionally fetches a receipt when the filter is registered. QueryOnChain func(context.Context) (*types.Receipt, error) - // MaxWait filter option waits some number of blocks without a filter match after - // which point will auto-unsubscribe the filter. This is useful to help automatically - // remove filters which likely won't come up. + // MaxWait limits blocks without a match before matching is exhausted. + // Receipts already queued for finality are still finalized. // // nil : use the ReceiptsListener option FilterMaxWaitNumBlocks value as the default - // -1 : set value to ReceiptsListener option NumFinality * 3 + // <0 : use ReceiptsListener option NumBlocksToFinality * 2 // 0 : option is disabled, and has no limit on wait. filters need to be manually unsubscribed - // N : a specified number of blocks without a match before unsusbcribe + // N : a specified number of blocks without a match before exhaustion MaxWait *int } @@ -169,6 +167,7 @@ type FilterCond struct { type filter struct { options FilterOptions cond FilterCond + match func(context.Context, Receipt) (bool, error) // startBlockNum is the first block number observed once filter is active startBlockNum uint64 @@ -206,7 +205,7 @@ func (f *filter) SearchCache(searchCache bool) FilterQuery { return f } -// DEPRECATED: please use QueryChainForTxnHash instead, which is the same thing, renamed to be more clear +// Deprecated: use QueryOnChainTxnHash instead. func (f *filter) SearchOnChain(searchOnChain bool) FilterQuery { f.options.QueryOnChainTxnHash = searchOnChain return f @@ -240,6 +239,9 @@ func (f *filter) Cond() FilterCond { } func (f *filter) Match(ctx context.Context, receipt Receipt) (bool, error) { + if f.match != nil { + return f.match(ctx, receipt) + } c := f.cond if c.TxnHash != nil { @@ -302,3 +304,28 @@ func (f *filter) closeExhausted() { func (f *filter) Exhausted() <-chan struct{} { return f.exhausted } + +// Fetch helpers own their options, counters, and exhaustion signal. A custom +// Filterer keeps its matching behavior without receiving helper option mutations. +func snapshotFilter(source Filterer) *filter { + options := source.Options() + if options.MaxWait != nil { + value := *options.MaxWait + options.MaxWait = &value + } + snapshot := &filter{options: options, cond: source.Cond(), exhausted: make(chan struct{})} + if builtIn, ok := source.(*filter); ok { + snapshot.match = builtIn.match + } else { + snapshot.match = source.Match + } + return snapshot +} + +func builtinFilter(filterer Filterer) *filter { + if owner, ok := filterer.(*filterOwner); ok { + filterer = owner.Filterer + } + builtIn, _ := filterer.(*filter) + return builtIn +} diff --git a/ethreceipts/finalizer.go b/ethreceipts/finalizer.go index c9ffa2db..85ed7dcf 100644 --- a/ethreceipts/finalizer.go +++ b/ethreceipts/finalizer.go @@ -2,96 +2,132 @@ package ethreceipts import ( "math/big" - "sort" "sync" - "github.com/0xsequence/ethkit" + "github.com/0xsequence/ethkit/go-ethereum/common" ) +// Public filter IDs are labels; distinct filters can share one label. +type receiptKey struct { + txnHash common.Hash + blockHash common.Hash + generation uint64 + owner *filterOwner +} + +func receiptOwner(receipt Receipt, owner *filterOwner) receiptKey { + return receiptKey{receipt.TransactionHash(), receipt.BlockHash(), receipt.generation, owner} +} + type finalizer struct { queue []finalTxn - txns map[ethkit.Hash]struct{} + txns map[receiptKey]struct{} numBlocksToFinality *big.Int mu sync.Mutex } - type finalTxn struct { receipt Receipt blockNum *big.Int } -func (f *finalizer) len() int { +func (f *finalizer) setFinality(num int) { f.mu.Lock() defer f.mu.Unlock() - return len(f.queue) + f.numBlocksToFinality = big.NewInt(int64(num)) } - -// func (f *finalizer) lastBlockNum() *big.Int { -// f.mu.Lock() -// defer f.mu.Unlock() -// if len(f.queue) == 0 { -// return big.NewInt(0) -// } -// return f.queue[0].blockNum -// } - -func (f *finalizer) enqueue(filterID uint64, receipt Receipt, blockNum *big.Int) { +func (f *finalizer) enqueue(owner *filterOwner, receipt Receipt, blockNum *big.Int) { f.mu.Lock() defer f.mu.Unlock() - - if receipt.Final { - // do not enqueue if the receipt is already final + if receipt.Final || blockNum == nil { return } - - txnHash := receipt.TransactionHash() - - // txn id based on the hash + filterID to ensure we get finalize callback for any unique filterID - txnID := txnHash - if filterID > 0 { - for i := 0; i < 8; i++ { - txnID[i] = txnID[i] + byte(filterID>>uint(i)) - } - } - - if _, ok := f.txns[txnID]; ok { - // update the blockNum if we already have this txn, as it could have been included - // again after a reorg in a new block + key := receiptOwner(receipt, owner) + if _, ok := f.txns[key]; ok { for i, entry := range f.queue { - if entry.receipt.TransactionHash() == txnHash { - f.queue[i] = finalTxn{receipt, blockNum} + if receiptOwner(entry.receipt, entry.receipt.owner) == key { + f.queue[i] = finalTxn{receipt, new(big.Int).Set(blockNum)} + return } } - return } - - // append new - f.queue = append(f.queue, finalTxn{receipt, blockNum}) - f.txns[txnID] = struct{}{} - - // sort block order from oldest to newest in case of a reorg - if len(f.queue) >= 2 && f.queue[0].blockNum.Cmp(f.queue[1].blockNum) < 0 { - sort.SliceStable(f.queue, func(i, j int) bool { - return f.queue[i].blockNum.Cmp(f.queue[j].blockNum) < 0 - }) + f.queue = append(f.queue, finalTxn{receipt, new(big.Int).Set(blockNum)}) + f.txns[key] = struct{}{} +} +func (f *finalizer) invalidateBlock(block blockRef) []Receipt { + f.mu.Lock() + defer f.mu.Unlock() + var invalidated []Receipt + retained := f.queue[:0] + for _, txn := range f.queue { + if txn.receipt.BlockHash() == block.hash && txn.receipt.generation == block.generation { + invalidated = append(invalidated, txn.receipt) + delete(f.txns, receiptOwner(txn.receipt, txn.receipt.owner)) + } else { + retained = append(retained, txn) + } } + clear(f.queue[len(retained):]) + f.queue = retained + return invalidated } - -func (f *finalizer) dequeue(currentBlockNum *big.Int) []finalTxn { +func (f *finalizer) invalidateOwner(owner *filterOwner) { f.mu.Lock() defer f.mu.Unlock() - - finalTxns := []finalTxn{} - + retained := f.queue[:0] for _, txn := range f.queue { - if currentBlockNum.Cmp(big.NewInt(0).Add(txn.blockNum, f.numBlocksToFinality)) > 0 { - finalTxns = append(finalTxns, txn) + if txn.receipt.owner == owner { + delete(f.txns, receiptOwner(txn.receipt, owner)) + } else { + retained = append(retained, txn) } } - - if len(finalTxns) > 0 { - f.queue = f.queue[len(finalTxns):] + clear(f.queue[len(retained):]) + f.queue = retained +} +func (f *finalizer) hasOwner(owner *filterOwner) bool { + f.mu.Lock() + defer f.mu.Unlock() + for _, txn := range f.queue { + if txn.receipt.owner == owner { + return true + } } - - return finalTxns + return false +} +func (f *finalizer) findOwner(filter Filterer) *filterOwner { + f.mu.Lock() + defer f.mu.Unlock() + for _, txn := range f.queue { + if sameFilter(txn.receipt.Filter, filter) { + return txn.receipt.owner + } + } + return nil +} +func (f *finalizer) clear() { + f.mu.Lock() + defer f.mu.Unlock() + f.queue = nil + clear(f.txns) +} +func (f *finalizer) dequeue(currentBlockNum *big.Int) []finalTxn { + f.mu.Lock() + defer f.mu.Unlock() + // Zero is unresolved until Run selects the network's finality policy. + if f.numBlocksToFinality == nil || f.numBlocksToFinality.Sign() <= 0 { + return nil + } + var finalized []finalTxn + retained := f.queue[:0] + for _, txn := range f.queue { + if currentBlockNum.Cmp(new(big.Int).Add(txn.blockNum, f.numBlocksToFinality)) >= 0 { + finalized = append(finalized, txn) + delete(f.txns, receiptOwner(txn.receipt, txn.receipt.owner)) + } else { + retained = append(retained, txn) + } + } + clear(f.queue[len(retained):]) + f.queue = retained + return finalized } diff --git a/ethreceipts/finalizer_test.go b/ethreceipts/finalizer_test.go new file mode 100644 index 00000000..70eb3c00 --- /dev/null +++ b/ethreceipts/finalizer_test.go @@ -0,0 +1,215 @@ +package ethreceipts + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestFinalizerFinalityBoundary(t *testing.T) { + tx, _, _ := hardeningTxn(t, 3000) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions(), b, hardeningBlock(101), hardeningBlock(102)) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + // Queue the receipt as it would have been at block 100, before head 102. + s.finalizer.enqueue(owner, Receipt{receipt: hardeningReceipt(b, tx), owner: owner, Filter: owner.Filterer}, b.Number()) + if err := s.finalizeReceipts(big.NewInt(101)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + if !l.isBlockFinal(b.Number()) { + t.Fatal("block is not final at the two-block threshold") + } + if err := s.finalizeReceipts(big.NewInt(102)); err != nil { + t.Fatal(err) + } + r := hardeningRead(t, s) + if !r.Final || r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q { + t.Fatal("wrong final receipt at the two-block threshold") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} + +func TestFetchFinalityAfterExhaustion(t *testing.T) { + tx, _, _ := hardeningTxn(t, 3001) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + hardeningStart(t, l) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + mined, waitFinal, err := l.FetchTransactionReceiptWithFinality(ctx, tx.Hash(), 1) + if err != nil { + t.Fatal(err) + } + if mined == nil || mined.Final || mined.Reorged || mined.BlockHash() != b.Hash() { + t.Fatalf("wrong mined receipt: %+v", mined) + } + l.mu.Lock() + s := l.subscribers[0] + l.mu.Unlock() + f := builtinFilter(s.Filters()[0]) + // A shallow reorg can exhaust matching while the mined block's finality + // remains queued. Use the same exhaustion steps as the listener. + s.exhaustFilter(f) + f.closeExhausted() + done := make(chan struct{}) + var final *Receipt + var finalErr error + go func() { + defer close(done) + final, finalErr = waitFinal(ctx) + }() + select { + case <-done: + t.Fatalf("finality stopped before its threshold: %v", finalErr) + case <-time.After(100 * time.Millisecond): + } + if err := s.finalizeReceipts(big.NewInt(102)); err != nil { + t.Fatal(err) + } + select { + case <-done: + if finalErr != nil || final == nil || !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || final.BlockHash() != b.Hash() { + t.Fatalf("wrong final receipt after exhaustion: %+v, error: %v", final, finalErr) + } + case <-ctx.Done(): + t.Fatal("finality did not complete after exhaustion") + } + if l.NumSubscribers() != 0 { + t.Error("fetch helper did not unsubscribe after finality") + } +} + +func TestHardeningSubscribeBeforeRunFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 2) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, DefaultOptions, b, hardeningBlock(101)) + s := l.Subscribe(FilterTxnHash(tx.Hash())).(*subscriber) + defer s.Unsubscribe() + hardeningStart(t, l) + if r := hardeningRead(t, s); r.Final { + t.Fatal("receipt final before network threshold") + } + processed := make(chan struct{}) + s.AddFilter(FilterLogs(func([]*types.Log) bool { return false }).SearchCache(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { close(processed); return nil, nil })) + select { + case <-processed: + case <-time.After(time.Second): + t.Fatal("registration did not complete") + } + hardeningNoReceipt(t, s) + if len(s.Filters()) != 2 { + t.Fatal("premature finality removed transaction filter") + } +} + +func TestHardeningFinalizerOrderAndOwners(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 4) + tx2, _, _ := hardeningTxn(t, 5) + newer, older := hardeningBlock(101, tx1), hardeningBlock(100, tx2) + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + return hardeningReceipt(newer, tx1), nil + } + return hardeningReceipt(older, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) + q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(7).Finalize(true) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + for _, r := range []*types.Receipt{hardeningReceipt(newer, tx1), hardeningReceipt(older, tx2)} { + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: r}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + hardeningRead(t, s) + } + for _, tc := range []struct { + head int64 + tx *types.Transaction + b *ethmonitor.Block + }{{102, tx2, older}, {103, tx1, newer}} { + if err := s.finalizeReceipts(big.NewInt(tc.head)); err != nil { + t.Fatal(err) + } + owners := map[Filterer]bool{} + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.TransactionHash() != tc.tx.Hash() || r.BlockHash() != tc.b.Hash() { + t.Errorf("wrong final at %d: %+v", tc.head, r) + } + owners[r.Filter] = true + case <-time.After(100 * time.Millisecond): + t.Errorf("missing owner final at %d", tc.head) + } + } + if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { + t.Errorf("finality lost distinct owners with same public ID at %d", tc.head) + } + } + hardeningNoReceipt(t, s) +} + +func TestHardeningExhaustedOwnerKeepsQueuedFinal(t *testing.T) { + tx, _, _ := hardeningTxn(t, 17) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).MaxWait(1) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + // Exhaustion removes matching and closes its signal, but documented pending + // finality remains owned. Exercise that state without relying on a live chain. + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + q.(*filter).closeExhausted() + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Filter != q || final.BlockHash() != b.Hash() { + t.Fatal("exhaustion canceled a queued final") + } + hardeningNoReceipt(t, s) +} + +func TestHardeningClearCancelsExhaustedFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 18) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + q.(*filter).closeExhausted() + s.ClearFilters() + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} diff --git a/ethreceipts/helpers_test.go b/ethreceipts/helpers_test.go new file mode 100644 index 00000000..e01ab671 --- /dev/null +++ b/ethreceipts/helpers_test.go @@ -0,0 +1,231 @@ +package ethreceipts + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/ethrpc" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/0xsequence/ethkit/go-ethereum/crypto" + "github.com/goware/breaker" +) + +// Only the RPC methods used by the listener are provided. No external node is needed. +type hardeningProvider struct { + ethrpc.RawInterface + chainID func(context.Context) (*big.Int, error) + receipt func(context.Context, common.Hash) (*types.Receipt, error) +} + +func (p *hardeningProvider) ChainID(ctx context.Context) (*big.Int, error) { + if p.chainID != nil { + return p.chainID(ctx) + } + return big.NewInt(1), nil +} + +func (p *hardeningProvider) TransactionReceipt(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if p.receipt != nil { + return p.receipt(ctx, h) + } + return nil, ethereum.NotFound +} + +func hardeningBlock(num int64, txns ...*types.Transaction) *ethmonitor.Block { + b := types.NewBlockWithHeader(&types.Header{Number: big.NewInt(num), BlockHash: common.BigToHash(big.NewInt(num)), ParentHash: common.BigToHash(big.NewInt(num - 1)), GasLimit: 30_000_000, Time: uint64(num)}) + return ðmonitor.Block{Block: b.WithBody(types.Body{Transactions: txns}), Event: ethmonitor.Added, OK: true} +} + +func hardeningTxn(t *testing.T, nonce uint64) (*types.Transaction, common.Address, common.Address) { + t.Helper() + key, err := crypto.HexToECDSA("1111111111111111111111111111111111111111111111111111111111111111") + if err != nil { + t.Fatal(err) + } + from := crypto.PubkeyToAddress(key.PublicKey) + to := common.HexToAddress("0x2222222222222222222222222222222222222222") + tx, err := types.SignTx(types.NewTransaction(nonce, to, big.NewInt(1), 21000, big.NewInt(1), nil), types.NewEIP155Signer(big.NewInt(1)), key) + if err != nil { + t.Fatal(err) + } + return tx, from, to +} + +func hardeningReceipt(b *ethmonitor.Block, tx *types.Transaction) *types.Receipt { + return &types.Receipt{TxHash: tx.Hash(), BlockHash: b.Hash(), BlockNumber: b.Number(), Status: 1} +} + +func hardeningListener(t *testing.T, p *hardeningProvider, opts Options, blocks ...*ethmonitor.Block) *ReceiptsListener { + t.Helper() + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger = log + mo.WithLogs = true + mo.BlockRetentionLimit = 100 + mo.Bootstrap = true + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if len(blocks) == 0 { + blocks = []*ethmonitor.Block{hardeningBlock(100)} + } + if err = m.Chain().BootstrapFromBlocks(blocks); err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + l.br = breaker.New(log, 0, 1, 0) + return l +} + +func hardeningStart(t *testing.T, l *ReceiptsListener) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + var runErr error + t.Cleanup(func() { + cancel() + l.Stop() + select { + case <-done: + if runErr != nil && !errors.Is(runErr, context.Canceled) { + t.Errorf("Run: %v", runErr) + } + case <-time.After(time.Second): + t.Error("Run did not stop") + } + }) + go func() { + defer close(done) + runErr = l.Run(ctx) + }() + deadline := time.After(time.Second) + for l.monitor.NumSubscribers() != 1 { + select { + case <-done: + t.Fatalf("Run exited before subscription: %v", runErr) + case <-deadline: + t.Fatal("Run did not subscribe to monitor") + default: + time.Sleep(time.Millisecond) + } + } +} + +func hardeningRead(t *testing.T, s Subscription) Receipt { + t.Helper() + select { + case r := <-s.TransactionReceipt(): + return r + case <-time.After(time.Second): + t.Fatal("missing receipt") + return Receipt{} + } +} + +func hardeningNoReceipt(t *testing.T, s Subscription) { + t.Helper() + select { + case r := <-s.TransactionReceipt(): + t.Errorf("unexpected receipt: txn=%s block=%s final=%v reorged=%v owner=%p", r.TransactionHash(), r.BlockHash(), r.Final, r.Reorged, r.Filter) + case <-time.After(30 * time.Millisecond): + } +} + +func hardeningProcess(t *testing.T, l *ReceiptsListener, s *subscriber, blocks ...*ethmonitor.Block) { + t.Helper() + if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + t.Fatal(err) + } +} + +func hardeningOptions() Options { opts := DefaultOptions; opts.NumBlocksToFinality = 2; return opts } + +// The real monitor polls this provider and builds/broadcasts reorg events. Tests +// advance its canonical RPC responses without changing monitor production APIs. +type ownershipLiveProvider struct { + *hardeningProvider + mu sync.Mutex + canonical map[uint64]*types.Block + blocks map[common.Hash]*types.Block +} + +func (p *ownershipLiveProvider) advance(blocks ...*types.Block) { + p.mu.Lock() + defer p.mu.Unlock() + for _, b := range blocks { + p.canonical[b.NumberU64()] = b + p.blocks[b.Hash()] = b + } +} + +func ownershipBlockPayload(b *types.Block) (json.RawMessage, error) { + if b == nil { + return nil, ethereum.NotFound + } + header := b.Header() + header.Difficulty = big.NewInt(0) + payload, err := json.Marshal(header) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + if err = json.Unmarshal(payload, &fields); err != nil { + return nil, err + } + fields["transactions"], err = json.Marshal(b.Transactions()) + if err != nil { + return nil, err + } + fields["uncles"] = json.RawMessage(`[]`) + return json.Marshal(fields) +} + +func (p *ownershipLiveProvider) RawBlockByNumber(_ context.Context, num *big.Int) (json.RawMessage, error) { + p.mu.Lock() + b := p.canonical[num.Uint64()] + p.mu.Unlock() + return ownershipBlockPayload(b) +} + +func (p *ownershipLiveProvider) RawBlockByHash(_ context.Context, hash common.Hash) (json.RawMessage, error) { + p.mu.Lock() + b := p.blocks[hash] + p.mu.Unlock() + return ownershipBlockPayload(b) +} + +func (p *ownershipLiveProvider) RawFilterLogs(context.Context, ethereum.FilterQuery) (json.RawMessage, error) { + return json.RawMessage(`[]`), nil +} + +func receiptsFixCollect(t *testing.T, s Subscription) []Receipt { + t.Helper() + var receipts []Receipt + timer := time.NewTimer(40 * time.Millisecond) + defer timer.Stop() + for { + select { + case r, ok := <-s.TransactionReceipt(): + if !ok { + t.Fatal("receipt channel closed unexpectedly") + } + receipts = append(receipts, r) + case <-timer.C: + return receipts + } + } +} diff --git a/ethreceipts/lifecycle_test.go b/ethreceipts/lifecycle_test.go new file mode 100644 index 00000000..d54359fa --- /dev/null +++ b/ethreceipts/lifecycle_test.go @@ -0,0 +1,467 @@ +package ethreceipts + +import ( + "context" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" + "github.com/goware/breaker" +) + +func TestSubscriptionUnsubscribe(t *testing.T) { + for _, workers := range []int{1, 8} { + name := "repeated" + if workers > 1 { + name = "concurrent" + } + t.Run(name, func(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + s := l.Subscribe() + other := l.Subscribe() + defer other.Unsubscribe() + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + defer func() { + if p := recover(); p != nil { + t.Errorf("Unsubscribe panicked: %v", p) + } + }() + <-start + s.Unsubscribe() + s.Unsubscribe() + }() + } + close(start) + wg.Wait() + if l.NumSubscribers() != 1 { + t.Fatal("Unsubscribe did not remove exactly one subscription") + } + select { + case <-s.Done(): + default: + t.Error("unsubscribed Done channel remains open") + } + select { + case _, ok := <-s.TransactionReceipt(): + if ok { + t.Error("unsubscribed receipt channel remains open") + } + case <-time.After(time.Second): + t.Error("unsubscribed receipt channel did not close") + } + select { + case <-other.Done(): + t.Error("Unsubscribe closed an unrelated subscription") + default: + } + }) + } +} + +func TestHardeningStopCancelsStartup(t *testing.T) { + entered := make(chan struct{}) + p := &hardeningProvider{chainID: func(ctx context.Context) (*big.Int, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() }} + l := hardeningListener(t, p, DefaultOptions) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + <-entered + l.Stop() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + cancel() + <-done + t.Error("Stop did not cancel startup ChainID request") + } +} + +func TestHardeningStopCancelsStartupBackoff(t *testing.T) { + entered := make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{chainID: func(context.Context) (*big.Int, error) { + if calls.Add(1) == 1 { + close(entered) + return nil, errors.New("transient ChainID failure") + } + return nil, breaker.ErrFatal + }} + l := hardeningListener(t, p, DefaultOptions) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + <-entered + // Allow the failed attempt to enter its one-second backoff. + time.Sleep(100 * time.Millisecond) + l.Stop() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("startup cancellation error: %v", err) + } + case <-time.After(500 * time.Millisecond): + cancel() + <-done + t.Error("Stop waited for startup ChainID backoff") + } + if calls.Load() != 1 { + t.Errorf("ChainID retried after cancellation: %d calls", calls.Load()) + } +} + +func TestHardeningMonitorStopCancelsRegistration(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + entered := make(chan struct{}) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return false }).QueryOnChain(func(ctx context.Context) (*types.Receipt, error) { close(entered); <-ctx.Done(); return nil, ctx.Err() })) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { done <- l.Run(context.Background()) }() + <-entered + l.monitor.UnsubscribeAll(errors.New("monitor closed")) + select { + case <-done: + case <-time.After(100 * time.Millisecond): + l.Stop() + <-done + t.Error("monitor closure did not cancel registration") + } +} + +func TestHardeningFetchSemaphoreCancellation(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + close(entered) + select { + case <-release: + return &types.Receipt{TxHash: h, BlockNumber: big.NewInt(100)}, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + }} + opts := hardeningOptions() + opts.MaxConcurrentFetchReceiptWorkers = 1 + l := hardeningListener(t, p, opts) + first := make(chan error, 1) + go func() { _, err := l.fetchTransactionReceipt(context.Background(), common.Hash{1}, true); first <- err }() + <-entered + ctx, cancel := context.WithCancel(context.Background()) + cancel() + second := make(chan error, 1) + go func() { _, err := l.fetchTransactionReceipt(ctx, common.Hash{2}, true); second <- err }() + blocked := false + select { + case err := <-second: + if !errors.Is(err, context.Canceled) { + t.Errorf("cancellation error: %v", err) + } + case <-time.After(100 * time.Millisecond): + blocked = true + } + close(release) + <-first + if blocked { + <-second + t.Error("canceled fetch waited for unrelated fetch slot") + } +} + +func TestHardeningFinalityWaitCancellation(t *testing.T) { + tx, _, _ := hardeningTxn(t, 10) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + hardeningStart(t, l) + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + mined, wait, err := l.FetchTransactionReceiptWithFinality(parent, tx.Hash()) + if err != nil { + t.Fatal(err) + } + if mined.Final { + t.Fatal("initial receipt final") + } + ctx, finalCancel := context.WithCancel(context.Background()) + finalCancel() + done := make(chan error, 1) + go func() { _, err := wait(ctx); done <- err }() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("wait error: %v", err) + } + case <-time.After(100 * time.Millisecond): + cancel() + <-done + t.Error("finality waiter ignored its own context") + } +} + +func TestHardeningConcurrentRunStop(t *testing.T) { + for i := 0; i < 5; i++ { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + _, l.ctxStop = context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var wg sync.WaitGroup + start := make(chan struct{}) + for j := 0; j < 16; j++ { + wg.Add(1) + go func() { defer wg.Done(); <-start; _ = l.Run(ctx) }() + } + wg.Add(1) + go func() { + defer wg.Done() + <-start + for j := 0; j < 16; j++ { + l.Stop() + } + }() + close(start) + wg.Wait() + } +} + +func TestHardeningMonitorClosureCancelsHeadWait(t *testing.T) { + p := &hardeningProvider{} + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger = log + mo.WithLogs = true + mo.BlockRetentionLimit = 100 + mo.Bootstrap = true + monitor, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, monitor, hardeningOptions()) + if err != nil { + t.Fatal(err) + } + parent, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- l.Run(parent) }() + deadline := time.After(time.Second) + for monitor.NumSubscribers() != 1 { + select { + case <-deadline: + t.Fatal("listener did not subscribe") + default: + time.Sleep(time.Millisecond) + } + } + monitor.UnsubscribeAll(errors.New("monitor closed before first block")) + select { + case <-done: + case <-time.After(100 * time.Millisecond): + cancel() + // Baseline LatestBlockNum has a fixed 30-second wait; let it finish + // without mutating the monitor's bootstrap state during concurrent reads. + select { + case <-done: + case <-time.After(32 * time.Second): + t.Fatal("Run did not clean up") + } + t.Error("monitor closure did not cancel waiting for first head") + } +} + +// blockingAlerter holds the subscriber channel's piping goroutine, as a slow +// alerting backend would, until released. +type blockingAlerter struct { + entered chan struct{} + once sync.Once + release chan struct{} +} + +func (a *blockingAlerter) Alert(context.Context, string, ...interface{}) { + a.once.Do(func() { close(a.entered) }) + <-a.release +} + +func TestReceiptsFixBlockedSendKeepsListenerUnlocked(t *testing.T) { + var txns []*types.Transaction + for nonce := uint64(820); nonce < 820+subscriberQueueWarning+2; nonce++ { + tx, _, _ := hardeningTxn(t, nonce) + txns = append(txns, tx) + } + b := hardeningBlock(100, txns...) + alerter := &blockingAlerter{entered: make(chan struct{}), release: make(chan struct{})} + opts := hardeningOptions() + opts.Alerter = alerter + l := hardeningListener(t, &hardeningProvider{}, opts) + q := FilterLogs(func([]*types.Log) bool { return true }) + s := l.Subscribe(q).(*subscriber) + owner := s.owner(q.(Filterer)) + published := make(chan struct{}) + go func() { + defer close(published) + // Nobody reads, so the receipt that passes subscriberQueueWarning raises + // an alert and the next one blocks in Send. + for _, tx := range txns { + s.publish(context.Background(), Receipt{receipt: hardeningReceipt(b, tx)}, owner) + } + }() + t.Cleanup(func() { + close(alerter.release) + select { + case <-published: + s.Unsubscribe() + case <-time.After(time.Second): + t.Error("publisher did not stop after releasing the alert") + } + }) + select { + case <-alerter.entered: + case <-published: + t.Fatal("publisher finished without raising a queue alert") + case <-time.After(time.Second): + t.Fatal("subscriber queue alert did not fire") + } + for deadline := time.Now().Add(200 * time.Millisecond); time.Now().Before(deadline); { + checked := make(chan struct{}) + go func() { + defer close(checked) + l.isCurrentBlock(b.Hash(), 0) + }() + select { + case <-checked: + case <-time.After(time.Second): + t.Fatal("receiptMu held across a blocked subscriber send") + } + } +} + +// publicationContext signals the first cancellation check without changing +// its result, so the test can cancel while the subsequent block check waits. +type publicationContext struct { + context.Context + once sync.Once + checked chan struct{} +} + +func (c *publicationContext) Err() error { + err := c.Context.Err() + c.once.Do(func() { close(c.checked) }) + return err +} + +func TestSubscriptionPublishCancellationDuringBlockCheck(t *testing.T) { + tx, _, _ := hardeningTxn(t, 9903) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions(), b) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + base, cancel := context.WithCancel(context.Background()) + defer cancel() + ctx := &publicationContext{Context: base, checked: make(chan struct{})} + done := make(chan bool, 1) + l.receiptMu.Lock() + go func() { + done <- s.publish(ctx, Receipt{receipt: hardeningReceipt(b, tx)}, owner) + }() + select { + case <-ctx.checked: + case <-time.After(time.Second): + l.receiptMu.Unlock() + t.Fatal("publication did not check cancellation") + } + cancel() + l.receiptMu.Unlock() + select { + case published := <-done: + if published { + t.Error("receipt published after cancellation during block validation") + } + case <-time.After(time.Second): + t.Fatal("publication did not finish") + } + if len(s.deliveries) != 0 || s.finalizer.hasOwner(owner) { + t.Error("canceled publication recorded delivery or queued finality") + } + hardeningNoReceipt(t, s) +} + +// customWaitFilter reports its own start block and never matches. +type customWaitFilter struct{ Filterer } + +func (customWaitFilter) StartBlockNum() uint64 { return 1 } +func (customWaitFilter) Match(context.Context, Receipt) (bool, error) { return false, nil } + +func TestReceiptsFixCustomFilterMaxWait(t *testing.T) { + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{}, + canonical: make(map[uint64]*types.Block), + blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 100 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99)}); err != nil { + t.Fatal(err) + } + l, err := NewReceiptsListener(log, p, m, hardeningOptions()) + if err != nil { + t.Fatal(err) + } + nobody := common.HexToAddress("0x3333333333333333333333333333333333333333") + custom := customWaitFilter{FilterFrom(nobody).MaxWait(1).(Filterer)} + builtin := FilterFrom(nobody).MaxWait(1) + s := l.Subscribe(custom, builtin) + defer s.Unsubscribe() + hardeningStart(t, l) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor Run did not stop") + } + }) + // One block at a time, so the built-in filter sees blocks after its start. + for num := int64(100); num <= 102; num++ { + p.advance(hardeningBlock(num).Block) + for deadline := time.Now().Add(5 * time.Second); m.LatestBlockNum().Int64() != num; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("monitor did not reach block %d", num) + } + } + } + select { + case <-builtin.(Filterer).Exhausted(): + case <-time.After(5 * time.Second): + t.Fatal("built-in filter did not exhaust") + } + if filters := s.Filters(); len(filters) != 1 || filters[0] != custom { + t.Fatal("custom filter was not left to manage its own MaxWait") + } +} diff --git a/ethreceipts/ownership_regression_test.go b/ethreceipts/ownership_regression_test.go new file mode 100644 index 00000000..b3d1cd5d --- /dev/null +++ b/ethreceipts/ownership_regression_test.go @@ -0,0 +1,572 @@ +package ethreceipts + +import ( + "context" + "errors" + "fmt" + "math/big" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningAddressFilters(t *testing.T) { + tx, from, to := hardeningTxn(t, 3) + b := hardeningBlock(100, tx) + for _, tc := range []struct { + name string + q FilterQuery + }{{"from", FilterFrom(from)}, {"to", FilterTo(to)}} { + t.Run(tc.name, func(t *testing.T) { + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions(), b) + s := l.Subscribe(tc.q.SearchCache(true)) + defer s.Unsubscribe() + hardeningStart(t, l) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() { + t.Fatalf("wrong matching transaction: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("signed address filter missed cached transaction") + } + }) + } +} + +func TestHardeningLimitOneSnapshots(t *testing.T) { + for _, sameBlock := range []bool{true, false} { + t.Run(map[bool]string{true: "same_block", false: "retained_blocks"}[sameBlock], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 7) + tx2, _, _ := hardeningTxn(t, 8) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + blocks := ethmonitor.Blocks{b1, b2} + if sameBlock { + b1 = hardeningBlock(100, tx1, tx2) + b2 = b1 + blocks = ethmonitor.Blocks{b1} + } + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + return hardeningReceipt(b1, tx1), nil + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true)).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + if _, err := l.processCachedBlocks(context.Background(), blocks, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + r := hardeningRead(t, s) + if r.TransactionHash() != tx1.Hash() { + t.Errorf("LimitOne did not select first match: %s", r.TransactionHash()) + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestHardeningConcurrentLimitOne(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 12) + tx2, _, _ := hardeningTxn(t, 13) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + close(entered) + select { + case <-release: + return hardeningReceipt(b1, tx1), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + done := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{fs}) + done <- err + }() + <-entered + if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b2}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + close(release) + <-done + r := hardeningRead(t, s) + if r.TransactionHash() != tx1.Hash() || r.Filter != q { + t.Errorf("concurrent LimitOne selected different transaction: %+v", r) + } + hardeningNoReceipt(t, s) + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.TransactionHash() != tx1.Hash() || final.Filter != q { + t.Errorf("wrong claimed final: %+v", final) + } + hardeningNoReceipt(t, s) +} + +func TestHardeningExplicitRemovalCancelsOnlyOwner(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove_one", true: "clear_all"}[clearAll], func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 16) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(1).Finalize(true) + q2 := FilterLogs(func([]*types.Log) bool { return true }).ID(2).Finalize(true) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, b) + hardeningRead(t, s) + hardeningRead(t, s) + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q1.(Filterer)) + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + if !clearAll { + r := hardeningRead(t, s) + if !r.Final || r.Filter != q2 || r.BlockHash() != b.Hash() { + t.Errorf("removal lost surviving owner: %+v", r) + } + } + hardeningNoReceipt(t, s) + }) + } +} + +type ownershipValueFilter struct { + Filterer + labels []string + match func(context.Context, Receipt) (bool, error) +} + +func (f ownershipValueFilter) Match(ctx context.Context, r Receipt) (bool, error) { + return f.match(ctx, r) +} + +func TestOwnershipCustomValueFilter(t *testing.T) { + tx, _, _ := hardeningTxn(t, 100) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(b, tx), nil }} + l := hardeningListener(t, p, hardeningOptions(), b) + base := FilterLogs(func([]*types.Log) bool { return false }).SearchCache(true).(Filterer) + var matches atomic.Int32 + q := ownershipValueFilter{base, []string{"custom"}, func(_ context.Context, r Receipt) (bool, error) { + matches.Add(1) + return r.TransactionHash() == tx.Hash(), nil + }} + s := l.Subscribe(q) + defer s.Unsubscribe() + gotFilter, ok := s.Filters()[0].(ownershipValueFilter) + if !ok || gotFilter.Filterer != base || gotFilter.labels[0] != "custom" { + t.Fatal("public Filters lost original custom value") + } + hardeningStart(t, l) + r := hardeningRead(t, s) + original, ok := r.Filter.(ownershipValueFilter) + if !ok || original.Filterer != base || original.labels[0] != "custom" || original.match == nil { + t.Fatal("receipt lost original custom Filterer value") + } + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || matches.Load() == 0 { + t.Fatal("custom Match behavior lost") + } + s.ClearFilters() +} + +type ownershipCollectionFilter struct { + Filterer + cancel context.CancelFunc +} + +func (f *ownershipCollectionFilter) Match(context.Context, Receipt) (bool, error) { + if f.cancel != nil { + f.cancel() + return false, nil + } + return false, errors.New("temporary match failure") +} + +func TestOwnershipCollectionAbort(t *testing.T) { + for _, canceled := range []bool{false, true} { + t.Run(map[bool]string{false: "matcher_error", true: "cancellation"}[canceled], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 101) + tx2, _, _ := hardeningTxn(t, 102) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + bad := &ownershipCollectionFilter{Filterer: FilterLogs(func([]*types.Log) bool { return false }).(Filterer)} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if canceled { + bad.cancel = cancel + } + s := l.Subscribe(q, bad).(*subscriber) + defer s.Unsubscribe() + _, err := s.matchFiltersAndPublish(ctx, s.Filters(), []Receipt{{receipt: hardeningReceipt(b1, tx1)}}) + if err == nil { + t.Error("expected matching abort") + } + if canceled && !errors.Is(err, context.Canceled) { + t.Errorf("cancellation error: %v", err) + } + if len(s.inFlight) != 0 || len(s.claims) != 0 { + t.Errorf("aborted collection retained work: inFlight=%d claims=%d", len(s.inFlight), len(s.claims)) + } + hardeningNoReceipt(t, s) + s.RemoveFilter(bad) + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b2, tx2)}}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("unused claim blocked later transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("aborted reservation blocked later processing") + } + }) + } +} + +func TestOwnershipQueryReuse(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove", true: "clear"}[clearAll], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 103) + tx2, _, _ := hardeningTxn(t, 104) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b1, tx1)}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q.(Filterer)) + } + if len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 { + t.Error("explicit removal retained canceled ownership") + } + s.AddFilter(q) + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b2, tx2)}}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q || r.Final { + t.Error("reused query kept old selection") + } + case <-time.After(100 * time.Millisecond): + t.Error("reused LimitOne query missed second transaction") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("canceled owner finalized in new lifetime") + } + case <-time.After(100 * time.Millisecond): + t.Error("new owner did not finalize") + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestOwnershipReaddRejectsOldWorker(t *testing.T) { + for _, clearAll := range []bool{false, true} { + t.Run(map[bool]string{false: "remove", true: "clear"}[clearAll], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 105) + tx2, _, _ := hardeningTxn(t, 106) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx1.Hash() { + close(entered) + select { + case <-release: + return hardeningReceipt(b1, tx1), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(b2, tx2), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{b1}, []*subscriber{s}, [][]Filterer{s.Filters()}) + done <- err + }() + <-entered + if clearAll { + s.ClearFilters() + } else { + s.RemoveFilter(q.(Filterer)) + } + s.AddFilter(q) + hardeningProcess(t, l, s, b2) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("new lifetime selected old transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("new lifetime suppressed by old claim") + } + close(release) + <-done + hardeningNoReceipt(t, s) + if len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 { + t.Error("completed/canceled lifetime retained state") + } + }) + } +} + +func TestOwnershipCompletionReleasesState(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + s := l.Subscribe().(*subscriber) + defer s.Unsubscribe() + for i := 0; i < 32; i++ { + tx, _, _ := hardeningTxn(t, uint64(200+i)) + b := hardeningBlock(100, tx) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(i%4 != 0) + s.AddFilter(q) + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + switch i % 4 { + case 1: + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + case 2: + s.RemoveFilter(q.(Filterer)) + case 3: + s.ClearFilters() + } + if len(s.Filters()) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 { + t.Fatalf("completion %d retained state: active=%d claims=%d inFlight=%d deliveries=%d", i, len(s.Filters()), len(s.claims), len(s.inFlight), len(s.deliveries)) + } + if len(s.finalizer.queue) != 0 || len(s.finalizer.txns) != 0 || len(s.pendingReceipts) != 0 { + t.Fatalf("completion %d retained queued ownership", i) + } + } +} + +func TestReceiptsFixCustomValueRemoval(t *testing.T) { + for _, pointer := range []bool{false, true} { + for _, exhausted := range []bool{false, true} { + for _, source := range []string{"original", "Filters", "Receipt.Filter"} { + t.Run(fmt.Sprintf("pointer=%v/exhausted=%v/%s", pointer, exhausted, source), func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 801) + tx2, _, _ := hardeningTxn(t, 802) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + custom := func(label string) Filterer { + return ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).ID(7).(Filterer), []string{label}, func(context.Context, Receipt) (bool, error) { return label != "", nil }} + } + value := custom("selected").(ownershipValueFilter) + var q Filterer = value + if pointer { + q = &value + } + other := custom("surviving") + s := l.Subscribe(q, other).(*subscriber) + defer s.Unsubscribe() + first, second := s.filterers()[0].(*filterOwner), s.filterers()[1].(*filterOwner) + fromFilters := s.Filters()[0] + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(b1, tx1)) + var mined Receipt + for i := 0; i < 2; i++ { + r := hardeningRead(t, s) + if r.owner == first { + mined = r + if pointer { + if r.Filter != q || fromFilters != q { + t.Fatal("public pointer identity changed") + } + } else if r.Filter.(ownershipValueFilter).Filterer != value.Filterer || fromFilters.(ownershipValueFilter).Filterer != value.Filterer { + t.Fatal("original public custom value changed") + } + } + } + if exhausted { + s.exhaustFilter(first) + value.Filterer.(*filter).closeExhausted() + } + candidate := q + switch source { + case "Filters": + candidate = fromFilters + case "Receipt.Filter": + candidate = mined.Filter + } + s.RemoveFilter(candidate) + if s.hasFilter(first) || s.finalizer.hasOwner(first) || !s.hasFilter(second) || !s.finalizer.hasOwner(second) { + t.Error("public removal did not isolate selected registration/finality") + } + process(hardeningReceipt(b2, tx2)) + late := receiptsFixCollect(t, s) + if len(late) != 1 || late[0].owner != second || late[0].TransactionHash() != tx2.Hash() || late[0].Final || late[0].Reorged { + t.Error("selected cancellation changed later surviving delivery") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(t, s) + seen := make(map[common.Hash]bool) + for _, r := range finals { + if r.owner != second || !r.Final || r.Reorged || seen[r.TransactionHash()] { + t.Error("removed custom value still finalized or surviving final duplicated") + } + seen[r.TransactionHash()] = true + } + if len(finals) != 2 || !seen[tx1.Hash()] || !seen[tx2.Hash()] { + t.Error("custom removal lost surviving finals") + } + }) + } + } + } +} + +func TestReceiptsFixSharedBaseAliases(t *testing.T) { + tx, _, _ := hardeningTxn(t, 803) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + base := FilterLogs(func([]*types.Log) bool { return false }).Finalize(true).(Filterer) + first := ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + second := ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + other := ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer), []string{"other"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + s := l.Subscribe(first, second, other).(*subscriber) + defer s.Unsubscribe() + owners := s.filterers() + if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) + } + for i := 0; i < 3; i++ { + hardeningRead(t, s) + } + // Shared base signals define aliases: old RemoveFilter removes the first + // matching registration. Independent callback identities use distinct bases. + s.RemoveFilter(second) + if s.hasFilter(owners[0].(*filterOwner)) || s.finalizer.hasOwner(owners[0].(*filterOwner)) || !s.hasFilter(owners[1].(*filterOwner)) || !s.hasFilter(owners[2].(*filterOwner)) { + t.Error("shared-base removal did not preserve first-match alias semantics") + } + s.RemoveFilter(s.Filters()[0]) + if s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) || !s.finalizer.hasOwner(owners[2].(*filterOwner)) { + t.Error("removing second base alias affected independent owner") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(t, s) + if len(finals) != 1 || finals[0].owner != owners[2] || !finals[0].Final || finals[0].TransactionHash() != tx.Hash() { + t.Error("shared aliases canceled an unrelated queued final") + } +} + +type receiptsFixComparableFilter struct { + Filterer + label string +} + +func (f receiptsFixComparableFilter) Match(context.Context, Receipt) (bool, error) { + return f.label != "", nil +} + +func TestReceiptsFixComparableSharedBaseIsolation(t *testing.T) { + for _, pointer := range []bool{false, true} { + t.Run(fmt.Sprintf("pointer=%v", pointer), func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 804) + b := hardeningBlock(100, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + base := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).(Filterer) + var first, second Filterer = receiptsFixComparableFilter{base, "first"}, receiptsFixComparableFilter{base, "second"} + if pointer { + first = &ownershipValueFilter{base, []string{"first"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + second = &ownershipValueFilter{base, []string{"second"}, func(context.Context, Receipt) (bool, error) { return true, nil }} + } + s := l.Subscribe(first, second).(*subscriber) + defer s.Unsubscribe() + owners := s.filterers() + if _, err := s.matchFiltersAndPublish(context.Background(), owners, []Receipt{{receipt: hardeningReceipt(b, tx)}}); err != nil { + t.Fatal(err) + } + hardeningRead(t, s) + hardeningRead(t, s) + s.RemoveFilter(second) + if !s.hasFilter(owners[0].(*filterOwner)) || !s.finalizer.hasOwner(owners[0].(*filterOwner)) || s.hasFilter(owners[1].(*filterOwner)) || s.finalizer.hasOwner(owners[1].(*filterOwner)) { + t.Fatal("base fallback overrode definitive comparable identity") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + finals := receiptsFixCollect(t, s) + if len(finals) != 1 || finals[0].owner != owners[0] || !finals[0].Final { + t.Fatal("comparable cancellation lost surviving owner") + } + }) + } +} + +type receiptsFixNilSignalFilter struct{ ownershipValueFilter } + +func (receiptsFixNilSignalFilter) Exhausted() <-chan struct{} { return nil } + +func TestReceiptsFixCustomValueWithoutIdentity(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + value := receiptsFixNilSignalFilter{ownershipValueFilter{FilterLogs(func([]*types.Log) bool { return true }).(Filterer), []string{"no identity"}, func(context.Context, Receipt) (bool, error) { return true, nil }}} + other := FilterLogs(func([]*types.Log) bool { return true }).(Filterer) + s := l.Subscribe(value, other).(*subscriber) + defer s.Unsubscribe() + s.RemoveFilter(s.Filters()[0]) + if len(s.Filters()) != 2 { + t.Fatal("unidentifiable callback value removed another registration") + } + s.ClearFilters() + s.AddFilter(&value, other) + s.RemoveFilter(s.Filters()[0]) + if len(s.Filters()) != 1 || s.Filters()[0] != other { + t.Fatal("pointer identity did not isolate custom nil-signal removal") + } +} diff --git a/ethreceipts/pending_receipts_test.go b/ethreceipts/pending_receipts_test.go new file mode 100644 index 00000000..54141e23 --- /dev/null +++ b/ethreceipts/pending_receipts_test.go @@ -0,0 +1,357 @@ +package ethreceipts + +import ( + "context" + "errors" + "math/big" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningPendingOwners(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6) + b := hardeningBlock(100, tx) + var failing atomic.Bool + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if failing.Load() { + return nil, errors.New("temporary provider failure") + } + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q1 := FilterLogs(func([]*types.Log) bool { return true }).ID(0).Finalize(true) + q2 := FilterTxnHash(tx.Hash()).ID(0) + s := l.Subscribe(q1, q2).(*subscriber) + defer s.Unsubscribe() + _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, chainID: big.NewInt(1)}}) + if err == nil { + t.Fatal("expected transient fetch failure") + } + failing.Store(false) + s.retryMu.Lock() + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + } + s.retryMu.Unlock() + s.retryPendingReceipts(context.Background()) + owners := map[Filterer]bool{} + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if r.Final || r.BlockHash() != b.Hash() { + t.Errorf("wrong recovered receipt: %+v", r) + } + owners[r.Filter] = true + case <-time.After(100 * time.Millisecond): + t.Error("missing recovered owner") + } + } + if !owners[q1.(Filterer)] || !owners[q2.(Filterer)] { + t.Error("retry lost an owner") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.BlockHash() != b.Hash() { + t.Errorf("wrong recovered final: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing recovered final") + } + } +} + +func TestOwnershipPendingDropReleasesClaim(t *testing.T) { + for _, notFound := range []bool{false, true} { + t.Run(map[bool]string{false: "retry_limit", true: "not_found"}[notFound], func(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 108) + tx2, _, _ := hardeningTxn(t, 109) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + p := &hardeningProvider{receipt: func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx2.Hash() { + return hardeningReceipt(b2, tx2), nil + } + return nil, errors.New("temporary receipt failure") + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx1, blockHash: b1.Hash(), blockNum: b1.Number()}}); err == nil { + t.Fatal("expected initial pending fetch failure") + } + if len(s.pendingReceipts) != 1 || len(s.claims) != 1 { + t.Fatal("pending fetch lost its selection") + } + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + pending.attempts = maxReceiptRetryAttempts - 1 + } + if notFound { + p.receipt = func(_ context.Context, h common.Hash) (*types.Receipt, error) { + if h == tx2.Hash() { + return hardeningReceipt(b2, tx2), nil + } + return nil, ethereum.NotFound + } + } + s.retryPendingReceipts(context.Background()) + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 { + t.Error("discarded pending work retained an undelivered claim") + } + hardeningProcess(t, l, s, b2) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx2.Hash() || r.Filter != q { + t.Error("released pending claim selected wrong transaction") + } + case <-time.After(100 * time.Millisecond): + t.Error("discarded pending work blocked a new transaction") + } + }) + } +} + +func TestReceiptsFixRetryDeadlineAfterFetch(t *testing.T) { + for _, terminal := range []string{"retry", "remove", "clear", "rollback"} { + t.Run(terminal, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 800) + b := hardeningBlock(100, tx) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), b) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + seed := Receipt{transaction: tx, blockHash: b.Hash(), blockNum: b.Number()} + if !s.reserve(seed, owner) { + t.Fatal("initial reservation failed") + } + s.addPendingReceipt(seed, owner) + s.releaseReservation(seed, owner) + key := receiptOwner(seed, owner) + s.pendingReceipts[key].nextRetryAt = time.Time{} + pending := s.pendingReceipts[key] + parent := context.Background() + attempt, cancel := context.WithTimeout(parent, 200*time.Millisecond) + defer cancel() + s.deliveryMu.Lock() + done := make(chan struct{}) + go func() { s.retryPendingReceipts(attempt); close(done) }() + deadline := time.Now().Add(time.Second) + for { + _, cached, _ := l.pastReceipts.Get(parent, tx.Hash().Hex()) + if cached && len(l.fetchSem) == 0 { + break + } + if time.Now().After(deadline) { + s.deliveryMu.Unlock() + t.Fatal("receipt fetch did not complete before publication") + } + time.Sleep(time.Millisecond) + } + <-attempt.Done() + terminalDone := make(chan struct{}) + go func() { + defer close(terminalDone) + switch terminal { + case "remove": + s.RemoveFilter(q.(Filterer)) + case "clear": + s.ClearFilters() + case "rollback": + ref := l.invalidateBlock(parent, b) + s.rollbackBlock(ref) + } + }() + s.deliveryMu.Unlock() + <-done + <-terminalDone + hardeningNoReceipt(t, s) + if parent.Err() != nil { + t.Fatal("retry child canceled its listener parent") + } + if terminal == "retry" { + if s.pendingReceipts[key] != pending || len(s.claims) != 1 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Errorf("expired successful retry lost exact ownership: pending=%d claims=%d", len(s.pendingReceipts), len(s.claims)) + } + if s.pendingReceipts[key] == nil { + return + } + if pending.attempts != 1 || pending.nextRetryAt.After(time.Now().Add(time.Second)) { + t.Error("successful expired attempt was not promptly released for retry") + } + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(parent) + r := hardeningRead(t, s) + if r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q || r.Final || r.Reorged { + t.Error("later active retry lost mined identity") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + r = hardeningRead(t, s) + if !r.Final || r.Reorged || r.TransactionHash() != tx.Hash() || r.BlockHash() != b.Hash() || r.Filter != q { + t.Error("later active retry lost finality identity") + } + if calls.Load() != 1 { + t.Error("later retry discarded its successful cached receipt") + } + } else { + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Error("expired retry resurrected terminal owner/block state") + } + s.retryPendingReceipts(parent) + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestReceiptsFixPartialPendingCandidate(t *testing.T) { + for _, terminal := range []string{"retry", "rollback", "readoption", "remove", "clear"} { + t.Run(terminal, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 808) + replacement, _, _ := hardeningTxn(t, 809) + old, next := hardeningBlock(100, tx), hardeningBlock(101, replacement) + var ready atomic.Bool + var calls atomic.Int32 + l := hardeningListener(t, &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if !ready.Load() { + return nil, errors.New("provider temporarily unavailable") + } + return hardeningReceipt(old, tx), nil + }}, hardeningOptions(), old) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true). + QueryOnChain(func(context.Context) (*types.Receipt, error) { return &types.Receipt{TxHash: tx.Hash()}, nil }) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + if err := l.queryFilterOnChain(context.Background(), s, s.filterers()); err != nil { + t.Fatal(err) + } + if len(s.pendingReceipts) != 1 || s.claims[owner] != tx.Hash() { + t.Fatal("partial public callback did not enter the pending completion path") + } + var key receiptKey + var pending *pendingReceipt + for k, p := range s.pendingReceipts { + key, pending = k, p + } + if key.blockHash != (common.Hash{}) { + t.Fatal("partial callback already knew its block") + } + pending.nextRetryAt = time.Time{} + ready.Store(true) + attempt, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + s.deliveryMu.Lock() + done := make(chan struct{}) + go func() { s.retryPendingReceipts(attempt); close(done) }() + deadline := time.Now().Add(time.Second) + for { + _, cached, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if cached && len(l.fetchSem) == 0 { + break + } + if time.Now().After(deadline) { + s.deliveryMu.Unlock() + t.Fatal("partial pending fetch did not complete") + } + time.Sleep(time.Millisecond) + } + <-attempt.Done() + s.deliveryMu.Unlock() + <-done + hardeningNoReceipt(t, s) + if s.pendingReceipts[key] != pending || pending.attempts != 1 || calls.Load() != 2 { + t.Fatal("successful canceled completion did not retain its exact pending owner") + } + if terminal == "retry" { + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(context.Background()) + mined := hardeningRead(t, s) + if mined.Filter != q || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != old.Hash() || mined.generation != 0 || mined.Final || mined.Reorged { + t.Fatal("partial pending live retry lost its learned candidate") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if final.Filter != q || final.TransactionHash() != tx.Hash() || final.BlockHash() != old.Hash() || !final.Final || final.Reorged || calls.Load() != 2 { + t.Fatal("partial retry lost cached completion or canonical finality") + } + hardeningNoReceipt(t, s) + return + } + switch terminal { + case "rollback", "readoption": + ref := l.invalidateBlock(context.Background(), old) + if terminal == "readoption" { + // Even if authoritative same-hash acceptance wins before subscriber + // cleanup, the learned generation-0 candidate must stay invalid. + l.acceptBlock(l.monitor.GetBlock(old.Hash())) + pending.nextRetryAt = time.Time{} + s.retryPendingReceipts(context.Background()) + hardeningNoReceipt(t, s) + } + s.rollbackBlock(ref) + case "remove": + s.RemoveFilter(q.(Filterer)) + case "clear": + s.ClearFilters() + } + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.inFlight) != 0 || len(s.deliveries) != 0 || len(s.finalizer.queue) != 0 { + t.Error("terminal invalidation retained a learned pending candidate or selection") + } + s.retryPendingReceipts(context.Background()) + hardeningNoReceipt(t, s) + if terminal == "remove" || terminal == "clear" { + s.AddFilter(q) + if s.owner(q.(Filterer)) == owner { + t.Fatal("new registration reused the canceled owner") + } + } + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: hardeningReceipt(next, replacement)}}); err != nil { + t.Fatal(err) + } + select { + case mined := <-s.TransactionReceipt(): + if mined.Filter != q || mined.TransactionHash() != replacement.Hash() || mined.BlockHash() != next.Hash() || mined.Final || mined.Reorged { + t.Error("fresh replacement lost its exact mined identity") + } + case <-time.After(100 * time.Millisecond): + t.Error("orphan pending claim blocked a fresh canonical replacement") + return + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if final.Filter != q || final.TransactionHash() != replacement.Hash() || final.BlockHash() != next.Hash() || !final.Final || final.Reorged { + t.Error("terminal invalidation published an orphan final") + } + if len(s.pendingReceipts) != 0 || len(s.claims) != 0 || len(s.Filters()) != 0 || calls.Load() != 2 { + t.Error("terminal candidate or completed owner was retained/re-fetched") + } + hardeningNoReceipt(t, s) + }) + } +} diff --git a/ethreceipts/receipt.go b/ethreceipts/receipt.go index 01ad8b96..94af72df 100644 --- a/ethreceipts/receipt.go +++ b/ethreceipts/receipt.go @@ -21,9 +21,12 @@ type Receipt struct { transaction *types.Transaction receipt *types.Receipt logs []*types.Log + blockNum *big.Int + blockHash common.Hash + generation uint64 + owner *filterOwner - // TODOXXX: this intermediate type is lame.. with new ethrpc we can remove - // NOTE: we only use this for From/To address resolution currently + // Cache transaction message decoding for From/To address resolution. message atomic.Value } @@ -61,7 +64,7 @@ func (r *Receipt) BlockNumber() *big.Int { if r.receipt != nil { return r.receipt.BlockNumber } else { - return nil + return r.blockNum } } @@ -69,7 +72,7 @@ func (r *Receipt) BlockHash() ethkit.Hash { if r.receipt != nil { return r.receipt.BlockHash } else { - return ethkit.Hash{} + return r.blockHash } } @@ -175,25 +178,22 @@ func (r *Receipt) To() common.Address { } func (r *Receipt) AsMessage() (*core.Message, error) { - msg, ok := r.message.Load().(*core.Message) - if !ok { - return nil, fmt.Errorf("ethreceipts: Receipt.message type-assertion fail, unexpected") + cached := r.message.Load() + if cached != nil { + msg, ok := cached.(*core.Message) + if !ok { + return nil, fmt.Errorf("ethreceipts: Receipt.message type-assertion fail, unexpected") + } + if msg != nil { + return msg, nil + } } - if msg != nil { - return msg, nil + if r.transaction == nil || r.chainID == nil { + return nil, fmt.Errorf("ethreceipts: transaction and chainID required to resolve message") } - // TODOXXX: avoid using AsMessage as its fairly expensive operation, especially - // to do it for every txn for every filter. - // TODO: in order to do this, we'll have to update ethrpc with a different - // implementation to just use raw types, aka, ethrpc/types.go with Block/Transaction/Receipt/Log .. txnMsg, err := ethtxn.AsMessage(r.transaction, r.chainID) if err != nil { - // NOTE: this should never happen, but lets log in case it does. In the - // future, we should just not use go-ethereum for these types. - // l.log.Warn(fmt.Sprintf("unexpected failure of txn (%s index %d) on block %d (total txns=%d) AsMessage(..): %s", - // txn.Hash(), i, block.NumberU64(), len(block.Transactions()), err, - // )) return nil, err } r.message.Store(txnMsg) diff --git a/ethreceipts/receipt_fetch_test.go b/ethreceipts/receipt_fetch_test.go new file mode 100644 index 00000000..59c9b634 --- /dev/null +++ b/ethreceipts/receipt_fetch_test.go @@ -0,0 +1,258 @@ +package ethreceipts + +import ( + "context" + "errors" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningQueryOnChainUsesCompleteReceipt(t *testing.T) { + tx, _, _ := hardeningTxn(t, 9) + b := hardeningBlock(90, tx) + complete := hardeningReceipt(b, tx) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + return nil, ethereum.NotFound + }} + l := hardeningListener(t, p, hardeningOptions()) + s := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil })).(*subscriber) + defer s.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.Receipt() != complete || r.BlockHash() != b.Hash() { + t.Fatal("callback receipt replaced") + } + case <-time.After(100 * time.Millisecond): + t.Error("complete callback receipt was not delivered") + } + if calls.Load() != 0 { + t.Errorf("complete callback caused %d redundant RPCs", calls.Load()) + } +} + +func TestHardeningFetchPreservesSharedQuery(t *testing.T) { + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }) + s := l.Subscribe(q) + defer s.Unsubscribe() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { defer wg.Done(); _, _, _ = l.FetchTransactionReceiptWithFilter(ctx, q) }() + } + wg.Wait() + opts := s.Filters()[0].Options() + if opts.LimitOne || opts.SearchCache { + t.Fatalf("Fetch rewrote active subscription: %+v", opts) + } +} + +// This builder becomes a Filterer only after the optional Finalize step. +type fetchFinalizeBuilder struct{ FilterQuery } + +func (b *fetchFinalizeBuilder) LimitOne(v bool) FilterQuery { + b.FilterQuery = b.FilterQuery.LimitOne(v) + return b +} + +func (b *fetchFinalizeBuilder) SearchCache(v bool) FilterQuery { + b.FilterQuery = b.FilterQuery.SearchCache(v) + return b +} + +func TestFetchFilterCompatibility(t *testing.T) { + for _, name := range []string{"builtin", "custom_pointer", "custom_value", "builder", "finalize_builder"} { + t.Run(name, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 3002) + b := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), b) + var calls atomic.Int32 + match := func(context.Context, Receipt) (bool, error) { calls.Add(1); return true, nil } + base := FilterLogs(func([]*types.Log) bool { calls.Add(1); return true }).MaxWait(4).(Filterer) + var query FilterQuery = base + var public Filterer = base + builder := false + switch name { + case "custom_pointer", "custom_value": + // Only the custom Match accepts receipts, proving it survives the snapshot. + base = FilterLogs(func([]*types.Log) bool { return false }).MaxWait(4).(Filterer) + custom := ownershipValueFilter{base, []string{"fetch"}, match} + if name == "custom_pointer" { + public = &custom + } else { + public = custom + } + query = public + case "builder": + query = struct{ FilterQuery }{base} + builder = true + case "finalize_builder": + query = &fetchFinalizeBuilder{base} + builder = true + } + var other Subscription + if !builder { + other = l.Subscribe(public) + defer other.Unsubscribe() + } + hardeningStart(t, l) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + mined, waitFinal, err := l.FetchTransactionReceiptWithFilter(ctx, query, true) + if err != nil { + t.Fatal(err) + } + if mined == nil || mined.Final || mined.Reorged || mined.TransactionHash() != tx.Hash() || !sameFilter(mined.Filter, public) { + t.Fatalf("wrong mined receipt or public filter: %+v", mined) + } + if calls.Load() == 0 { + t.Fatal("source matching behavior was not used") + } + if !builder { + opts := public.Options() + if opts.LimitOne || opts.SearchCache || opts.Finalize || opts.MaxWait == nil || *opts.MaxWait != 4 { + t.Fatalf("fetch mutated shared filter options: %+v", opts) + } + } + if public.StartBlockNum() != 0 || public.LastMatchBlockNum() != 0 { + t.Fatal("fetch mutated public filter counters") + } + select { + case <-public.Exhausted(): + t.Fatal("fetch exhausted the public filter") + default: + } + l.mu.Lock() + var helper *subscriber + for _, s := range l.subscribers { + if s != other { + helper = s + } + } + l.mu.Unlock() + if helper == nil { + t.Fatal("fetch helper unsubscribed before finality") + } + if err := helper.finalizeReceipts(big.NewInt(102)); err != nil { + t.Fatal(err) + } + final, err := waitFinal(ctx) + if err != nil || final == nil || !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || !sameFilter(final.Filter, public) { + t.Fatalf("wrong final receipt or public filter: %+v, error: %v", final, err) + } + wantSubscribers := 0 + if other != nil { + wantSubscribers = 1 + } + if l.NumSubscribers() != wantSubscribers { + t.Error("fetch completion removed the wrong subscription") + } + }) + } +} + +type ownershipGateFilter struct { + Filterer + entered, release chan struct{} + calls atomic.Int32 +} + +func (f *ownershipGateFilter) Match(ctx context.Context, _ Receipt) (bool, error) { + if f.calls.Add(1) == 1 { + close(f.entered) + select { + case <-f.release: + case <-ctx.Done(): + return false, ctx.Err() + } + } + return false, nil +} + +func TestOwnershipStaleFetchPreservesCache(t *testing.T) { + for _, sameHash := range []bool{false, true} { + t.Run(map[bool]string{false: "remined_block", true: "readopted_generation"}[sameHash], func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 107) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + if sameHash { + canonical = old + } + complete := hardeningReceipt(canonical, tx) + var calls atomic.Int32 + var unavailable atomic.Bool + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if unavailable.Load() { + return nil, ethereum.NotFound + } + return complete, nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + gate := &ownershipGateFilter{Filterer: FilterLogs(func([]*types.Log) bool { return false }).(Filterer), entered: make(chan struct{}), release: make(chan struct{})} + s := l.Subscribe(q, gate).(*subscriber) + defer s.Unsubscribe() + done := make(chan error, 1) + go func() { + _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{transaction: tx, blockHash: old.Hash(), blockNum: old.Number()}}) + done <- err + }() + <-gate.entered + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + hardeningRead(t, s) + if sameHash { + if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{canonical}, []*subscriber{s}, [][]Filterer{s.Filters()}); err != nil { + t.Fatal(err) + } + } else { + hardeningProcess(t, l, s, canonical) + } + hardeningRead(t, s) + cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Fatal("canonical receipt was not cached") + } + close(gate.release) + <-done + cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Error("late collection destroyed canonical cache") + } + unavailable.Store(true) + if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{old.Hash(), 0}); !errors.Is(err, ethereum.NotFound) { + t.Errorf("stale request error: %v", err) + } + cached, found, _ = l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != complete { + t.Error("stale expected block/generation deleted current cache") + } + r, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || r != complete { + t.Errorf("fresh cache query depended on unavailable origin: %v", err) + } + if calls.Load() != 1 { + t.Errorf("stale work refetched a current canonical result: calls=%d", calls.Load()) + } + }) + } +} diff --git a/ethreceipts/reorg_test.go b/ethreceipts/reorg_test.go new file mode 100644 index 00000000..f697aece --- /dev/null +++ b/ethreceipts/reorg_test.go @@ -0,0 +1,981 @@ +package ethreceipts + +import ( + "context" + "errors" + "io" + "log/slog" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestHardeningRollbackRemining(t *testing.T) { + tx, _, _ := hardeningTxn(t, 1) + old, newBlock := hardeningBlock(100, tx), hardeningBlock(102, tx) + var current atomic.Pointer[types.Receipt] + current.Store(hardeningReceipt(old, tx)) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { calls.Add(1); return current.Load(), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, old) + first := hardeningRead(t, s) + if first.Final || first.Reorged || first.BlockHash() != old.Hash() { + t.Fatalf("initial receipt: %+v", first) + } + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case rollback := <-s.TransactionReceipt(): + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.TransactionHash() != tx.Hash() { + t.Fatalf("rollback: %+v", rollback) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing rollback receipt") + } + if calls.Load() != 1 { + t.Errorf("rollback fetched orphan receipt: %d RPC calls", calls.Load()) + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + current.Store(hardeningReceipt(newBlock, tx)) + hardeningProcess(t, l, s, newBlock) + mined := hardeningRead(t, s) + if mined.BlockHash() != newBlock.Hash() || mined.Reorged || mined.Final { + t.Fatalf("re-mined receipt owns wrong block: %+v", mined) + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Reorged || final.BlockHash() != newBlock.Hash() || final.Filter != q { + t.Fatalf("canonical final receipt: %+v", final) + } +} + +func TestHardeningInflightRollbackRemining(t *testing.T) { + tx, _, _ := hardeningTxn(t, 11) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + select { + case <-release: + return hardeningReceipt(old, tx), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return hardeningReceipt(canonical, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + snapshot := s.Filters() + oldDone := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}) + oldDone <- err + }() + <-entered + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case rollback := <-s.TransactionReceipt(): + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != old.Hash() || rollback.Filter != q { + t.Errorf("wrong in-flight rollback: %+v", rollback) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing in-flight rollback") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + hardeningProcess(t, l, s, canonical) + select { + case mined := <-s.TransactionReceipt(): + if mined.BlockHash() != canonical.Hash() || mined.Reorged || mined.Final || mined.Filter != q { + t.Errorf("wrong canonical mined receipt: %+v", mined) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing canonical mined receipt") + } + close(release) + select { + case <-oldDone: + case <-time.After(time.Second): + t.Fatal("old fetch did not finish") + } + // A registration snapshot retained before rollback must also remain invalid. + if _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{old}, []*subscriber{s}, [][]Filterer{snapshot}); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + select { + case final := <-s.TransactionReceipt(): + if !final.Final || final.Reorged || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Errorf("wrong canonical final: %+v", final) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing canonical final") + } + cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || cached.BlockHash != canonical.Hash() { + t.Errorf("late orphan RPC poisoned receipt cache: receipt=%+v error=%v", cached, err) + } + if calls.Load() != 2 { + t.Errorf("unexpected fetch count: %d", calls.Load()) + } +} + +// This matcher deliberately needs the fetched receipt, which removed block data +// cannot supply. Rollback must use its previously delivered owner and receipt. +type hardeningStatusFilter struct{ Filterer } + +func (f *hardeningStatusFilter) Match(_ context.Context, r Receipt) (bool, error) { + return r.Status() == 1, nil +} + +func TestHardeningRollbackRetainsCustomOwner(t *testing.T) { + tx, _, _ := hardeningTxn(t, 14) + b := hardeningBlock(100, tx) + complete := hardeningReceipt(b, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return complete, nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := &hardeningStatusFilter{FilterLogs(func([]*types.Log) bool { return true }).Finalize(true).QueryOnChain(func(context.Context) (*types.Receipt, error) { return complete, nil }).(Filterer)} + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), s, s.Filters()); err != nil { + t.Fatal(err) + } + first := hardeningRead(t, s) + if first.Filter != q || first.Receipt() != complete { + t.Fatal("wrong custom filter owner or receipt") + } + removed := *b + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case r := <-s.TransactionReceipt(): + if !r.Reorged || r.Final || r.Filter != q || r.Receipt() != complete { + t.Errorf("rollback lost retained data/owner: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("custom complete-receipt matcher lost rollback") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) +} + +func TestHardeningRollbackInvalidatesPending(t *testing.T) { + tx, _, _ := hardeningTxn(t, 15) + old, canonical := hardeningBlock(100, tx), hardeningBlock(102, tx) + var failing atomic.Bool + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if failing.Load() { + return nil, errors.New("transient receipt error") + } + return hardeningReceipt(canonical, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningProcess(t, l, s, old) + s.retryMu.Lock() + pendingBefore := len(s.pendingReceipts) + s.retryMu.Unlock() + if pendingBefore != 1 { + t.Fatalf("expected one pending owner, got %d", pendingBefore) + } + removed := *old + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + s.retryMu.Lock() + pendingAfter := len(s.pendingReceipts) + s.retryMu.Unlock() + if pendingAfter != 0 { + t.Errorf("rollback retained %d orphan pending deliveries", pendingAfter) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Reorged || r.Filter != q || r.BlockHash() != old.Hash() { + t.Errorf("wrong pending rollback: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing pending rollback") + } + failing.Store(false) + hardeningProcess(t, l, s, canonical) + mined := hardeningRead(t, s) + if mined.BlockHash() != canonical.Hash() || mined.Reorged { + t.Fatal("retry ownership interfered with re-mining") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Fatal("wrong re-mined final owner") + } +} + +func TestHardeningSameHashReadoptionRejectsOldWork(t *testing.T) { + tx, _, _ := hardeningTxn(t, 19) + block := hardeningBlock(100, tx) + oldReceipt, canonicalReceipt := hardeningReceipt(block, tx), hardeningReceipt(block, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + select { + case <-release: + return oldReceipt, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return canonicalReceipt, nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + fs := s.Filters() + oldDone := make(chan error, 1) + go func() { + _, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}) + oldDone <- err + }() + <-entered + removed := *block + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.BlockHash() != block.Hash() || rollback.Filter != q { + t.Fatalf("wrong rollback: %+v", rollback) + } + // A cache snapshot taken before removal cannot authoritatively re-adopt it. + hardeningProcess(t, l, s, block) + hardeningNoReceipt(t, s) + if calls.Load() != 1 { + t.Errorf("stale snapshot cleared removal marker: %d fetches", calls.Load()) + } + // The live block-processing entry point receives the canonical Added event. + if _, err := l.processBlocks(context.Background(), ethmonitor.Blocks{block}, []*subscriber{s}, [][]Filterer{fs}); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { + t.Errorf("wrong readopted receipt: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("canonical same-hash re-adoption did not reopen delivery") + } + close(release) + <-oldDone + hardeningNoReceipt(t, s) + if calls.Load() != 2 { + t.Errorf("expected fresh readoption fetch, got %d", calls.Load()) + } + cached, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || cached != canonicalReceipt { + t.Errorf("old in-flight work replaced canonical cache: receipt=%p error=%v", cached, err) + } + // A callback with missing block data must acquire the fresh block generation + // when completing its receipt from cache, rather than inheriting generation zero. + partial := &types.Receipt{TxHash: tx.Hash()} + completion := l.Subscribe(FilterLogs(func([]*types.Log) bool { return true }).QueryOnChain(func(context.Context) (*types.Receipt, error) { return partial, nil })).(*subscriber) + defer completion.Unsubscribe() + if err := l.queryFilterOnChain(context.Background(), completion, completion.Filters()); err != nil { + t.Fatal(err) + } + select { + case r := <-completion.TransactionReceipt(): + if r.Receipt() != canonicalReceipt || r.BlockHash() != block.Hash() { + t.Error("incomplete callback lost readopted receipt") + } + case <-time.After(100 * time.Millisecond): + t.Error("missing-data callback lost canonical generation") + } + if err := s.finalizeReceipts(big.NewInt(103)); err != nil { + t.Fatal(err) + } + select { + case r := <-s.TransactionReceipt(): + if !r.Final || r.Reorged || r.Filter != q || r.Receipt() != canonicalReceipt { + t.Errorf("wrong readopted final: %+v", r) + } + case <-time.After(100 * time.Millisecond): + t.Error("missing readopted final") + } + hardeningNoReceipt(t, s) +} + +func TestOwnershipLiveExhaustedRollbackReleasesOwner(t *testing.T) { + tx, _, _ := hardeningTxn(t, 110) + b := hardeningBlock(100, tx) + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(b, tx), nil + }}, + canonical: make(map[uint64]*types.Block), + blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 100 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99)}); err != nil { + t.Fatal(err) + } + opts := hardeningOptions() + opts.NumBlocksToFinality = 10 + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true).MaxWait(1) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + hardeningStart(t, l) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor Run did not stop") + } + }) + wait := func(what string, condition func() bool) { + t.Helper() + deadline := time.Now().Add(12 * time.Second) + for !condition() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } + } + alternate := func(num, fork int64, parent common.Hash) *types.Block { + header := hardeningBlock(num).Header() + header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+fork)), parent + return types.NewBlockWithHeader(header) + } + p.advance(b.Block) + mined := hardeningRead(t, s) + if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Final || mined.Reorged { + t.Fatal("wrong initial mined owner") + } + wait("first live match", func() bool { return q.(Filterer).LastMatchBlockNum() == 100 }) + old101 := hardeningBlock(101) + p.advance(old101.Block) + wait("block 101", func() bool { return m.LatestBlockNum().Int64() == 101 }) + alt101 := alternate(101, 10000, b.Hash()) + alt102 := alternate(102, 10000, alt101.Hash()) + p.advance(alt101, alt102) + wait("live reorg counter reset", func() bool { + return q.(Filterer).StartBlockNum() == 102 && q.(Filterer).LastMatchBlockNum() == 0 + }) + alt103 := alternate(103, 10000, alt102.Hash()) + p.advance(alt103) + select { + case <-q.(Filterer).Exhausted(): + case <-time.After(time.Second): + t.Fatal("live MaxWait did not exhaust owner") + } + s.deliveryMu.Lock() + queued := len(s.finalizer.queue) == 1 && len(s.claims) == 1 + s.deliveryMu.Unlock() + if !queued || len(s.Filters()) != 0 { + t.Fatal("exhaustion lost queued finality ownership") + } + current := hardeningBlock(99).Block + for num := int64(100); num <= 104; num++ { + next := alternate(num, 20000, current.Hash()) + p.advance(next) + current = next + } + wait("rollback invalidation", func() bool { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + return m.LatestBlock().Hash() == current.Hash() && len(s.finalizer.queue) == 0 + }) + select { + case rollback := <-s.TransactionReceipt(): + if rollback.TransactionHash() != tx.Hash() || rollback.BlockHash() != b.Hash() || rollback.BlockNumber().Int64() != 100 || rollback.Filter != q || !rollback.Reorged || rollback.Final || rollback.Status() != 1 { + t.Error("wrong exhausted-owner rollback identity/state") + } + case <-time.After(100 * time.Millisecond): + t.Error("exhausted queued owner missed its delivered rollback") + } + checkRetired := func() { + t.Helper() + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.inFlight) != 0 || len(s.pendingReceipts) != 0 || len(s.finalizer.queue) != 0 || len(s.finalizer.txns) != 0 { + t.Errorf("inactive rollback owner retained state: claims=%d deliveries=%d", len(s.claims), len(s.deliveries)) + } + } + checkRetired() + for num := int64(105); num <= 112; num++ { + next := alternate(num, 20000, current.Hash()) + p.advance(next) + current = next + } + wait("finality advancement", func() bool { return m.LatestBlock().Hash() == current.Hash() }) + s.RemoveFilter(q.(Filterer)) + checkRetired() + hardeningNoReceipt(t, s) +} + +func TestOwnershipExhaustedRollbackKeepsOtherCandidates(t *testing.T) { + tx1, _, _ := hardeningTxn(t, 111) + tx2, _, _ := hardeningTxn(t, 112) + b1, b2 := hardeningBlock(100, tx1), hardeningBlock(101, tx2) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + other := FilterTxnHash(tx2.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q, other).(*subscriber) + defer s.Unsubscribe() + for _, b := range []*ethmonitor.Block{b1, b2} { + if _, err := s.matchFiltersAndPublish(context.Background(), s.Filters(), []Receipt{{receipt: hardeningReceipt(b, b.Transactions()[0])}}); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 3; i++ { + hardeningRead(t, s) + } + s.exhaustFilter(q.(Filterer)) + q.(*filter).closeExhausted() + removed := *b1 + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + select { + case r := <-s.TransactionReceipt(): + if r.TransactionHash() != tx1.Hash() || r.BlockHash() != b1.Hash() || r.Filter != q || !r.Reorged || r.Final { + t.Error("wrong exhausted retained rollback") + } + case <-time.After(100 * time.Millisecond): + t.Error("inactive owner missed rollback while another candidate remained") + } + if len(s.finalizer.queue) != 2 || len(s.Filters()) != 1 { + t.Fatal("rollback removed unrelated ownership") + } + if err := s.finalizeReceipts(big.NewInt(104)); err != nil { + t.Fatal(err) + } + seen := make(map[Filterer]bool) + for i := 0; i < 2; i++ { + r := hardeningRead(t, s) + if r.TransactionHash() != tx2.Hash() || r.BlockHash() != b2.Hash() || !r.Final || r.Reorged || seen[r.Filter] { + t.Error("rollback lost or duplicated another owned final") + } + seen[r.Filter] = true + } + if !seen[q.(Filterer)] || !seen[other.(Filterer)] || len(s.claims) != 0 || len(s.deliveries) != 0 || len(s.Filters()) != 0 { + t.Error("finished owners were not retired") + } + hardeningNoReceipt(t, s) +} + +func TestReceiptsFixRollbackPreservesMinedSelection(t *testing.T) { + tx, _, _ := hardeningTxn(t, 805) + other, _, _ := hardeningTxn(t, 806) + old, replacement, canonical := hardeningBlock(100, tx), hardeningBlock(101, other), hardeningBlock(102, tx) + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions()) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(old, tx)) + hardeningRead(t, s) + s.rollbackBlock(l.invalidateBlock(context.Background(), old)) + rollback := hardeningRead(t, s) + if s.claims[owner] != tx.Hash() || !rollback.Reorged || rollback.Filter != q || rollback.BlockHash() != old.Hash() { + t.Fatal("rollback released an already-mined selection") + } + process(hardeningReceipt(replacement, other)) + hardeningNoReceipt(t, s) + process(hardeningReceipt(canonical, tx)) + mined := hardeningRead(t, s) + if mined.Reorged || mined.Final || mined.TransactionHash() != tx.Hash() || mined.BlockHash() != canonical.Hash() || mined.Filter != q { + t.Fatal("retained selection did not re-mine correctly") + } + if err := s.finalizeReceipts(big.NewInt(105)); err != nil { + t.Fatal(err) + } + final := hardeningRead(t, s) + if !final.Final || final.Reorged || final.TransactionHash() != tx.Hash() || final.BlockHash() != canonical.Hash() || final.Filter != q { + t.Fatal("retained mined selection finalized an orphan") + } + hardeningNoReceipt(t, s) +} + +func TestReceiptsFixOrphanedSelectionReleasedAtFinality(t *testing.T) { + tx, _, _ := hardeningTxn(t, 808) + other, _, _ := hardeningTxn(t, 809) + var chain []*ethmonitor.Block + for num := int64(100); num <= 110; num++ { + chain = append(chain, hardeningBlock(num)) + } + l := hardeningListener(t, &hardeningProvider{}, hardeningOptions(), chain...) + header := hardeningBlock(109).Header() + header.BlockHash = common.HexToHash("0xdead") + orphan := ðmonitor.Block{Block: types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: []*types.Transaction{tx}}), Event: ethmonitor.Added, OK: true} + canonical := hardeningBlock(110, other) + q := FilterLogs(func([]*types.Log) bool { return true }).LimitOne(true).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + owner := s.owner(q.(Filterer)) + process := func(b *types.Receipt) { + t.Helper() + if _, err := s.matchFiltersAndPublish(context.Background(), s.filterers(), []Receipt{{receipt: b}}); err != nil { + t.Fatal(err) + } + } + process(hardeningReceipt(orphan, tx)) + if mined := hardeningRead(t, s); mined.Final || mined.BlockHash() != orphan.Hash() { + t.Fatal("orphan was delivered as final") + } + s.rollbackBlock(l.invalidateBlock(context.Background(), orphan)) + hardeningRead(t, s) + if err := s.finalizeReceipts(big.NewInt(110)); err != nil { + t.Fatal(err) + } + if s.claims[owner] != tx.Hash() { + t.Fatal("selection released while its txn could still be re-mined") + } + // The bootstrapped monitor cannot advance, so shrink the finality depth to + // put the orphan's height past it. + l.mu.Lock() + l.options.NumBlocksToFinality = 1 + l.mu.Unlock() + if err := s.finalizeReceipts(big.NewInt(110)); err != nil { + t.Fatal(err) + } + hardeningNoReceipt(t, s) + if _, claimed := s.claims[owner]; claimed || len(s.deliveries) != 0 || len(s.Filters()) != 1 { + t.Fatal("finalized orphan kept its selection") + } + process(hardeningReceipt(canonical, other)) + mined := hardeningRead(t, s) + if mined.Reorged || mined.Final || mined.TransactionHash() != other.Hash() || mined.BlockHash() != canonical.Hash() || mined.Filter != q { + t.Fatal("released owner did not select the canonical txn") + } + if s.claims[owner] != other.Hash() { + t.Fatal("canonical txn did not take over the selection") + } +} + +func TestReceiptsReadoptedPendingSurvivesChainAdvance(t *testing.T) { + ctx := context.Background() + tx, _, _ := hardeningTxn(t, 9901) + b := hardeningBlock(100, tx) + var failing atomic.Bool + var calls atomic.Int32 + failing.Store(true) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if failing.Load() { + return nil, errors.New("temporary receipt provider failure") + } + return hardeningReceipt(b, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), b) + l.invalidateBlock(ctx, b) + l.acceptBlock(b) + if !l.isCurrentBlock(b.Hash(), 1) { + t.Fatal("readopted block did not become current") + } + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + filters := [][]Filterer{s.filterers()} + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{b}, []*subscriber{s}, filters); err != nil { + t.Fatal(err) + } + if len(s.pendingReceipts) != 1 { + t.Fatal("missing pending canonical receipt") + } + // Hold the retry while canonical events advance beyond retention plus + // finality. Receipt backoff is time based and can outlive that block horizon. + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Now().Add(time.Hour) + } + for num := int64(101); num <= 203; num++ { + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{hardeningBlock(num)}, []*subscriber{s}, filters); err != nil { + t.Fatal(err) + } + } + failing.Store(false) + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Time{} + } + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{hardeningBlock(204)}, []*subscriber{s}, filters); err != nil { + t.Fatal(err) + } + r := hardeningRead(t, s) + if r.Reorged || r.BlockHash() != b.Hash() || r.Filter != q || r.Receipt() == nil { + t.Fatal("incorrect recovered canonical receipt") + } + if len(s.pendingReceipts) != 0 || calls.Load() != 2 { + t.Fatalf("canonical retry did not recover: pending=%d, origin calls=%d", len(s.pendingReceipts), calls.Load()) + } +} + +func TestReceiptsRejectLateOrphanAfterChainAdvance(t *testing.T) { + tx, _, _ := hardeningTxn(t, 9902) + header := hardeningBlock(100).Header() + header.BlockHash = common.BigToHash(big.NewInt(1_000_100)) + b := ðmonitor.Block{Block: types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: []*types.Transaction{tx}}), Event: ethmonitor.Added, OK: true} + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + close(entered) + select { + case <-release: + return hardeningReceipt(b, tx), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + }} + l := hardeningListener(t, p, hardeningOptions(), b) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + done := make(chan struct{}) + var receipt *types.Receipt + var fetchErr error + t.Cleanup(func() { + cancel() + unblock() + select { + case <-done: + case <-time.After(time.Second): + t.Error("receipt fetch did not stop") + } + }) + go func() { + defer close(done) + receipt, fetchErr = l.fetchTransactionReceipt(ctx, tx.Hash(), false) + }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("receipt RPC did not start") + } + removed := *b + removed.Event = ethmonitor.Removed + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{&removed}, nil, nil); err != nil { + t.Fatal(err) + } + // Replace the orphan and advance past the former pruning horizon while + // the pre-removal RPC is still in flight. + for num := int64(100); num <= 204; num++ { + if _, err := l.processBlocks(ctx, ethmonitor.Blocks{hardeningBlock(num)}, nil, nil); err != nil { + t.Fatal(err) + } + } + unblock() + select { + case <-done: + if receipt != nil || !errors.Is(fetchErr, ethereum.NotFound) { + t.Fatalf("late orphan RPC was accepted: receipt=%v, error=%v", receipt != nil, fetchErr) + } + case <-ctx.Done(): + t.Fatal("receipt RPC did not return") + } + if _, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); found { + t.Error("late orphan receipt entered cache") + } +} + +func TestReceiptsFixBatchMatchesAnyAddedBlock(t *testing.T) { + tx, _, _ := hardeningTxn(t, 810) + mined, empty := hardeningBlock(100, tx), hardeningBlock(101) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { return hardeningReceipt(mined, tx), nil }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + matched, err := l.processCachedBlocks(context.Background(), ethmonitor.Blocks{mined, empty}, []*subscriber{s}, [][]Filterer{s.filterers()}) + if err != nil { + t.Fatal(err) + } + if !matched[0][0] { + t.Error("match in an earlier block of the batch was lost") + } + hardeningRead(t, s) + removed := *mined + removed.Event = ethmonitor.Removed + matched, err = l.processBlocks(context.Background(), ethmonitor.Blocks{&removed}, []*subscriber{s}, [][]Filterer{s.filterers()}) + if err != nil { + t.Fatal(err) + } + if rollback := hardeningRead(t, s); !rollback.Reorged { + t.Fatal("missing rollback receipt") + } + if matched[0][0] { + t.Error("rollback of an earlier match counted as a new match") + } +} + +func TestReceiptsFixReadoptionAfterRetention(t *testing.T) { + for _, scenario := range []string{"canonical_retention", "obsolete_incarnation", "later_removal", "alternate_retention"} { + t.Run(scenario, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 807) + b := hardeningBlock(100, tx) + old, canonical := hardeningReceipt(b, tx), hardeningReceipt(b, tx) + entered, release := make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + var releaseOnce sync.Once + p := &ownershipLiveProvider{ + hardeningProvider: &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + if calls.Add(1) == 1 { + close(entered) + // Deliberately finish an old RPC after its request timeout, + // to verify invalidation of late provider completions. + <-release + return old, nil + } + return canonical, nil + }}, + canonical: make(map[uint64]*types.Block), blocks: make(map[common.Hash]*types.Block), + } + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mo := ethmonitor.DefaultOptions + mo.Logger, mo.WithLogs, mo.Bootstrap = log, true, true + mo.StreamingDisabled, mo.PrefetchConcurrency, mo.PollingInterval = true, 0, 5*time.Millisecond + mo.BlockRetentionLimit = 50 + m, err := ethmonitor.NewMonitor(p, mo) + if err != nil { + t.Fatal(err) + } + if err = m.Chain().BootstrapFromBlocks(ethmonitor.Blocks{hardeningBlock(99), b}); err != nil { + t.Fatal(err) + } + initialSnapshot := *m.LatestBlock() + opts := hardeningOptions() + opts.NumBlocksToFinality = 200 + l, err := NewReceiptsListener(log, p, m, opts) + if err != nil { + t.Fatal(err) + } + q := FilterLogs(func([]*types.Log) bool { return true }).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + stream := m.Subscribe("delayed receipt events") + defer stream.Unsubscribe() + oldDone := make(chan error, 1) + go func() { + _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), true, blockRef{b.Hash(), 0}) + oldDone <- err + }() + <-entered + releaseOld := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(releaseOld) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- m.Run(ctx) }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Errorf("monitor Run: %v", err) + } + case <-time.After(3 * time.Second): + t.Error("monitor did not stop") + } + }) + consume := func() ethmonitor.Blocks { + t.Helper() + select { + case batch := <-stream.Blocks(): + return batch + case <-time.After(10 * time.Second): + t.Fatal("missing canonical monitor event") + return nil + } + } + process := func(batch ethmonitor.Blocks, cached bool) { + t.Helper() + var err error + if cached { + _, err = l.processCachedBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + } else { + _, err = l.processBlocks(context.Background(), batch, []*subscriber{s}, [][]Filterer{s.filterers()}) + } + if err != nil { + t.Fatal(err) + } + } + fork := func(from, through, offset int64, parent common.Hash) { + for num := from; num <= through; num++ { + header := hardeningBlock(num).Header() + header.BlockHash, header.ParentHash = common.BigToHash(big.NewInt(num+offset)), parent + next := types.NewBlockWithHeader(header) + p.advance(next) + parent = next.Hash() + } + } + waitHeight := func(height int64) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for m.LatestBlockNum().Int64() != height { + if time.Now().After(deadline) { + t.Fatal("monitor did not advance canonical height") + } + time.Sleep(time.Millisecond) + } + } + // The first real reorg removes H, invalidating the held generation-0 RPC. + fork(100, 101, 10000, hardeningBlock(99).Hash()) + process(consume(), false) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.Final || rollback.BlockHash() != b.Hash() || rollback.TransactionHash() != tx.Hash() || rollback.Filter != q { + t.Fatal("wrong original removal identity") + } + process(ethmonitor.Blocks{&initialSnapshot}, true) + hardeningNoReceipt(t, s) + // Rebuild the same hash H in a fresh, owned monitor incarnation. + p.advance(b.Block, hardeningBlock(101).Block, hardeningBlock(102).Block) + readoption := consume() + var added *ethmonitor.Block + for _, event := range readoption { + if event.Event == ethmonitor.Added && event.Hash() == b.Hash() { + added = event + } + } + if added == nil { + t.Fatal("monitor omitted readopted incarnation") + } + incarnation, valid := added.CanonicalState() + if incarnation == 0 || !valid { + t.Fatal("Added input was not the owned canonical monitor event") + } + process(ethmonitor.Blocks{added}, true) + hardeningNoReceipt(t, s) + negative := scenario == "later_removal" || scenario == "alternate_retention" + var removed *ethmonitor.Block + if negative { + fork(100, 103, 20000, hardeningBlock(99).Hash()) + for _, event := range consume() { + if event.Event == ethmonitor.Removed && event.Hash() == b.Hash() { + removed = event + } + } + if removed == nil { + t.Fatal("monitor omitted later removal") + } + } + if scenario == "canonical_retention" { + fork(103, 155, 0, hardeningBlock(102).Hash()) + waitHeight(155) + } else if scenario == "alternate_retention" { + fork(104, 155, 20000, common.BigToHash(big.NewInt(20103))) + waitHeight(155) + } + if scenario == "canonical_retention" || scenario == "alternate_retention" { + if m.GetBlock(b.Hash()) != nil || m.OldestBlockNum().Int64() != 106 { + t.Fatal("H was not evicted more than one block behind retention") + } + } + if scenario == "obsolete_incarnation" { + // A removed tracked snapshot cannot fall back to a newer retained + // incarnation of the same hash, even during authoritative processing. + if m.GetBlock(b.Hash()) == nil { + t.Fatal("newer same-hash incarnation is missing") + } + process(ethmonitor.Blocks{&initialSnapshot}, false) + hardeningNoReceipt(t, s) + if !l.blockStates[b.Hash()].removed { + t.Error("obsolete tracked incarnation used same-hash fallback") + } + } + process(readoption, false) + if negative { + hardeningNoReceipt(t, s) + if !l.blockStates[b.Hash()].removed { + t.Error("later-removed Added reopened canonical marker") + } + process(ethmonitor.Blocks{added, removed}, false) + for _, r := range receiptsFixCollect(t, s) { + if !r.Reorged || r.Final { + t.Error("same-batch removal published stale mined/final receipt") + } + } + if _, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false); !errors.Is(err, ethereum.NotFound) { + t.Errorf("later removal reopened hash lookup: %v", err) + } + } else { + select { + case mined := <-s.TransactionReceipt(): + if mined.TransactionHash() != tx.Hash() || mined.BlockHash() != b.Hash() || mined.Filter != q || mined.Reorged || mined.Final || mined.generation != 1 { + t.Error("readopted incarnation lost exact mined ownership") + } + case <-time.After(100 * time.Millisecond): + t.Error("canonical readoption after retention was rejected") + } + result, err := l.fetchTransactionReceipt(context.Background(), tx.Hash(), false) + if err != nil || result != canonical { + t.Errorf("canonical readoption hash lookup failed: %v", err) + } + } + releaseOld() + if err := <-oldDone; !errors.Is(err, ethereum.NotFound) { + t.Errorf("old generation RPC escaped invalidation: %v", err) + } + if !negative { + cached, found, _ := l.pastReceipts.Get(context.Background(), tx.Hash().Hex()) + if !found || cached != canonical { + t.Error("old RPC replaced fresh canonical cache") + } + } + hardeningNoReceipt(t, s) + }) + } +} diff --git a/ethreceipts/stale_receipt_test.go b/ethreceipts/stale_receipt_test.go new file mode 100644 index 00000000..9fea2901 --- /dev/null +++ b/ethreceipts/stale_receipt_test.go @@ -0,0 +1,170 @@ +package ethreceipts + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/0xsequence/ethkit/ethmonitor" + "github.com/0xsequence/ethkit/go-ethereum" + "github.com/0xsequence/ethkit/go-ethereum/common" + "github.com/0xsequence/ethkit/go-ethereum/core/types" +) + +func TestReceiptsStaleRPCRecoversCurrentCandidate(t *testing.T) { + for _, pendingFirst := range []bool{false, true} { + name := "initial_response" + if pendingFirst { + name = "pending_retry" + } + t.Run(name, func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6000) + a, b := hardeningBlock(100, tx), hardeningBlock(102, tx) + var current atomic.Pointer[types.Receipt] + current.Store(hardeningReceipt(a, tx)) + var providerError atomic.Bool + var calls atomic.Int32 + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + calls.Add(1) + if providerError.Load() { + return nil, errors.New("temporary provider failure") + } + return current.Load(), nil + }} + l := hardeningListener(t, p, hardeningOptions()) + q := FilterTxnHash(tx.Hash()).SearchCache(false).QueryOnChainTxnHash(false).Finalize(true) + s := l.Subscribe(q).(*subscriber) + defer s.Unsubscribe() + ctx := context.Background() + hardeningProcess(t, l, s, a) + mined := hardeningRead(t, s) + if mined.BlockHash() != a.Hash() || mined.Reorged || mined.Final { + t.Fatalf("wrong initial mined receipt: %+v", mined) + } + removed := *a + removed.Event = ethmonitor.Removed + hardeningProcess(t, l, s, &removed) + rollback := hardeningRead(t, s) + if !rollback.Reorged || rollback.BlockHash() != a.Hash() { + t.Fatalf("wrong rollback notification: %+v", rollback) + } + providerError.Store(pendingFirst) + hardeningProcess(t, l, s, b) + providerError.Store(false) + pendingCount := func() int { + s.retryMu.Lock() + defer s.retryMu.Unlock() + return len(s.pendingReceipts) + } + makeDue := func() { + s.retryMu.Lock() + defer s.retryMu.Unlock() + for _, pending := range s.pendingReceipts { + pending.nextRetryAt = time.Now().Add(-time.Second) + } + } + if pendingFirst { + if pendingCount() != 1 { + t.Fatal("provider failure did not queue B") + } + makeDue() + hardeningProcess(t, l, s, hardeningBlock(103)) + } + if pendingCount() != 1 { + t.Errorf("stale A response discarded current B candidate: pending=%d", pendingCount()) + } + if _, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); found { + t.Fatal("stale A receipt was cached") + } + hardeningNoReceipt(t, s) + + // The provider recovers; only an unrelated later block triggers retry. + canonical := hardeningReceipt(b, tx) + current.Store(canonical) + makeDue() + callsBefore := calls.Load() + hardeningProcess(t, l, s, hardeningBlock(104)) + if calls.Load() <= callsBefore { + t.Fatal("provider recovered but B was never retried") + } + mined = hardeningRead(t, s) + if mined.BlockHash() != b.Hash() || mined.TransactionHash() != tx.Hash() || mined.Reorged || mined.Final || mined.Filter != q { + t.Fatalf("wrong recovered mined receipt: %+v", mined) + } + hardeningProcess(t, l, s, hardeningBlock(105)) + final := hardeningRead(t, s) + if final.BlockHash() != b.Hash() || final.TransactionHash() != tx.Hash() || final.Reorged || !final.Final || final.Filter != q { + t.Fatalf("wrong recovered final receipt: %+v", final) + } + if pendingCount() != 0 { + t.Error("delivered B remained pending") + } + if cached, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); !found || cached != canonical { + t.Error("canonical B receipt was not cached") + } + hardeningNoReceipt(t, s) + }) + } +} + +func TestReceiptsStaleRPCKeepsObsoleteWorkTerminal(t *testing.T) { + t.Run("expected_generation_removed_during_fetch", func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6001) + b := hardeningBlock(100, tx) + entered, release := make(chan struct{}), make(chan struct{}) + p := &hardeningProvider{receipt: func(ctx context.Context, _ common.Hash) (*types.Receipt, error) { + close(entered) + select { + case <-release: + return hardeningReceipt(b, tx), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + }} + l := hardeningListener(t, p, hardeningOptions(), b) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := l.fetchTransactionReceipt(ctx, tx.Hash(), true, blockRef{b.Hash(), 0}) + done <- err + }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("RPC did not start") + } + l.invalidateBlock(ctx, b) + l.acceptBlock(b) + if !l.isCurrentBlock(b.Hash(), 1) { + t.Fatal("same-hash re-adoption did not advance the generation") + } + fresh := hardeningReceipt(b, tx) + l.pastReceipts.Set(ctx, tx.Hash().Hex(), fresh) + close(release) + if err := <-done; !errors.Is(err, ethereum.NotFound) { + t.Fatalf("obsolete expected generation became retryable: %v", err) + } + if cached, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); !found || cached != fresh { + t.Error("obsolete completion replaced the current cache entry") + } + }) + t.Run("hash_query_without_current_candidate", func(t *testing.T) { + tx, _, _ := hardeningTxn(t, 6002) + a := hardeningBlock(100, tx) + p := &hardeningProvider{receipt: func(context.Context, common.Hash) (*types.Receipt, error) { + return hardeningReceipt(a, tx), nil + }} + l := hardeningListener(t, p, hardeningOptions(), a) + ctx := context.Background() + l.invalidateBlock(ctx, a) + if _, err := l.fetchTransactionReceipt(ctx, tx.Hash(), true); !errors.Is(err, ethereum.NotFound) { + t.Fatalf("unscoped orphan query became retryable: %v", err) + } + if _, found, _ := l.pastReceipts.Get(ctx, tx.Hash().Hex()); found { + t.Error("unscoped orphan query cached stale data") + } + }) +} diff --git a/ethreceipts/subscription.go b/ethreceipts/subscription.go index 0d59f0a5..b6efd413 100644 --- a/ethreceipts/subscription.go +++ b/ethreceipts/subscription.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/big" + "reflect" "sync" "time" @@ -33,10 +34,17 @@ var ( type Subscription interface { TransactionReceipt() <-chan Receipt Done() <-chan struct{} + // Unsubscribe closes the subscription. Repeated and concurrent calls are safe. Unsubscribe() Filters() []Filterer AddFilter(filters ...FilterQuery) + // RemoveFilter removes the first matching active registration, or its queued + // finality owner if inactive. Comparable filters retain Go equality. Otherwise + // callback-bearing values of the same type use a stable nonnil Exhausted signal + // as base identity. Values sharing a base are aliases; distinct callbacks need + // distinct bases or pointer identities for independent removal. Values without + // a stable signal should be registered as pointers for individual removal. RemoveFilter(filter Filterer) ClearFilters() } @@ -48,17 +56,28 @@ type subscriber struct { ch channel.Channel[Receipt] done chan struct{} unsubscribe func() - filters []Filterer + filters []*filterOwner // Protected by mu. finalizer *finalizer mu sync.Mutex - pendingReceipts map[common.Hash]*pendingReceipt + // retryMu protects pendingReceipts and the mutable fields of each pendingReceipt. + pendingReceipts map[receiptKey]*pendingReceipt retryMu sync.Mutex + + // deliveryMu serializes publication, rollback, finality, owner retirement + // and channel sends, and protects deliveries, inFlight and claims. Lock order + // is deliveryMu, retryMu, then the listener's receiptMu. RPC calls run + // outside these locks. + deliveryMu sync.Mutex + deliveries map[receiptKey]Receipt + inFlight map[receiptKey]struct{} + claims map[*filterOwner]common.Hash } type pendingReceipt struct { receipt Receipt - filterer Filterer + completed *Receipt // Learned candidate; receipt retains the original queue key. + owner *filterOwner attempts int nextRetryAt time.Time } @@ -80,225 +99,524 @@ func (s *subscriber) Unsubscribe() { s.unsubscribe() } +// Every registration owns a comparable token; the public filter may contain +// slices, maps or functions and may be reused in a later registration. +type filterOwner struct{ Filterer } + +func sameFilter(a, b Filterer) bool { + if reflect.ValueOf(a).Comparable() && reflect.ValueOf(b).Comparable() { + return a == b + } + if reflect.DeepEqual(a, b) { + return true + } + if reflect.TypeOf(a) != reflect.TypeOf(b) { + return false + } + identity := a.Exhausted() + return identity != nil && identity == b.Exhausted() +} + func (s *subscriber) Filters() []Filterer { s.mu.Lock() defer s.mu.Unlock() filters := make([]Filterer, len(s.filters)) - copy(filters, s.filters) + for i, owner := range s.filters { + filters[i] = owner.Filterer + } return filters } -func (s *subscriber) AddFilter(filterQueries ...FilterQuery) { - if len(filterQueries) == 0 { - return +func (s *subscriber) filterers() []Filterer { + s.mu.Lock() + defer s.mu.Unlock() + filters := make([]Filterer, len(s.filters)) + for i, owner := range s.filters { + filters[i] = owner } + return filters +} - filters := make([]Filterer, len(filterQueries)) - for i, query := range filterQueries { +func (s *subscriber) AddFilter(queries ...FilterQuery) { + if len(queries) == 0 { + return + } + owners := make([]*filterOwner, len(queries)) + filters := make([]Filterer, len(queries)) + for i, query := range queries { filterer, ok := query.(Filterer) if !ok { panic("ethreceipts: unexpected") } - filters[i] = filterer + owners[i] = &filterOwner{filterer} + filters[i] = owners[i] } - s.mu.Lock() - if len(s.filters)+len(filters) > maxFiltersPerListener { - // too many filters, ignore the extra filter. not ideal, but better than - // deadlocking - s.listener.log.Warn(fmt.Sprintf("ethreceipts: subscriber has too many filters (%d), ignoring extra", len(s.filters)+len(filters))) - // TODO: maybe return an error or force-unsubscribe instead? + if len(s.filters)+len(owners) > maxFiltersPerListener { + s.listener.log.Warn(fmt.Sprintf("ethreceipts: subscriber has too many filters (%d), ignoring extra", len(s.filters)+len(owners))) s.mu.Unlock() return } - s.filters = append(s.filters, filters...) + s.filters = append(s.filters, owners...) s.mu.Unlock() - - // TODO: maybe add non-blocking push structure like in relayer queue select { case s.listener.registerFiltersCh <- registerFilters{subscriber: s, filters: filters}: - // ok default: s.listener.log.Warn("ethreceipts: listener registerFiltersCh full, dropping filter register") } } -func (s *subscriber) RemoveFilter(filter Filterer) { +func (s *subscriber) owner(filter Filterer) *filterOwner { + if owner, ok := filter.(*filterOwner); ok { + return owner + } s.mu.Lock() defer s.mu.Unlock() + for _, owner := range s.filters { + if sameFilter(owner.Filterer, filter) { + return owner + } + } + return nil +} + +func (s *subscriber) RemoveFilter(filter Filterer) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + owner := s.owner(filter) + if owner == nil { + owner = s.finalizer.findOwner(filter) + } + if owner != nil { + s.retireOwner(owner, true) + } +} +func (s *subscriber) removeActive(owner *filterOwner) { + s.mu.Lock() + defer s.mu.Unlock() for i, f := range s.filters { - if f == filter { - s.filters = append(s.filters[:i], s.filters[i+1:]...) + if f == owner { + copy(s.filters[i:], s.filters[i+1:]) + s.filters[len(s.filters)-1] = nil + s.filters = s.filters[:len(s.filters)-1] return } } } +// Caller holds deliveryMu. Exhaustion retains queued finals, while explicit +// cancellation and automatic completion release the complete owner lifetime. +func (s *subscriber) retireOwner(owner *filterOwner, cancelFinality bool) { + s.removeActive(owner) + if cancelFinality { + s.finalizer.invalidateOwner(owner) + } + if cancelFinality || !s.finalizer.hasOwner(owner) { + delete(s.claims, owner) + for key := range s.deliveries { + if key.owner == owner { + delete(s.deliveries, key) + } + } + } + for key := range s.inFlight { + if key.owner == owner { + delete(s.inFlight, key) + } + } + s.retryMu.Lock() + for key := range s.pendingReceipts { + if key.owner == owner { + delete(s.pendingReceipts, key) + } + } + s.retryMu.Unlock() +} + +func (s *subscriber) exhaustFilter(filter Filterer) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if owner := s.owner(filter); owner != nil { + s.retireOwner(owner, false) + } +} + func (s *subscriber) ClearFilters() { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + s.mu.Lock() + s.filters = nil + s.mu.Unlock() + s.finalizer.clear() + clear(s.claims) + clear(s.inFlight) + clear(s.deliveries) + s.retryMu.Lock() + clear(s.pendingReceipts) + s.retryMu.Unlock() +} + +// Reserve matches before fetching so concurrent registration and live processing +// cannot select different transactions for one LimitOne filter. +func (s *subscriber) reserve(receipt Receipt, owner *filterOwner) bool { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if !s.hasFilter(owner) || !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + return false + } + if s.claims == nil { + s.claims = make(map[*filterOwner]common.Hash) + } + if owner.Options().LimitOne { + if txn, claimed := s.claims[owner]; claimed && txn != receipt.TransactionHash() { + return false + } + s.claims[owner] = receipt.TransactionHash() + } + key := receiptOwner(receipt, owner) + if _, exists := s.deliveries[key]; exists { + return false + } + if s.inFlight == nil { + s.inFlight = make(map[receiptKey]struct{}) + } + if _, exists := s.inFlight[key]; exists { + return false + } + s.retryMu.Lock() + _, pending := s.pendingReceipts[key] + s.retryMu.Unlock() + if pending { + return false + } + s.inFlight[key] = struct{}{} + return true +} + +func (s *subscriber) releaseReservation(receipt Receipt, owner *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + delete(s.inFlight, receiptOwner(receipt, owner)) + s.releaseClaim(owner) +} + +// Caller holds deliveryMu. Keep selection only while real owner work remains. +func (s *subscriber) releaseClaim(owner *filterOwner) { + if s.finalizer.hasOwner(owner) { + return + } + for key := range s.inFlight { + if key.owner == owner { + return + } + } + for key := range s.deliveries { + if key.owner == owner { + return + } + } + s.retryMu.Lock() + defer s.retryMu.Unlock() + for key := range s.pendingReceipts { + if key.owner == owner { + return + } + } + delete(s.claims, owner) +} + +func (s *subscriber) hasFilter(owner *filterOwner) bool { s.mu.Lock() defer s.mu.Unlock() - s.filters = s.filters[:0] + for _, f := range s.filters { + if f == owner { + return true + } + } + return false } -// matchFiltersAndPublish matches the given receipts against the provided filterers, -// fetches any missing receipt data as needed, and notifies the subscriber of matches. -func (s *subscriber) matchFiltersAndPublish(ctx context.Context, filterers []Filterer, receipts []Receipt) ([]bool, error) { - oks := make([]bool, len(filterers)) +// RPC waits happen outside deliveryMu. Rollback, delivery and finalization share +// this lock so an invalidated in-flight receipt cannot be published afterwards. +func (s *subscriber) publish(ctx context.Context, receipt Receipt, owner *filterOwner) bool { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + return s.publishLocked(ctx, receipt, owner) +} + +// Caller holds deliveryMu so retry publication and pending ownership commit +// together with Remove/Clear and rollback invalidation. +func (s *subscriber) publishLocked(ctx context.Context, receipt Receipt, owner *filterOwner) bool { + // deliveryMu, not receiptMu, orders this check against rollback: + // invalidateBlock runs before rollbackBlock, which waits for deliveryMu and + // reports any delivery recorded here. + if ctx.Err() != nil || !s.hasFilter(owner) { + return false + } + // Block validation can wait for receiptMu; recheck cancellation afterward. + if !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) || ctx.Err() != nil { + return false + } + key := receiptOwner(receipt, owner) + if _, delivered := s.deliveries[key]; delivered { + return true + } + if owner.Options().LimitOne { + if txn, claimed := s.claims[owner]; claimed && txn != receipt.TransactionHash() { + return false + } + } + receipt.Filter = owner.Filterer + receipt.owner = owner + receipt.Final = s.listener.isBlockFinal(receipt.BlockNumber()) + if !receipt.Final && owner.Options().Finalize { + s.finalizer.enqueue(owner, receipt, receipt.BlockNumber()) + } + if s.deliveries == nil { + s.deliveries = make(map[receiptKey]Receipt) + } + s.deliveries[key] = receipt + s.ch.Send(receipt) + if owner.Options().LimitOne && (!owner.Options().Finalize || receipt.Final) { + s.retireOwner(owner, true) + } + return true +} + +func (s *subscriber) rollbackBlock(block blockRef) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + for _, receipt := range s.finalizer.invalidateBlock(block) { + // Queued finality still owns the delivered receipt after exhaustion, + // including when finalized-block history has already been pruned. + key := receiptOwner(receipt, receipt.owner) + if _, retained := s.deliveries[key]; !retained { + if s.deliveries == nil { + s.deliveries = make(map[receiptKey]Receipt) + } + s.deliveries[key] = receipt + } + } + s.retryMu.Lock() + invalidatedPending := make(map[*filterOwner]struct{}) + for key, pending := range s.pendingReceipts { + candidate := pending.receipt + if pending.completed != nil { + candidate = *pending.completed + } + if candidate.BlockHash() == block.hash && candidate.generation == block.generation { + invalidatedPending[key.owner] = struct{}{} + delete(s.pendingReceipts, key) + } + } + s.retryMu.Unlock() + // A custom matcher may require fields that lightweight block receipts lack. + // Previously published owners already matched, so use their retained data. + inactive := make(map[*filterOwner]struct{}) + for key, receipt := range s.deliveries { + if key.blockHash != block.hash || key.generation != block.generation { + continue + } + if !receipt.Reorged { + receipt.Final = false + receipt.Reorged = true + s.deliveries[key] = receipt + s.ch.Send(receipt) + } + if !s.hasFilter(key.owner) { + inactive[key.owner] = struct{}{} + } + } + // Notify every delivered receipt before retiring an exhausted lifetime. + for owner := range inactive { + if !s.finalizer.hasOwner(owner) { + s.retireOwner(owner, false) + } + } + for owner := range invalidatedPending { + s.releaseClaim(owner) + } +} - // Collect matches that need receipt fetching - type matchedReceipt struct { - receipt Receipt - filtererIdx int - filterer Filterer +func (s *subscriber) rollback(receipt Receipt, owner *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + key := receiptOwner(receipt, owner) + if !s.hasFilter(owner) { + return + } + if txn, claimed := s.claims[owner]; owner.Options().LimitOne && claimed && txn != receipt.TransactionHash() { + return + } + if retained, ok := s.deliveries[key]; ok { + if retained.Reorged { + return + } + receipt = retained + } + receipt.Filter = owner.Filterer + receipt.owner = owner + receipt.Final = false + receipt.Reorged = true + if s.deliveries == nil { + s.deliveries = make(map[receiptKey]Receipt) } - var toFetch []matchedReceipt + s.deliveries[key] = receipt + s.ch.Send(receipt) +} - // First pass: find all matches +func (s *subscriber) fetchReceipt(ctx context.Context, receipt Receipt) (Receipt, error) { + l := s.listener + l.receiptMu.Lock() + started := l.reorgRevision + l.receiptMu.Unlock() + expected := blockRef{receipt.BlockHash(), receipt.generation} + r, err := l.fetchTransactionReceipt(ctx, receipt.TransactionHash(), true, expected) + if err != nil { + return receipt, err + } + if expected.hash == (common.Hash{}) { + l.receiptMu.Lock() + valid := l.validFetchedBlock(r.BlockHash, blockRef{}, started) + receipt.generation = l.blockStates[r.BlockHash].generation + l.receiptMu.Unlock() + if !valid { + return receipt, ethereum.NotFound + } + } + receipt.receipt = r + receipt.logs = r.Logs + return receipt, nil +} + +func (s *subscriber) matchFiltersAndPublish(ctx context.Context, filterers []Filterer, receipts []Receipt) ([]bool, error) { + oks := make([]bool, len(filterers)) + type match struct { + receipt Receipt + owner *filterOwner + } + var matches []match + owners := make([]*filterOwner, len(filterers)) + for i, filter := range filterers { + owners[i] = s.owner(filter) + } for _, receipt := range receipts { - for i, filterer := range filterers { - matched, err := filterer.Match(ctx, receipt) + for i, owner := range owners { + if ctx.Err() != nil { + return oks, ctx.Err() + } + if owner == nil || !s.hasFilter(owner) { + continue + } + matched, err := owner.Match(ctx, receipt) if err != nil { return oks, superr.New(ErrFilterMatch, err) } - if !matched { - // skip, not a match continue } - - // its a match oks[i] = true - - if !receipt.Reorged { - toFetch = append(toFetch, matchedReceipt{ - receipt: receipt, - filtererIdx: i, - filterer: filterer, - }) + if receipt.Reorged { + s.rollback(receipt, owner) + continue } + matches = append(matches, match{receipt, owner}) } } - - if len(toFetch) == 0 { - return oks, nil - } - - // Fetch receipts concurrently sem := make(chan struct{}, maxConcurrentReceiptFetches) - g, gctx := errgroup.WithContext(ctx) - - for _, item := range toFetch { - item := item // capture loop variable + // One owner's fetch failure must not cancel another owner's delivery or retry. + var g errgroup.Group + for _, item := range matches { + if ctx.Err() != nil { + break + } + if !s.reserve(item.receipt, item.owner) { + continue + } g.Go(func() error { + defer s.releaseReservation(item.receipt, item.owner) select { case sem <- struct{}{}: defer func() { <-sem }() - case <-gctx.Done(): - return gctx.Err() + case <-ctx.Done(): + return ctx.Err() } - - // Fetch transaction receipt - r, err := s.listener.fetchTransactionReceipt(gctx, item.receipt.TransactionHash(), true) - if err != nil { - if errors.Is(err, ethereum.NotFound) { - // not found, don't retry - return superr.Wrap(fmt.Errorf("txn %s not found", item.receipt.TransactionHash()), err) + receipt := item.receipt + if receipt.receipt == nil || receipt.receipt.BlockNumber == nil || receipt.receipt.TxHash == (common.Hash{}) || receipt.receipt.BlockHash == (common.Hash{}) { + completed, err := s.fetchReceipt(ctx, receipt) + if err != nil { + if !errors.Is(err, ethereum.NotFound) && ctx.Err() == nil { + s.addPendingReceipt(receipt, item.owner) + } + return superr.Wrap(fmt.Errorf("failed to fetch txn %s receipt", receipt.TransactionHash()), err) } - - // might be a provider issue, add to pending receipts for retry - s.addPendingReceipt(item.receipt, item.filterer) - return superr.Wrap(fmt.Errorf("failed to fetch txn %s receipt due to node issue", item.receipt.TransactionHash()), err) - } - - // Update receipt with fetched data - item.receipt.receipt = r - item.receipt.logs = r.Logs - item.receipt.Filter = item.filterer - - // Finality enqueue if filter asked to Finalize, and receipt isn't already final - if !item.receipt.Final && item.filterer.Options().Finalize { - s.finalizer.enqueue(item.filterer.FilterID(), item.receipt, item.receipt.BlockNumber()) - } - - // LimitOne will auto unsubscribe now if were not also waiting for finalizer, - // and if the returned txn isn't one that has been reorged - // - // NOTE: when Finalize is set, we don't want to remove this filter until the txn finalizes, - // because its possible that it can reorg and we have to fetch it again after being re-mined. - // So we only remove the filter now if the filter finalizer isn't used, otherwise the - // finalizer will remove the LimitOne filter - toFinalize := item.filterer.Options().Finalize && !item.receipt.Final - if item.filterer.Options().LimitOne && !toFinalize { - s.RemoveFilter(item.receipt.Filter) - } - - // Check if receipt is already final, in case comes from cache when - // previously final was not toggled. - if s.listener.isBlockFinal(item.receipt.BlockNumber()) { - item.receipt.Final = true + receipt = completed } - - // Broadcast to subscribers (needs mutex as multiple goroutines may send) - s.ch.Send(item.receipt) - + s.publish(ctx, receipt, item.owner) return nil }) } - - // Wait for all fetches to complete - if err := g.Wait(); err != nil { - return oks, err + err := g.Wait() + if err == nil && ctx.Err() != nil { + err = ctx.Err() } - - return oks, nil + return oks, err } func (s *subscriber) finalizeReceipts(blockNum *big.Int) error { - // check subscriber finalizer - finalizer := s.finalizer - if finalizer.len() == 0 { - return nil - } - - finalTxns := finalizer.dequeue(blockNum) - if len(finalTxns) == 0 { - // no matching txns which have been finalized - return nil + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + for _, txn := range s.finalizer.dequeue(blockNum) { + receipt := txn.receipt + owner := receipt.owner + if !receipt.Reorged && s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + receipt.Final = true + s.ch.Send(receipt) + if owner != nil && (owner.Cond().TxnHash != nil || owner.Options().LimitOne) { + s.retireOwner(owner, true) + } + } + if !s.hasFilter(owner) && !s.finalizer.hasOwner(owner) { + s.retireOwner(owner, false) + } } - - // dispatch to subscriber finalized receipts - for _, x := range finalTxns { - if x.receipt.Reorged { - // for removed receipts, just skip - continue + // Retain mined data until its block is final, for receipt-free rollback. + pruned := make(map[*filterOwner]struct{}) + for key, receipt := range s.deliveries { + if receipt.BlockNumber() != nil && s.listener.isBlockFinal(receipt.BlockNumber()) { + delete(s.deliveries, key) + pruned[key.owner] = struct{}{} } - - // mark receipt as final, and send the receipt payload to the subscriber - x.receipt.Final = true - - // send to the subscriber - s.ch.Send(x.receipt) - - // Automatically remove filters for finalized txn hashes, as they won't come up again. - filter := x.receipt.Filter - if filter != nil && (filter.Cond().TxnHash != nil || filter.Options().LimitOne) { - s.RemoveFilter(filter) + } + // Rollback keeps a LimitOne selection so its orphaned txn can be re-mined. + // Once the orphan's height is final, free the owner to select another txn. + for owner := range pruned { + if _, claimed := s.claims[owner]; claimed { + s.releaseClaim(owner) } } - return nil } -func (s *subscriber) addPendingReceipt(receipt Receipt, filterer Filterer) { +func (s *subscriber) addPendingReceipt(receipt Receipt, owner *filterOwner) { + s.deliveryMu.Lock() + defer s.deliveryMu.Unlock() + if !s.hasFilter(owner) { + return + } s.retryMu.Lock() defer s.retryMu.Unlock() txnHash := receipt.TransactionHash() + key := receiptOwner(receipt, owner) + if !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + return + } if s.pendingReceipts == nil { // lazy init - s.pendingReceipts = make(map[common.Hash]*pendingReceipt) + s.pendingReceipts = make(map[receiptKey]*pendingReceipt) } if len(s.pendingReceipts) >= maxPendingReceipts { @@ -310,14 +628,14 @@ func (s *subscriber) addPendingReceipt(receipt Receipt, filterer Filterer) { return } - if _, exists := s.pendingReceipts[txnHash]; exists { + if _, exists := s.pendingReceipts[key]; exists { // already pending, skip return } - s.pendingReceipts[txnHash] = &pendingReceipt{ + s.pendingReceipts[key] = &pendingReceipt{ receipt: receipt, - filterer: filterer, + owner: owner, attempts: 1, nextRetryAt: time.Now().Add(1 * time.Second), // first retry after 1s } @@ -359,6 +677,7 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { wg.Add(1) go func(p *pendingReceipt) { defer wg.Done() + defer s.releaseReservation(p.receipt, p.owner) select { case sem <- struct{}{}: @@ -366,100 +685,114 @@ func (s *subscriber) retryPendingReceipts(ctx context.Context) { case <-ctx.Done(): // If context is cancelled, release the claim so the item can be retried later. s.retryMu.Lock() - if current, ok := s.pendingReceipts[p.receipt.TransactionHash()]; ok && current == p { + if current, ok := s.pendingReceipts[receiptOwner(p.receipt, p.owner)]; ok && current == p { current.nextRetryAt = time.Now().Add(100 * time.Millisecond) // small delay to avoid immediate retry } s.retryMu.Unlock() return } - // Attempt to fetch the receipt - txnHash := p.receipt.TransactionHash() - r, err := s.listener.fetchTransactionReceipt(ctx, txnHash, true) - - s.retryMu.Lock() - defer s.retryMu.Unlock() - - // Check if the item still exists and is the same one we claimed. - currentPending, exists := s.pendingReceipts[txnHash] - if !exists || currentPending != p { - s.listener.log.Debug("Pending receipt is stale or already processed, skipping retry", "txnHash", txnHash.String()) - return - } + s.retryPendingReceipt(ctx, p) + }(pending) + } - if err != nil { - if errors.Is(err, ethereum.NotFound) { - // Transaction genuinely doesn't exist - remove from queue - delete(s.pendingReceipts, txnHash) - s.listener.log.Debug("Receipt not found after retry, removing from queue", "txnHash", txnHash.String()) - return - } + wg.Wait() +} - // Provider error - update retry state directly on the pointer. - currentPending.attempts++ - if currentPending.attempts >= maxReceiptRetryAttempts { - delete(s.pendingReceipts, txnHash) - s.listener.log.Error( - "Failed to fetch receipt after max retries", - "txnHash", txnHash.String(), - "attempts", currentPending.attempts, - "error", err, - ) - // TODO: perhaps we should close the subscription here as we failed - // to deliver a receipt after many attempts? - return - } +// retryPendingReceipt fetches and commits one claimed pending receipt. The caller +// holds its semaphore slot and handles reservation cleanup. +func (s *subscriber) retryPendingReceipt(ctx context.Context, p *pendingReceipt) { + // Attempt to fetch the receipt + txnHash := p.receipt.TransactionHash() + key := receiptOwner(p.receipt, p.owner) + s.retryMu.Lock() + candidate := p.receipt + if p.completed != nil { + candidate = *p.completed + } + s.retryMu.Unlock() + receipt, err := s.fetchReceipt(ctx, candidate) - // Exponential backoff for next retry - backoff := time.Duration(1< maxWaitBetweenRetries { - backoff = maxWaitBetweenRetries - } - currentPending.nextRetryAt = time.Now().Add(backoff) - - s.listener.log.Debug( - "Receipt fetch failed, will retry", - "txnHash", txnHash.String(), - "attempt", currentPending.attempts, - "nextRetryIn", backoff, - ) - return - } + s.retryMu.Lock() - // Remove from pending list - delete(s.pendingReceipts, txnHash) + // Check if the item still exists and is the same one we claimed. + currentPending, exists := s.pendingReceipts[key] + if !exists || currentPending != p { + s.retryMu.Unlock() + s.listener.log.Debug("Pending receipt is stale or already processed, skipping retry", "txnHash", txnHash.String()) + return + } - // Update receipt with fetched data - p.receipt.receipt = r - p.receipt.logs = r.Logs - p.receipt.Filter = p.filterer + if err != nil { + defer s.retryMu.Unlock() + if errors.Is(err, ethereum.NotFound) { + // NotFound is terminal for missing receipts and obsolete block generations. + delete(s.pendingReceipts, key) + s.listener.log.Debug("Receipt not found after retry, removing from queue", "txnHash", txnHash.String()) + return + } - // Check finality - if s.listener.isBlockFinal(r.BlockNumber) { - p.receipt.Final = true - } + // Provider error - update retry state directly on the pointer. + currentPending.attempts++ + if currentPending.attempts >= maxReceiptRetryAttempts { + delete(s.pendingReceipts, key) + s.listener.log.Error( + "Failed to fetch receipt after max retries", + "txnHash", txnHash.String(), + "attempts", currentPending.attempts, + "error", err, + ) + return + } - // Handle finalization queue if needed - if !p.receipt.Final && p.filterer.Options().Finalize { - s.finalizer.enqueue(p.filterer.FilterID(), p.receipt, r.BlockNumber) - } + // Exponential backoff for next retry + backoff := time.Duration(1< maxWaitBetweenRetries { + backoff = maxWaitBetweenRetries + } + currentPending.nextRetryAt = time.Now().Add(backoff) - // Handle LimitOne filter removal - toFinalize := p.filterer.Options().Finalize && !p.receipt.Final - if p.filterer.Options().LimitOne && !toFinalize { - s.RemoveFilter(p.filterer) - } + s.listener.log.Debug( + "Receipt fetch failed, will retry", + "txnHash", txnHash.String(), + "attempt", currentPending.attempts, + "nextRetryIn", backoff, + ) + return + } - // Send to subscriber - s.ch.Send(p.receipt) + attempts := currentPending.attempts + s.retryMu.Unlock() - s.listener.log.Info( - "Successfully fetched receipt after retry", - "txnHash", txnHash.String(), - "attempts", currentPending.attempts, - ) - }(pending) + s.deliveryMu.Lock() + s.retryMu.Lock() + currentPending, exists = s.pendingReceipts[key] + s.retryMu.Unlock() + if !exists || currentPending != p { + s.deliveryMu.Unlock() + return + } + published := s.publishLocked(ctx, receipt, p.owner) + s.retryMu.Lock() + if current, exists := s.pendingReceipts[key]; exists && current == p { + if published || !s.hasFilter(p.owner) || !s.listener.isCurrentBlock(receipt.BlockHash(), receipt.generation) { + delete(s.pendingReceipts, key) + } else { + // A successful fetch can outlive its child deadline while waiting + // for delivery. Keep this exact valid owner available to a live retry. + current.completed = &receipt + current.nextRetryAt = time.Now().Add(100 * time.Millisecond) + } + } + s.retryMu.Unlock() + s.deliveryMu.Unlock() + if !published { + return } - wg.Wait() + s.listener.log.Info( + "Successfully fetched receipt after retry", + "txnHash", txnHash.String(), + "attempts", attempts, + ) } diff --git a/go.mod b/go.mod index a8a1b939..6c5bb73b 100644 --- a/go.mod +++ b/go.mod @@ -22,11 +22,11 @@ require ( github.com/google/gofuzz v1.2.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 - github.com/goware/breaker v0.2.0 + github.com/goware/breaker v0.3.2 github.com/goware/cachestore-mem v0.2.2 github.com/goware/cachestore-redis v0.2.1 github.com/goware/cachestore2 v0.12.3 - github.com/goware/channel v0.5.0 + github.com/goware/channel v0.6.0 github.com/goware/pp v0.0.3 github.com/goware/superr v0.0.2 github.com/holiman/uint256 v1.3.2 diff --git a/go.sum b/go.sum index 4eeddb69..4696bd83 100644 --- a/go.sum +++ b/go.sum @@ -96,16 +96,16 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/goware/breaker v0.2.0 h1:MJOaFjHwQ7h5/nCtjAUuOHhJeLCFF2GeUnXsBDMXq1k= -github.com/goware/breaker v0.2.0/go.mod h1:ijCEfXAa0j6w7IoHA4v6Sox2W6U9HUbI/t+5x0zGaug= +github.com/goware/breaker v0.3.2 h1:rVT/0UqQ70b3PN78Gj9dtZRDFJ1ZK8sm/fA8FkpOyCc= +github.com/goware/breaker v0.3.2/go.mod h1:CfgulI2zAaSLSnIeT8EN3mOJeWKSUbC26mGaveebT3E= github.com/goware/cachestore-mem v0.2.2 h1:toE6/1QMQQcQLJQpTIiTDAIHWLN4zvihoqZHq41cPns= github.com/goware/cachestore-mem v0.2.2/go.mod h1:KpXr+yVajbeN0s+CX/08hdSww2WhjtAL1pmNy69S03w= github.com/goware/cachestore-redis v0.2.1 h1:bMdkzGuy6Dsybq2chv3kRwu8UGoJz3aQBO9Cu41vEqQ= github.com/goware/cachestore-redis v0.2.1/go.mod h1:+8rOAfL1qNLNiXHe8+WACPk+I9kaLOAfArj2Z7FDiWg= github.com/goware/cachestore2 v0.12.3 h1:V4VODChSAV29p8htHj8Lb36Hvv28CLrJsw49gx0h+ks= github.com/goware/cachestore2 v0.12.3/go.mod h1:PR+lXK8UXa/wjKB7mpIj6HtRhC7vbcRXx4b5F1Av/ik= -github.com/goware/channel v0.5.0 h1:cOllKceCH5Xhibs0v8jtPJ81ez3L7WpYri/OU+9IBfg= -github.com/goware/channel v0.5.0/go.mod h1:Eai0KCjphDZ44M/qT7G1ZE6lZfywiTFvwV3Xc6cDPdo= +github.com/goware/channel v0.6.0 h1:eUY5Un8OiUgink+giDPRW0Vd4crdoGOvsNostRckv8s= +github.com/goware/channel v0.6.0/go.mod h1:Eai0KCjphDZ44M/qT7G1ZE6lZfywiTFvwV3Xc6cDPdo= github.com/goware/pp v0.0.3 h1:2Yv0IFGOpVjCDayPYzrqskCe9qmGoKBIyu6Uy//LVUU= github.com/goware/pp v0.0.3/go.mod h1:shID9y83CUGdg/BfO0SrVhchPpIAcT3ArfLVkq3x7tQ= github.com/goware/singleflight v0.3.0 h1:b+OM844fuHzanOlE84WeI+G8YMksUY636v0bdcAfnHE= @@ -160,7 +160,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=