diff --git a/.github/ci-groups.yml b/.github/ci-groups.yml index cfd026315..aae748262 100644 --- a/.github/ci-groups.yml +++ b/.github/ci-groups.yml @@ -29,4 +29,5 @@ excluded: - integration_test # Requires live Dash node - dash-fuzz # Honggfuzz binary targets, tested by fuzz.yml - masternode-seeds-fetcher # Tooling binary; needs live dashd RPC, exercised by update-masternode-seeds.yml + - dash-spv-bench # Benchmarking binary - git-state # Compile-time proc-macro with no tests, exercised via dash-spv diff --git a/.github/scripts/check_ci_status.sh b/.github/scripts/check_ci_status.sh index e7dcdeb2c..47a94ab21 100755 --- a/.github/scripts/check_ci_status.sh +++ b/.github/scripts/check_ci_status.sh @@ -8,8 +8,27 @@ set -euo pipefail PR_NUMBER="$1" REPO="$2" -gh pr checks "$PR_NUMBER" --repo "$REPO" --json name,bucket --jq ' - [.[] | select(.name | test("^(validate-triggers|evaluate)$") | not)] | +# gh pr checks signals check state through its exit code: 0 when every check +# passed (and, with --json, also when some failed), 8 when checks are still +# pending. Any other code is gh itself failing (auth, rate limit, network, +# unknown PR); surface that instead of misreading it as an empty check set, +# which would silently strip the label and hide the error. Keep stderr visible. +set +e +CHECKS=$(gh pr checks "$PR_NUMBER" --repo "$REPO" --json name,bucket) +EXIT=$? +set -e + +if [ "$EXIT" -ne 0 ] && [ "$EXIT" -ne 8 ]; then + exit "$EXIT" +fi + +if [ -z "$CHECKS" ]; then + echo "no_checks" + exit 0 +fi + +echo "$CHECKS" | jq -r ' + [.[] | select(.name | test("^(validate-triggers|evaluate|ready-for-review-trigger)$") | not)] | if length == 0 then "no_checks" elif all(.bucket == "pass" or .bucket == "skipping") then "all_passed" elif any(.bucket == "fail") then "has_failures" diff --git a/.github/scripts/ci_config.py b/.github/scripts/ci_config.py index 3ab0b5ade..c539b261f 100755 --- a/.github/scripts/ci_config.py +++ b/.github/scripts/ci_config.py @@ -153,6 +153,13 @@ def run_group_tests(args): cmd = ["cargo", "test", "-p", crate, "--all-features"] result = subprocess.run(cmd) + if ( + result.returncode == 0 + and not coverage + and crate == "key-wallet-manager" + ): + result = subprocess.run(["cargo", "test", "-p", crate]) + if result.returncode != 0: failed.append(crate) github_error(f"Test failed for {crate} on {args.os}") diff --git a/.github/workflows/ready-for-review-trigger.yml b/.github/workflows/ready-for-review-trigger.yml new file mode 100644 index 000000000..3529c82ad --- /dev/null +++ b/.github/workflows/ready-for-review-trigger.yml @@ -0,0 +1,25 @@ +name: Ready for Review Trigger + +# Fork pull requests run pull_request and pull_request_review events with a +# read-only token and no secrets, so they cannot add or remove labels. This +# workflow does nothing but complete, which fires a workflow_run event that the +# privileged "Ready for Review Label" workflow reacts to with a write token. +on: + pull_request: + types: [ready_for_review, labeled, converted_to_draft] + pull_request_review: + types: [submitted, dismissed] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: {} + +jobs: + relay: + name: ready-for-review-trigger + runs-on: ubuntu-latest + steps: + - name: Relay to applier + run: echo "Relaying pull request event to the ready-for-review applier via workflow_run." diff --git a/.github/workflows/ready-for-review.yml b/.github/workflows/ready-for-review.yml index cc02490e2..6f23183a9 100644 --- a/.github/workflows/ready-for-review.yml +++ b/.github/workflows/ready-for-review.yml @@ -1,27 +1,33 @@ name: Ready for Review Label +# Runs only on workflow_run so it holds a write-capable token even for pull +# requests opened from forks. Events tied to the fork head (pull_request, +# pull_request_review) get a read-only token and no secrets, so they cannot +# mutate labels themselves. The companion "Ready for Review Trigger" workflow +# relays those events here by completing and firing this workflow_run. on: - pull_request: - types: [ready_for_review, labeled] - pull_request_review: - types: [submitted, dismissed] workflow_run: - workflows: ["Continuous integration", "Pre-commit Checks", "Sanitizer"] + workflows: + - "Continuous integration" + - "Pre-commit Checks" + - "Sanitizer" + - "Ready for Review Trigger" types: [completed] concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.workflow_run.head_branch }} + group: ${{ github.workflow }}-${{ github.event.workflow_run.head_sha }} cancel-in-progress: true permissions: + contents: read pull-requests: write checks: read + issues: write jobs: # Ensure all pull_request-triggered workflows are monitored by workflow_run validate-triggers: runs-on: ubuntu-latest - if: github.event_name == 'workflow_run' steps: - uses: actions/checkout@v4 with: @@ -34,10 +40,7 @@ jobs: SELF=".github/workflows/ready-for-review.yml" # Extract workflow names from the workflow_run trigger list in this file - MONITORED=$(grep -A1 'workflows:' "$SELF" \ - | grep '\[' \ - | tr ',' '\n' \ - | sed 's/.*"\([^"]*\)".*/\1/') + MONITORED=$(grep -E '^\s+- "' "$SELF" | sed 's/.*"\([^"]*\)".*/\1/') MISSING="" for f in .github/workflows/*.yml; do @@ -73,14 +76,6 @@ jobs: evaluate: runs-on: ubuntu-latest - permissions: - pull-requests: write - checks: read - issues: write - # Skip non-CodeRabbit reviews - if: >- - github.event_name != 'pull_request_review' || - github.event.review.user.login == 'coderabbitai[bot]' steps: - uses: actions/checkout@v4 with: @@ -90,54 +85,42 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPO: ${{ github.repository }} - EVENT_NAME: ${{ github.event_name }} - EVENT_ACTION: ${{ github.event.action }} - REVIEW_STATE: ${{ github.event.review.state }} - PR_NUMBER_DIRECT: ${{ github.event.pull_request.number }} - LABEL_NAME: ${{ github.event.label.name }} - PULL_REQUESTS_JSON: ${{ toJson(github.event.workflow_run.pull_requests) }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} run: | - # On review dismissal or changes requested, remove label and exit - if [ "$EVENT_NAME" = "pull_request_review" ]; then - if [ "$EVENT_ACTION" = "dismissed" ] || [ "$REVIEW_STATE" = "changes_requested" ]; then - echo "CodeRabbit review dismissed or changes requested, removing label." - gh pr edit "$PR_NUMBER_DIRECT" --repo "$REPO" --remove-label "ready-for-review" || true - exit 0 - fi - fi - - # On merge-conflict label, remove ready-for-review and exit - if [ "$EVENT_NAME" = "pull_request" ] && [ "$EVENT_ACTION" = "labeled" ]; then - if [ "$LABEL_NAME" = "merge-conflict" ]; then - echo "Merge conflict detected, removing ready-for-review label." - gh pr edit "$PR_NUMBER_DIRECT" --repo "$REPO" --remove-label "ready-for-review" || true - fi - exit 0 - fi - - # Collect PR numbers depending on the event type - if [ "$EVENT_NAME" = "workflow_run" ]; then - PR_NUMBERS=$(echo "$PULL_REQUESTS_JSON" | jq -r '.[].number') - else - PR_NUMBERS="$PR_NUMBER_DIRECT" - fi + # Resolve the PR(s) for the head commit. github.event.workflow_run's + # pull_requests array is empty for fork PRs, and the commit-to-PR + # endpoint does not index fork heads, so match the head sha against + # the list of open PRs, which works regardless of where the branch + # lives. Matching the exact sha also ignores stale runs whose commit + # the PR has already moved past. + PR_NUMBERS=$(gh api --paginate "repos/$REPO/pulls?state=open&per_page=100" \ + --jq ".[] | select(.head.sha == \"$HEAD_SHA\") | .number") if [ -z "$PR_NUMBERS" ]; then - echo "No associated pull requests found, skipping." + echo "No open pull request found for $HEAD_SHA, skipping." exit 0 fi for PR_NUMBER in $PR_NUMBERS; do echo "=== Checking PR #$PR_NUMBER ===" - # Skip draft PRs - IS_DRAFT=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json isDraft --jq '.isDraft') - if [ "$IS_DRAFT" = "true" ]; then - echo "PR is a draft. Skipping." + PR_JSON=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json isDraft,labels) + + if [ "$(echo "$PR_JSON" | jq -r '.isDraft')" = "true" ]; then + echo "PR is a draft. Removing ready-for-review label if present." + gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "ready-for-review" || true continue fi - # Check if CodeRabbit has approved + # A merge conflict disqualifies the PR regardless of the other gates. + if [ "$(echo "$PR_JSON" | jq -r '[.labels[].name] | index("merge-conflict") != null')" = "true" ]; then + echo "Merge conflict present. Removing ready-for-review label if present." + gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "ready-for-review" || true + continue + fi + + # Check if CodeRabbit has approved. A dismissed or changes-requested + # review leaves the latest state != APPROVED and removes the label. CODERABBIT_STATE=$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/reviews" | jq -rs ' add | map(select(.user.login == "coderabbitai[bot]")) @@ -148,7 +131,8 @@ jobs: echo "CodeRabbit review state: $CODERABBIT_STATE" if [ "$CODERABBIT_STATE" != "APPROVED" ]; then - echo "CodeRabbit has not approved. Skipping." + echo "CodeRabbit has not approved. Removing label if present." + gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "ready-for-review" || true continue fi diff --git a/.github/workflows/sanitizer.yml b/.github/workflows/sanitizer.yml index f44639514..839bb8fe8 100644 --- a/.github/workflows/sanitizer.yml +++ b/.github/workflows/sanitizer.yml @@ -18,6 +18,13 @@ on: permissions: contents: read +# Pinned nightly: nightly-2026-07-14 (rustc daf2e5e18) ICEs while expanding the +# libtest harness for dash-spv-ffi ("attribute is missing tokens: +# rustc_test_entrypoint_marker" in rustc_ast/src/attr/mod.rs). Bump or restore +# the floating `nightly` once a fixed nightly ships. +env: + SANITIZER_TOOLCHAIN: nightly-2026-07-13 + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -28,8 +35,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ env.SANITIZER_TOOLCHAIN }} components: rust-src - name: Run tests with ASAN @@ -41,11 +49,11 @@ jobs: SKIP_DASHD_TESTS: 1 run: | # FFI crates (C interop) - cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \ + cargo "+$SANITIZER_TOOLCHAIN" test -Zbuild-std --target x86_64-unknown-linux-gnu \ -p key-wallet-ffi -p dash-spv-ffi --lib --tests # Core crypto crates (unsafe optimizations) - cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \ + cargo "+$SANITIZER_TOOLCHAIN" test -Zbuild-std --target x86_64-unknown-linux-gnu \ -p dashcore -p dashcore_hashes --lib --tests tsan: @@ -53,8 +61,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ env.SANITIZER_TOOLCHAIN }} components: rust-src - name: Run tests with TSAN @@ -65,5 +74,5 @@ jobs: SKIP_DASHD_TESTS: 1 run: | # Async crate with concurrent code - cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \ + cargo "+$SANITIZER_TOOLCHAIN" test -Zbuild-std --target x86_64-unknown-linux-gnu \ -p dash-spv --lib --tests diff --git a/.gitignore b/.gitignore index e8ca067f8..4f55150c6 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ __pycache__/ # Build scripts artifacts *.log +target-linux/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a9dd317ca..bb675c4e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,154 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## 0.45.0 - 2026-10-21 + +### Highlights + +- **Dash Core v24 support:** v24 masternode lists in `rpc-json` (nested platform addresses, shared and multi-payout masternodes, non-IP services), CbTx v4, ProRegTx, ProUpServTx and ProUpRegTx version 3, raw payloads for unknown special transactions, `getblockchaininfo` and `getdeploymentinfo`, and asset locks decoded without their payload length prefix (#1093, #1136, #1137, #1138, #1152, #1154, #1155, #1165, #1167). +- **GroveDB bincode:** binary serialization moved to `grovedb-bincode` 2.1.0, sharing its traits with Platform (#1005). +- **Wallet funding and reservations:** UTXO and receive-address reservations, owner-tagged reservations, funding from several account types, a CoinJoin funding account and drain mode for asset locks, and OP_RETURN outputs (#812, #818, #916, #925, #915, #944, #922). +- **SPV sync reliability:** sync anchored at the wallet birth height, filter sync no longer stalls after a restart or a full sync, peer selection and stall handling reworked, in-flight downloads requeued on peer disconnect, and committed ranges rescanned for newly derived scripts (#848, #835, #816, #943, #941, #866, #989). +- **Security hardening:** forged ChainLocks are rejected, oversized headers messages are refused, and the RPC client no longer logs request arguments and results, which can carry credentials (#1059, #1140, #1147). + +### Added + +- `Signer::extended_public_key` (#817) @shumkov +- Address pool gap limits can be changed at runtime (#822) @ZocoLini +- Track internal address pools for CoinJoin accounts (#821) @ZocoLini +- More account types and a strategy to consume every available UTXO (#823) @ZocoLini +- `Zeroize` for `ExtendedPrivKey` (#833) @lklimek +- Seed the sync checkpoint from `birth_height` (#693) @xdustinface +- Anchor sync at the wallet birth height checkpoint in `dash-spv` (#848) @xdustinface +- One-shot transaction decode to (address, amount) structs in `key-wallet-ffi` (#870) @QuantumExplorer +- Typed special transaction payload on `FFITransactionRecord` (#876) @QuantumExplorer +- Floor mainnet sync at the HD/BIP39 activation height (#877) @xdustinface +- Provider key derivation API for operator and platform node keys at an index (#881) @QuantumExplorer +- Dedicated `PlatformNodeId` newtype for `platform_node_id` fields (#885) @QuantumExplorer +- Missing DIP-15 derivation constants in `dip9` (#886) @shumkov +- Stamp retained transaction records with their in-block position (#891) @QuantumExplorer +- Track network acceptance of broadcast transactions in `dash-spv` (#913) @QuantumExplorer +- `dash-spv-bench` multi-scenario sync benchmark harness (#908, #965, #967) @ZocoLini +- Reserve receive addresses on hand-out (#818) @xdustinface +- CoinJoin funding account and drain mode for asset lock builds (#915) @QuantumExplorer +- Owner-tagged reservations to close the broadcast-release race (#916) @bfoss765 +- Lossless persistence event channel in `key-wallet-manager` (#927) @QuantumExplorer +- OP_RETURN outputs, output order control and change to VIN0 (#922) @romchornyi +- Fund transactions from several account types (#925) @ZocoLini +- DashPay contact account funding (#929) @ZocoLini +- Fund an asset lock from a caller-chosen list of accounts (#944) @QuantumExplorer +- Canonical `is_contact_owned` predicate on account types (#952) @QuantumExplorer +- Prune spent single-use CoinJoin addresses from the filter scan query (#949) @QuantumExplorer +- Accelerate compressed header sync (#950) @PastaPastaPasta +- Let a drain carry zero-value OP_RETURN data outputs (#928) @HashEngineering +- Carry the sweep winner's mined height on `TransactionsSwept` (#975) @romchornyi +- Payload finalization seam for input-committing special transactions (#991) @QuantumExplorer +- Let a build fund from only the inputs it was given (#994) @romchornyi +- **Breaking:** decode and encode CbTx v4 `merkleRootAssetUnlocks` (#1137) @ZocoLini +- **Breaking:** decode and encode ProUpServTx and ProUpRegTx version 3 (#1138) @ZocoLini +- **Breaking:** decode and encode ProRegTx version 3 (#1152) @ZocoLini +- Decode Core v24 `getblockchaininfo` and add `getdeploymentinfo` (#1165) @shumkov +- Decode asset locks ignoring their payload length prefix (#1167) @shumkov + +### Changed + +- **Breaking:** the `bincode` feature and binary serialization dependencies now use + `grovedb-bincode` 2.1.0 and its derive macros from crates.io, sharing the same + serialization traits as Platform. Existing bincode encodings and C interfaces + are unchanged, but Rust consumers must use the fork for compatible `Encode`/`Decode` traits. + The dependency switch does not automatically opt types into `DecodeUntrusted`. + (#1005) @QuantumExplorer +- **Breaking:** parse Core v24 masternode lists in `rpc-json`, including nested platform addresses, shared and multi-payout masternodes, non-IP services and partial address diffs (#1093) @shumkov +- **Breaking:** decode unknown special transaction types as raw payloads (#1136) @ZocoLini +- **Breaking:** keep special payloads raw when their layout is unknown (#1154) @ZocoLini +- **Breaking:** temporarily expose the coinbase payload of the newest masternode list diff as `MasternodeListEngine::latest_coinbase()` and `DashSpvClient::latest_coinbase()`. It is not verified, so a single peer can serve any payload, and it is replaced by a verified version in a later release (#1204) @shumkov +- **Breaking:** name the outpoints a sweep releases (#962) @romchornyi +- **Breaking:** unify mnemonic parsing on one auto-detecting path (#981) @PastaPastaPasta +- Move `extended_public_key` out of `Signer` into the `ExtendedPubKeySigner` subtrait (#839) @QuantumExplorer +- Refactor `ManagedAccountType::from_account_type` (#828) @ZocoLini +- Compute fewer block hashes (#834) @ZocoLini +- SIMD-batched SipHash-2-4 for BIP158 filter matching (#845) @ZocoLini +- Peer reputation module cleanup (#328) @ZocoLini +- Widen `DEFAULT_COINJOIN_GAP_LIMIT` from 30 to 100 for dashj parity (#868) @bfoss765 +- Replace `TARGET_PEERS` with the existing `max_peers` config field (#827) @ZocoLini +- Reduce debug log verbosity in `dash-spv` (#917) @ZocoLini +- Add recent mainnet and testnet checkpoints (#857) @xdustinface +- Refresh masternode seed files and the mainnet seed list (#805, #874, #906, #963, #983, #970) + +### Fixed + +- Honor the account index in the DashPay derivation path (#813) @shumkov +- Prevent a filter sync crash on a new block after a full sync (#816) @xdustinface +- Resolve a just-retired quorum by walking earlier masternode lists (#801) @xdustinface +- Reprocess blocks on rescan against newly derived scripts (#820) @xdustinface +- Compute the SML entry hash with `SER_GETHASH` semantics (#798) @xdustinface +- Remove a release build warning (#825) @ZocoLini +- Reserve selected UTXOs to prevent double spends (#812) @xdustinface +- Degrade gracefully when advisory file locking is unsupported (#841) @QuantumExplorer +- Avoid filter sync getting stuck on the first new block after a restart (#835) @xdustinface +- Don't build asset locks on unconfirmed funds (#836) @xdustinface +- Preserve UTXO flags across reprocessing (#837) @xdustinface +- Actionable error when `add_account(_, None)` hits a keyless wallet (#847) @lklimek +- Derive the storage lockfile path from the full directory name (#862) @QuantumExplorer +- Zeroize `ExtendedBLSPrivKey` and `ExtendedEd25519PrivKey` on drop (#859) @QuantumExplorer +- Match masternode owner and voting key hashes in compact filters (#863) @xdustinface +- Match masternode collateral outpoints in compact filters (#864) @xdustinface +- Return the fee the transaction actually pays from `build` and `build_signed` (#872) @QuantumExplorer +- Cover CoinJoin and DashPay accounts for AssetLock routing (#867) @bfoss765 +- Derive BLS operator and Ed25519 platform node keys per DashSync and dashbls (#879) @QuantumExplorer +- Extend the gap limit for operator keys like owner and voting keys (#882) @QuantumExplorer +- Platform node id is the Tenderdash node ID, not hash160 (#884) @QuantumExplorer +- Normalize `PlatformNodeId` to canonical byte order at the consensus boundary (#889) @QuantumExplorer +- Don't preload filters below the stored range on filter download start (#893) @QuantumExplorer +- Fix issue 911 in `key-wallet` (#914) @ZocoLini +- Rewrite branch-and-bound coin selection (#919) @ZocoLini +- Recover InstantSend locks received before quorum sync completes (#895) @bfoss765 +- Record an out-of-order UTXO spend so it no longer shows a phantom unspent balance (#909) @bfoss765 +- Fix the dashd process port assignment in tests (#923) @ZocoLini +- Stop counting a contact's watch-only coins as the wallet's (#926) @romchornyi +- Never offer an outpoint the builder already holds (#931) @QuantumExplorer +- Stop the filter rescan loop when the chain anchors above the wallet (#933) @xdustinface +- Use `WalletAccountCreationOptions::default()` in the benchmark framework (#938) @ZocoLini +- Requeue in-flight downloads when a single peer disconnects (#941) @xdustinface +- Reliable peer selection and stall handling during sync (#943) @xdustinface +- Reissue requeued masternode requests from `tick` (#953) @xdustinface +- Key rotation ChainLock signatures per quorum work block (#934) @xdustinface +- Resume on the invariant `start_download` asserts (#955) @romchornyi +- Release clean storage segments below the committed height during long scans (#946) @bfoss765 +- Enable `key-wallet/getrandom` for mnemonic generation (#945) @QuantumExplorer +- Stop a never-broadcast transaction from crediting money that does not exist (#961) @romchornyi +- Promote finished header segments from the tick, not only on a message (#960) @romchornyi +- Rescan committed filter ranges for newly derived scripts (#866) @HashEngineering +- Collect a sweep's surviving inputs once instead of per candidate (#966) @romchornyi +- Extend the CFHeaders queue from the tick, not only on a header event (#964) @romchornyi +- Walk a sweep's descendant closure once instead of per generation (#969) @romchornyi +- Recover masternode sync from a rejected QRInfo (#947) @bfoss765 +- Accept non-English BIP-39 mnemonics on all parse paths (#980) @llbartekll +- Count acceptance signals that arrived before the broadcast (#982) @romchornyi +- Coalesce committed-range filter rescans across batch commits (#974) @romchornyi +- Build `dash-spv-bench` on a Linux host and survive having no terminal (#987) @ZocoLini +- Report the transactions each account knows in `dash-spv-bench` (#984) @ZocoLini +- Stop pruning spent CoinJoin addresses from the filter scan (#985) @ZocoLini +- Coinbase maturity clock (#996) @ZocoLini +- Promote confirmed spend-only transactions (#998) @shumkov +- Ask every account whether a transaction is new, not just the matched ones (#1000) @ZocoLini +- Keep a spend-first coin recognisable so its spend stays in history (#1001) @ZocoLini +- Discover DashPay asset unlock receipts (#1004) @PastaPastaPasta +- Stop losing derived scripts, and close the loop on wallet state (#989) @ZocoLini +- Prune observed spends during the initial sync, so a wallet persisted mid-sync loads again (#1014) @ZocoLini +- Only decode a special payload from transaction version 3 (#1155) @ZocoLini +- Keep the pre-v4 serde shape of `CoinbasePayload` (#1149) @ZocoLini +- Send the ChainLock block hash in display order to `submitchainlock` (#1148) @ZocoLini +- Consult every fund-bearing account for provider transactions (#1009) @ZocoLini +- Extend DIP-15 contact pools as their payments arrive (#1035) @ZocoLini +- Keep a mined transaction promotable when its IS lock arrives late (#1037) @ZocoLini +- Re-apply a spend whose coin was funded after it (#1015) @ZocoLini +- Reject a ChainLock that the masternode list below its signing height refuses (#1059) @ZocoLini +- Reject headers messages above Core's batch limit (#1140) @ZocoLini +- Keep RPC request arguments and results out of the logs (#1147) @ZocoLini +- Sweep a conflict loser's keys-only records too (#1143) @lklimek +- Set LLMQ params to Dash Core's values (#1187) @ZocoLini + ## 0.44.0 - 2026-07-01 ### Added diff --git a/Cargo.toml b/Cargo.toml index 30e6bff94..83db5f348 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,13 @@ [workspace] -members = ["dash", "dash-network", "hashes", "internals", "fuzz", "rpc-client", "rpc-json", "rpc-integration-test", "key-wallet", "key-wallet-manager", "key-wallet-ffi", "dash-spv", "dash-spv-ffi", "git-state", "dash-network-seeds", "masternode-seeds-fetcher"] +members = ["dash", "dash-network", "hashes", "internals", "fuzz", "rpc-client", "rpc-json", "rpc-integration-test", "key-wallet", "key-wallet-manager", "key-wallet-ffi", "dash-spv", "dash-spv-ffi", "git-state", "dash-network-seeds", "masternode-seeds-fetcher", "dash-spv-bench"] resolver = "2" +[workspace.dependencies] +bincode = { package = "grovedb-bincode", version = "=2.1.0" } +bincode_derive = { package = "grovedb-bincode-derive", version = "=2.1.0" } + [workspace.package] -version = "0.44.0" +version = "0.45.0" [patch.crates-io] dashcore_hashes = { path = "hashes" } diff --git a/README.md b/README.md index 56c020b7f..609d965e9 100644 --- a/README.md +++ b/README.md @@ -47,22 +47,58 @@ Supports (or should support) * JSONRPC interaction with Dash Core * FFI bindings for C/Swift integration (dash-spv-ffi, key-wallet-ffi) * High-level wallet management with transaction building and UTXO management +* Dash special transaction data types, including DIP4 coinbase payloads and + quorum commitments +* Simplified masternode list and QRInfo processing, including optional LLMQ + commitment and BLS signature validation +* ChainLock and InstantSend lock verification through the masternode list + engine and `dash-spv` + +## Dash verification support + +This workspace contains reusable building blocks for applications that need to +verify Dash-specific data without making an RPC call for every operation. +`dashcore` provides the protocol types, hashing, simplified masternode list and +LLMQ machinery, while `dash-spv` adds peer synchronization, persistent header +and compact-filter storage, and ChainLock and InstantSend processing. The +`dash-spv-ffi` crate exposes the SPV client through a C-compatible API suitable +for C, C++, Swift, and other languages that can call a C ABI. + +These components do not together form a standalone implementation of all Dash +Core consensus rules. See the consensus limitations below before choosing a +trust model or using them for security-sensitive validation. # Known limitations ## Consensus -This library **must not** be used for consensus code (i.e. fully validating -blockchain data). It technically supports doing this, but doing so is very -ill-advised because there are many deviations, known and unknown, between -this library and the Dash Core reference implementation. In a consensus -based cryptocurrency such as Dash it is critical that all parties are -using the same rules to validate data, and this library does not and might -never implement the same rules as Core. - -Given the complexity of both C++ and Rust, it is unlikely that this will -ever be fixed, and there are no plans to do so. Of course, patches to -fix specific consensus incompatibilities are welcome. +This library **must not** be treated as a drop-in replacement for Dash Core's +full block-validation pipeline. It implements a useful subset of the required +primitives and verification logic, but it does not maintain all of the +consensus state or apply every contextual rule required to independently accept +and construct mainnet blocks. + +Notably, the workspace does not currently provide a complete equivalent of +Dash Core's stateful block connection and block-template logic, including all +of the following as one validated pipeline: + +* expected-difficulty and complete contextual header validation; +* UTXO state transitions and all transaction and special-transaction rules; +* subsidy, fee, masternode and Platform reward, and coinbase payout checks; +* governance object, superblock trigger, and superblock payout validation; +* asset-lock credit-pool accounting and withdrawal limits; and +* construction of consensus-valid mining templates. + +Support for parsing a payload or verifying an individual commitment, quorum, +ChainLock, or InstantSend lock should not be interpreted as validation of the +block and state that produced it. Applications must define their trust and +state model explicitly and use Dash Core when exact agreement with the network's +consensus implementation is required. The C-compatible bindings expose the SPV +and wallet APIs; they are not a general full-consensus validation ABI. + +Consensus compatibility is difficult to establish independently of the C++ +reference implementation, and known and unknown deviations may exist. Patches +that close specific, tested gaps are welcome. ## Support for 16-bit pointer sizes diff --git a/contrib/run_clippy.py b/contrib/run_clippy.py index 054a61098..29b0b50c9 100755 --- a/contrib/run_clippy.py +++ b/contrib/run_clippy.py @@ -4,7 +4,7 @@ import subprocess import sys -cmd = [ +base = [ "cargo", "clippy", "--workspace", "--all-features", @@ -14,8 +14,16 @@ # Exclude dash-fuzz on Windows (honggfuzz doesn't support Windows) if sys.platform == "win32": - cmd.extend(["--exclude", "dash-fuzz"]) + base.extend(["--exclude", "dash-fuzz"]) -cmd.extend(["--", "-D", "warnings"]) +profiles = [ + ("debug", []), + ("release", ["--release"]), +] -sys.exit(subprocess.call(cmd)) +for label, extra in profiles: + cmd = base + extra + ["--", "-D", "warnings"] + rc = subprocess.call(cmd) + if rc != 0: + print(f"Clippy failed ({label}): {' '.join(cmd)}", file=sys.stderr) + sys.exit(rc) diff --git a/dash-network-seeds/seeds/mainnet.txt b/dash-network-seeds/seeds/mainnet.txt index 4f5ee7c72..e1bbb4e3c 100644 --- a/dash-network-seeds/seeds/mainnet.txt +++ b/dash-network-seeds/seeds/mainnet.txt @@ -1,15 +1,15 @@ -# Auto-generated by masternode-seeds-fetcher on 2026-05-18T07:36:21Z for mainnet +# Auto-generated by masternode-seeds-fetcher on 2026-08-24T06:22:54Z for mainnet # Source: Dash P2P network (mnlistdiff) -# Primary peer: 168.119.87.131:9999 -# Tip block hash: 0000000000000013670d971ce450b3774deb714558cd52edcb2c8c486a86b048 -# Tip block height: 2473351 -# 2054 seeds (1697 regular + 357 evo) of 2943 total masternodes, valid-only -# Probe summary: core_reachable=2036/2054 platform_reachable=354/357 ssl_valid=296/357 +# Primary peer: 185.239.209.6:9999 +# Tip block hash: 000000000000001737bdb25276b44631abc26d7e74ad8c0284176696372f16bd +# Tip block height: 2527048 +# 2089 seeds (1749 regular + 340 evo) of 2985 total masternodes, valid-only +# Probe summary: core_reachable=2068/2089 platform_reachable=337/340 ssl_valid=256/340 # Columns: # Values: core_reach/plat_reach=ok|timeout|refused|error|?, core_sync=sync|-N|+N|?, plat_live=ok|none|?|-, plat_ssl=valid|expired|self-signed|untrusted|no-handshake|?|- # Do not edit manually — refreshed weekly by .github/workflows/update-masternode-seeds.yml -regular 2.56.213.218:9999 - ok sync - - - -regular 2.56.213.221:9999 - ok sync - - - +regular 1.117.75.220:9999 - error ? - - - +regular 2.25.80.112:9999 - ok sync - - - evo 2.59.156.192:9999 443 ok sync ok ok valid regular 2.59.219.54:9999 - ok sync - - - regular 2.233.120.35:9999 - ok sync - - - @@ -27,39 +27,43 @@ evo 5.39.27.224:9999 443 ok sync ok ok valid regular 5.53.124.120:9999 - ok sync - - - evo 5.75.133.148:9999 443 ok sync ok ok valid regular 5.75.251.102:9999 - ok sync - - - -regular 5.135.14.86:9999 - timeout ? - - - +regular 5.135.14.86:9999 - ok sync - - - regular 5.161.110.79:9999 - ok sync - - - -regular 5.175.162.96:9999 - ok sync - - - -regular 5.175.162.97:9999 - ok sync - - - -regular 5.175.162.98:9999 - ok sync - - - -regular 5.175.162.99:9999 - ok sync - - - -regular 5.175.162.100:9999 - ok sync - - - -regular 5.175.162.101:9999 - ok sync - - - -regular 5.175.162.102:9999 - ok sync - - - -regular 5.175.162.104:9999 - ok sync - - - -regular 5.175.162.105:9999 - ok sync - - - -regular 5.175.162.106:9999 - ok sync - - - -regular 5.175.162.107:9999 - ok sync - - - -regular 5.175.162.108:9999 - ok sync - - - -regular 5.175.162.109:9999 - ok sync - - - -regular 5.175.162.110:9999 - ok sync - - - -regular 5.175.162.136:9999 - ok sync - - - -regular 5.175.162.137:9999 - ok sync - - - -regular 5.175.162.138:9999 - ok sync - - - -regular 5.175.162.139:9999 - ok sync - - - -regular 5.175.162.140:9999 - ok sync - - - -regular 5.175.162.141:9999 - ok sync - - - -regular 5.175.162.142:9999 - ok sync - - - -regular 5.175.162.156:9999 - ok sync - - - -regular 5.175.162.157:9999 - ok sync - - - -regular 5.175.162.158:9999 - ok sync - - - -regular 5.175.162.159:9999 - ok sync - - - -regular 5.175.162.160:9999 - ok sync - - - -regular 5.175.162.161:9999 - ok sync - - - -regular 5.175.162.162:9999 - ok sync - - - +regular 5.175.162.184:9999 - ok sync - - - +regular 5.175.162.185:9999 - ok sync - - - +regular 5.175.162.186:9999 - ok sync - - - +regular 5.175.162.187:9999 - ok sync - - - +regular 5.175.162.188:9999 - ok sync - - - +regular 5.175.162.189:9999 - ok sync - - - +regular 5.175.162.190:9999 - ok sync - - - +regular 5.175.162.191:9999 - ok sync - - - +regular 5.175.162.192:9999 - ok sync - - - +regular 5.175.162.193:9999 - ok sync - - - +regular 5.175.162.194:9999 - ok sync - - - +regular 5.175.162.195:9999 - ok sync - - - +regular 5.175.162.196:9999 - ok sync - - - +regular 5.175.162.197:9999 - ok sync - - - +regular 5.175.162.212:9999 - ok sync - - - +regular 5.175.162.213:9999 - ok sync - - - +regular 5.175.162.214:9999 - ok sync - - - +regular 5.175.162.215:9999 - ok sync - - - +regular 5.175.162.216:9999 - ok sync - - - +regular 5.175.162.217:9999 - ok sync - - - +regular 5.175.162.218:9999 - ok sync - - - +regular 5.175.162.219:9999 - ok sync - - - +regular 5.175.162.220:9999 - ok sync - - - +regular 5.175.162.221:9999 - ok sync - - - +regular 5.175.162.222:9999 - ok sync - - - +regular 5.175.162.223:9999 - ok sync - - - +regular 5.175.162.224:9999 - ok sync - - - +regular 5.175.162.225:9999 - ok sync - - - +regular 5.180.234.247:9999 - ok sync - - - evo 5.189.145.80:9999 443 ok sync ok ok valid evo 5.189.151.7:9999 443 ok sync ok ok valid evo 5.189.164.253:9999 443 ok sync ok ok valid +evo 5.199.166.45:9999 443 ok sync ok ok valid +evo 5.199.166.140:9999 443 ok sync ok ok valid +evo 5.199.166.218:9999 443 ok sync ok ok valid regular 8.219.0.187:9999 - ok sync - - - regular 8.219.1.166:9999 - ok sync - - - regular 8.219.4.138:9999 - ok sync - - - @@ -195,33 +199,46 @@ regular 8.222.149.162:9999 - ok sync - - - regular 8.222.149.195:9999 - ok sync - - - regular 8.222.150.74:9999 - ok sync - - - regular 8.222.151.173:9999 - ok sync - - - +regular 13.140.137.75:9999 - ok sync - - - +regular 13.140.137.151:9999 - ok sync - - - +regular 13.140.137.152:9999 - ok sync - - - +regular 13.140.137.153:9999 - ok sync - - - +regular 13.140.137.154:9999 - ok sync - - - +regular 13.140.144.201:9999 - ok sync - - - +regular 13.140.144.202:9999 - ok sync - - - regular 15.188.53.12:9999 - ok sync - - - -evo 15.235.102.215:9999 443 ok sync ok ok expired -evo 15.235.102.216:9999 443 ok sync ok ok expired +regular 15.204.112.226:9999 - ok sync - - - +evo 15.235.102.215:9999 443 ok sync ok ok valid +evo 15.235.102.216:9999 443 ok sync ok ok valid regular 23.88.22.66:9999 - ok sync - - - regular 23.88.22.67:9999 - ok sync - - - regular 23.88.22.68:9999 - ok sync - - - -evo 23.88.63.58:9999 443 ok sync ok ok valid +evo 23.88.63.58:9999 443 ok sync ok ok expired +regular 23.94.219.254:9999 - ok sync - - - regular 23.95.186.248:9999 - ok sync - - - -regular 23.163.0.174:9999 - ok sync - - - regular 31.10.97.36:9999 - ok sync - - - regular 31.56.38.0:9999 - ok sync - - - +regular 31.56.38.1:9999 - ok sync - - - regular 31.56.38.185:9999 - ok sync - - - regular 31.58.170.16:9999 - ok sync - - - +regular 31.58.170.80:9999 - ok sync - - - +regular 31.58.170.110:9999 - ok sync - - - regular 31.58.170.130:9999 - ok sync - - - regular 31.58.170.133:9999 - ok sync - - - regular 31.58.170.139:9999 - ok sync - - - regular 31.58.170.180:9999 - ok sync - - - regular 31.58.170.181:9999 - ok sync - - - regular 31.58.170.213:9999 - ok sync - - - +regular 31.58.170.222:9999 - ok sync - - - +regular 31.58.170.223:9999 - ok sync - - - regular 31.58.170.230:9999 - ok sync - - - +regular 31.77.108.136:9999 - ok sync - - - regular 31.148.99.148:9999 - ok sync - - - -evo 31.220.77.84:9999 443 ok sync ok ok valid evo 31.220.84.93:9999 443 ok sync ok ok valid -regular 31.220.87.198:9999 - ok sync - - - evo 31.220.91.43:9999 443 ok sync ok ok valid evo 31.220.91.60:9999 443 ok sync ok ok valid regular 34.209.37.222:9999 - ok sync - - - +evo 34.210.238.67:9999 443 ok sync ok ok valid regular 34.246.176.25:9999 - ok sync - - - regular 35.174.217.98:9999 - ok sync - - - regular 35.180.128.14:9999 - ok sync - - - @@ -239,19 +256,16 @@ evo 37.60.235.24:9999 443 ok sync ok ok valid evo 37.60.235.39:9999 443 ok sync ok ok valid evo 37.60.235.89:9999 443 ok sync ok ok valid evo 37.60.235.205:9999 443 ok sync ok ok valid -evo 37.60.235.218:9999 443 ok sync ok ok valid evo 37.60.235.224:9999 443 ok sync ok ok valid -evo 37.60.236.41:9999 443 ok sync ok ok valid evo 37.60.236.230:9999 443 ok sync ok ok valid evo 37.60.243.59:9999 443 ok sync ok ok valid evo 37.60.244.35:9999 443 ok sync ok ok valid -evo 37.60.244.87:9999 443 ok sync ok ok valid +evo 37.60.244.87:9999 443 ok sync ok ok expired evo 37.60.244.253:9999 443 ok sync ok ok valid evo 37.60.246.228:9999 443 ok sync ok ok valid evo 37.60.254.201:9999 443 ok sync ok ok valid evo 37.60.254.202:9999 443 ok sync ok ok valid evo 37.60.254.205:9999 443 ok sync ok ok valid -evo 37.60.254.211:9999 443 ok sync ok ok valid evo 37.60.254.213:9999 443 ok sync ok ok valid evo 37.60.255.160:9999 443 ok sync ok ok valid regular 37.97.227.21:9999 - ok sync - - - @@ -262,17 +276,26 @@ regular 37.139.11.82:9999 - ok sync - - - regular 37.139.29.66:9999 - ok sync - - - regular 38.54.12.111:9999 - ok sync - - - regular 38.102.125.108:9999 - ok sync - - - -evo 38.242.145.178:9999 443 ok sync ok ok valid evo 38.242.197.189:9999 443 ok sync ok ok valid evo 38.242.200.227:9999 443 ok sync ok ok valid evo 38.242.206.56:9999 443 ok sync ok ok valid evo 38.242.206.103:9999 443 ok sync ok ok valid regular 38.242.211.123:9999 - ok sync - - - evo 38.242.218.26:9999 443 ok sync ok ok valid -evo 43.133.171.101:9999 443 ok sync ok ok valid +regular 43.133.81.22:9999 - ok sync - - - +evo 43.133.171.101:9999 443 ok sync ok ok expired regular 43.134.180.113:9999 - ok sync - - - -evo 43.167.239.145:9999 443 ok sync ok ok valid -evo 43.167.244.109:9999 443 ok sync ok ok valid +regular 43.155.135.208:9999 - ok sync - - - +regular 43.155.187.252:9999 - ok sync - - - +regular 43.155.210.82:9999 - ok sync - - - +regular 43.155.210.116:9999 - ok sync - - - +regular 43.155.219.71:9999 - ok sync - - - +regular 43.155.229.96:9999 - ok sync - - - +regular 43.155.239.145:9999 - ok sync - - - +regular 43.155.239.243:9999 - ok sync - - - +regular 43.166.2.174:9999 - ok sync - - - +evo 43.167.239.145:9999 443 ok sync ok ok expired +evo 43.167.244.109:9999 443 ok sync ok ok expired regular 43.224.35.163:9999 - error ? - - - regular 43.229.77.46:9999 - ok sync - - - evo 44.240.99.214:9999 443 ok sync ok ok valid @@ -293,7 +316,6 @@ regular 45.58.56.82:9999 - ok sync - - - regular 45.58.56.87:9999 - ok sync - - - regular 45.61.186.121:9999 - ok sync - - - regular 45.63.21.44:9999 - ok sync - - - -regular 45.63.116.79:9999 - ok sync - - - regular 45.63.120.66:9999 - ok sync - - - regular 45.76.29.241:9999 - ok sync - - - evo 45.76.36.241:9999 443 ok sync ok ok valid @@ -302,12 +324,12 @@ regular 45.76.64.242:9999 - ok sync - - - regular 45.76.84.108:9999 - ok sync - - - regular 45.76.111.111:9999 - ok sync - - - regular 45.76.116.16:9999 - error ? - - - -evo 45.76.141.74:9999 443 ok sync ok ok valid +evo 45.76.141.74:9999 443 ok sync ok ok expired regular 45.76.205.140:9999 - ok sync - - - regular 45.76.236.39:9999 - ok sync - - - -evo 45.77.11.194:9999 443 ok sync ok ok valid +evo 45.77.11.194:9999 443 ok sync ok ok expired regular 45.77.15.33:9999 - ok sync - - - -evo 45.77.129.235:9999 443 ok sync ok ok valid +evo 45.77.129.235:9999 443 ok sync ok ok expired regular 45.77.171.78:9999 - ok sync - - - regular 45.79.18.106:9999 - ok sync - - - evo 45.85.147.192:9999 443 ok sync ok ok valid @@ -315,7 +337,7 @@ evo 45.94.58.58:9999 443 ok sync ok ok valid regular 45.128.156.30:9999 - ok sync - - - regular 45.135.180.68:9999 - ok sync - - - evo 45.135.180.70:9999 443 ok sync ok ok valid -evo 45.135.180.79:9999 443 ok sync ok ok valid +evo 45.135.180.79:9999 443 ok sync ok ok expired evo 45.135.180.114:9999 443 ok sync ok ok valid evo 45.135.180.130:9999 443 ok sync ok ok valid regular 45.135.180.221:9999 - ok sync - - - @@ -341,9 +363,9 @@ regular 46.4.162.116:9999 - ok sync - - - regular 46.4.162.117:9999 - ok sync - - - regular 46.4.162.118:9999 - ok sync - - - regular 46.4.162.119:9999 - ok sync - - - +regular 46.4.162.121:9999 - ok sync - - - regular 46.4.162.122:9999 - ok sync - - - regular 46.4.162.124:9999 - ok sync - - - -regular 46.4.162.125:9999 - ok sync - - - regular 46.4.162.127:9999 - ok sync - - - regular 46.4.217.227:9999 - ok sync - - - regular 46.4.217.229:9999 - ok sync - - - @@ -359,14 +381,13 @@ regular 46.4.217.242:9999 - ok sync - - - regular 46.4.217.244:9999 - ok sync - - - regular 46.4.217.245:9999 - ok sync - - - regular 46.4.217.246:9999 - ok sync - - - +regular 46.4.217.249:9999 - ok sync - - - regular 46.4.217.250:9999 - ok sync - - - regular 46.4.217.251:9999 - ok sync - - - regular 46.4.217.252:9999 - ok sync - - - regular 46.4.217.254:9999 - ok sync - - - -regular 46.10.241.191:9999 - error ? - - - -regular 46.30.189.187:9999 - ok sync - - - -regular 46.30.189.213:9999 - ok sync - - - regular 46.30.189.251:9999 - ok sync - - - +regular 46.36.35.244:9999 - ok sync - - - regular 46.36.40.242:9999 - ok sync - - - regular 46.62.132.144:9999 - ok sync - - - regular 46.62.245.210:9999 - ok sync - - - @@ -375,20 +396,19 @@ regular 46.101.37.234:9999 - ok sync - - - regular 46.101.60.13:9999 - ok sync - - - regular 46.101.84.163:9999 - ok sync - - - regular 46.101.108.68:9999 - ok sync - - - -regular 46.101.110.82:9999 - ok sync - - - regular 46.101.118.96:9999 - ok sync - - - regular 46.101.129.63:9999 - ok sync - - - -regular 46.101.195.230:9999 - ok sync - - - regular 46.101.240.34:9999 - ok sync - - - regular 46.148.231.121:9999 - ok sync - - - regular 46.224.50.151:9999 - ok sync - - - +regular 46.225.120.51:9999 - ok sync - - - regular 46.250.254.10:9999 - ok sync - - - regular 49.12.1.250:9999 - ok sync - - - -evo 49.12.102.105:9999 443 ok sync ok ok valid +evo 49.12.102.105:9999 443 ok sync ok ok expired regular 49.12.189.12:9999 - ok sync - - - regular 49.12.198.203:9999 - ok sync - - - regular 49.13.27.107:9999 - ok sync - - - -evo 49.13.28.255:9999 443 ok sync ok ok valid +evo 49.13.28.255:9999 443 ok sync ok ok expired regular 49.13.140.167:9999 - ok sync - - - evo 49.13.154.121:9999 443 ok sync ok ok expired evo 49.13.193.251:9999 443 ok sync ok ok valid @@ -396,56 +416,50 @@ regular 49.13.197.215:9999 - ok sync - - - regular 49.13.209.157:9999 - ok sync - - - regular 49.13.212.202:9999 - ok sync - - - regular 49.13.212.210:9999 - ok sync - - - -evo 49.13.237.193:9999 443 ok sync ok ok valid +evo 49.13.237.193:9999 443 ok sync ok ok expired +evo 50.116.28.103:9999 443 ok sync ok ok expired regular 51.38.27.202:9999 - ok sync - - - evo 51.68.3.29:9999 443 ok sync ok ok valid evo 51.68.35.98:9999 443 ok sync ok ok valid evo 51.77.216.185:9999 443 ok sync ok ok expired -evo 51.79.109.182:9999 443 ok sync ok ok expired +evo 51.79.109.182:9999 443 ok sync ok ok valid regular 51.158.167.31:9999 - ok sync - - - evo 51.158.169.237:9999 443 ok sync ok ok valid regular 51.158.188.88:9999 - ok sync - - - evo 51.161.34.134:9999 443 ok sync ok ok expired -evo 51.195.235.166:9999 443 ok sync ok ok valid +evo 51.195.235.166:9999 443 ok sync ok ok expired evo 52.10.213.198:9999 443 ok sync ok ok valid evo 52.33.9.172:9999 443 ok sync ok ok valid evo 52.36.102.91:9999 443 ok sync ok ok valid regular 54.37.38.226:9999 - ok sync - - - evo 54.69.95.118:9999 443 ok sync ok ok valid evo 57.128.212.163:9999 443 ok sync ok ok valid -evo 57.131.28.197:9999 443 ok sync ok ok valid +evo 57.131.28.197:9999 443 ok sync ok ok expired regular 62.60.244.174:9999 - ok sync - - - -evo 62.171.133.125:9999 443 ok sync ok ok expired -evo 62.171.136.93:9999 443 ok sync ok ok valid evo 62.171.136.245:9999 443 ok sync ok ok valid evo 62.171.138.186:9999 443 ok sync ok ok valid evo 62.171.144.159:9999 443 ok sync ok ok expired evo 62.171.144.192:9999 443 ok sync ok ok valid -evo 62.171.145.24:9999 443 ok sync ok ok valid evo 62.171.145.234:9999 443 ok sync ok ok valid regular 62.171.149.86:9999 - ok sync - - - evo 62.171.150.24:9999 443 ok sync ok ok valid evo 62.171.168.4:9999 443 ok sync ok ok valid evo 62.171.168.44:9999 443 ok sync ok ok valid -evo 62.171.170.14:9999 443 ok sync ok ok valid -evo 62.171.170.222:9999 443 ok sync ok ok valid evo 62.171.171.253:9999 443 ok sync ok ok valid evo 62.171.174.123:9999 443 ok sync ok ok valid evo 62.171.176.79:9999 443 ok sync ok ok valid -regular 62.171.177.236:9999 - ok sync - - - evo 62.171.178.71:9999 443 ok sync ok ok valid evo 62.171.189.0:9999 443 ok sync ok ok valid -regular 62.171.189.217:9999 - ok sync - - - +evo 63.186.97.17:9999 443 ok sync ok ok valid evo 64.176.10.71:9999 443 ok sync ok ok valid evo 64.176.35.235:9999 443 ok sync ok ok valid regular 64.225.96.193:9999 - ok sync - - - regular 64.226.66.91:9999 - ok sync - - - regular 64.226.81.185:9999 - ok sync - - - regular 64.227.32.83:9999 - ok sync - - - -regular 64.227.122.247:9999 - ok sync - - - +regular 64.227.35.91:9999 - ok sync - - - regular 64.227.139.86:9999 - ok sync - - - regular 64.227.165.75:9999 - ok sync - - - -regular 65.20.101.115:9999 - ok sync - - - regular 65.21.183.4:9999 - ok sync - - - evo 65.21.237.222:9999 443 ok sync ok ok expired evo 65.21.237.223:9999 443 ok sync ok ok expired @@ -466,13 +480,12 @@ evo 65.108.74.78:9999 443 ok sync refused none no-handshake evo 65.108.74.79:9999 443 ok sync ok ok expired evo 65.108.74.95:9999 443 ok sync ok ok expired evo 65.108.74.109:9999 443 ok sync ok ok expired -regular 65.108.145.8:9999 - ok sync - - - -regular 65.108.150.87:9999 - ok sync - - - regular 65.108.204.190:9999 - ok sync - - - regular 65.108.221.24:9999 - ok sync - - - evo 65.108.246.145:9999 443 ok sync ok ok valid -evo 65.109.84.201:9999 443 ok sync ok ok valid -evo 65.109.84.203:9999 443 ok sync ok ok valid +evo 65.109.84.201:9999 443 ok sync ok ok expired +evo 65.109.84.202:9999 443 ok sync ok ok valid +evo 65.109.84.203:9999 443 ok sync ok ok expired evo 65.109.84.204:9999 443 ok sync ok ok expired regular 65.109.93.110:9999 - ok sync - - - evo 65.109.108.138:9999 443 ok sync ok ok expired @@ -495,13 +508,13 @@ regular 66.244.243.70:9999 - ok sync - - - evo 66.245.196.52:9999 443 ok sync ok ok valid regular 68.183.2.66:9999 - ok sync - - - regular 68.183.36.87:9999 - ok sync - - - +regular 68.183.46.92:9999 - ok sync - - - regular 68.183.75.121:9999 - ok sync - - - regular 68.183.80.227:9999 - ok sync - - - regular 68.183.92.68:9999 - ok sync - - - regular 68.183.188.215:9999 - ok sync - - - regular 68.183.192.117:9999 - ok sync - - - regular 68.183.199.77:9999 - ok sync - - - -regular 68.183.201.176:9999 - ok sync - - - regular 68.183.207.246:9999 - ok sync - - - regular 68.183.208.90:9999 - ok sync - - - regular 68.183.235.156:9999 - ok sync - - - @@ -513,53 +526,59 @@ evo 72.60.38.160:9999 443 ok sync ok ok valid evo 72.62.58.108:9999 443 ok sync ok ok valid regular 74.50.90.113:9999 - ok sync - - - evo 75.119.128.71:9999 443 ok sync ok ok valid -regular 75.119.134.20:9999 - ok sync - - - evo 75.119.137.66:9999 443 ok sync ok ok valid evo 75.119.138.9:9999 443 ok sync ok ok valid evo 75.119.152.219:9999 443 ok sync ok ok valid evo 75.119.153.10:9999 443 ok sync ok ok valid evo 75.119.156.254:9999 443 ok sync ok ok valid +evo 77.20.160.6:9999 443 ok sync ok ok valid regular 77.42.16.46:9999 - ok sync - - - regular 77.42.65.17:9999 - ok sync - - - -evo 77.42.74.68:9999 443 ok sync ok ok valid +evo 77.42.74.68:9999 443 ok sync ok ok expired regular 77.42.91.72:9999 - ok sync - - - +regular 77.105.61.79:9999 - ok sync - - - +regular 77.105.61.119:9999 - ok sync - - - +regular 77.105.61.188:9999 - ok sync - - - +regular 77.105.61.212:9999 - ok sync - - - evo 77.221.148.204:9999 443 ok sync ok ok valid regular 77.223.99.4:9999 - ok sync - - - regular 77.237.235.0:9999 - ok sync - - - regular 77.237.236.66:9999 - ok sync - - - regular 77.237.236.214:9999 - ok sync - - - -regular 77.237.244.217:9999 - ok sync - - - -regular 77.237.245.19:9999 - ok sync - - - +regular 78.17.71.40:9999 - ok sync - - - +regular 78.17.212.249:9999 - ok sync - - - regular 78.47.119.20:9999 - ok sync - - - -regular 78.83.19.0:9999 - ok sync - - - +regular 78.83.19.0:9999 - error ? - - - evo 78.141.225.100:9999 443 ok sync ok ok valid -regular 78.141.240.214:9999 - ok sync - - - regular 79.143.181.108:9999 - ok sync - - - -regular 80.208.230.144:9999 - ok sync - - - regular 80.209.233.182:9999 - ok sync - - - regular 80.209.239.129:9999 - ok sync - - - evo 80.240.19.200:9999 443 ok sync ok ok valid regular 80.240.132.231:9999 - ok sync - - - regular 80.240.135.83:9999 - ok sync - - - regular 80.249.147.8:9999 - ok sync - - - +evo 81.14.228.87:9999 443 ok sync ok ok valid regular 81.227.250.51:9999 - ok sync - - - regular 82.180.146.192:9999 - ok sync - - - regular 82.196.1.10:9999 - ok sync - - - regular 82.202.230.83:9999 - ok sync - - - evo 82.208.20.153:9999 443 ok sync ok ok valid +regular 82.223.81.127:9999 - ok sync - - - +regular 83.149.100.184:9999 - ok sync - - - +regular 83.149.100.186:9999 - ok sync - - - regular 83.239.99.40:9999 - ok sync - - - regular 84.9.50.17:9999 - ok sync - - - +evo 84.32.22.20:9999 443 ok sync ok ok valid +evo 84.32.22.70:9999 443 ok sync ok ok valid +evo 84.32.22.141:9999 443 ok sync ok ok valid +evo 84.32.22.199:9999 443 ok sync ok ok valid regular 84.98.12.196:9999 - ok sync - - - regular 84.242.179.204:9999 - ok sync - - - -regular 84.247.132.230:9999 - error ? - - - regular 84.247.137.20:9999 - ok sync - - - evo 84.247.180.190:9999 443 ok sync ok ok valid evo 84.247.180.198:9999 443 ok sync ok ok valid -evo 84.247.180.200:9999 443 ok sync ok ok valid evo 84.247.180.201:9999 443 ok sync ok ok valid -evo 84.247.180.205:9999 443 ok sync ok ok valid evo 84.247.187.76:9999 443 ok sync ok ok valid -regular 85.155.188.138:9999 - ok sync - - - evo 85.190.243.3:9999 443 ok sync ok ok valid regular 85.209.241.2:9999 - ok sync - - - regular 85.209.241.3:9999 - ok sync - - - @@ -580,7 +599,6 @@ regular 85.209.241.22:9999 - ok sync - - - regular 85.209.241.23:9999 - ok sync - - - regular 85.209.241.24:9999 - ok sync - - - regular 85.209.241.25:9999 - ok sync - - - -regular 85.209.241.26:9999 - ok sync - - - regular 85.209.241.27:9999 - ok sync - - - regular 85.209.241.28:9999 - ok sync - - - regular 85.209.241.30:9999 - ok sync - - - @@ -622,21 +640,21 @@ regular 85.209.241.85:9999 - ok sync - - - regular 85.209.241.86:9999 - ok sync - - - regular 85.209.241.87:9999 - ok sync - - - regular 85.209.241.88:9999 - ok sync - - - -regular 85.209.241.89:9999 - ok sync - - - regular 85.209.241.91:9999 - ok sync - - - -regular 85.209.241.92:9999 - ok sync - - - regular 85.209.241.93:9999 - ok sync - - - regular 85.209.241.94:9999 - ok sync - - - regular 85.209.241.102:9999 - ok sync - - - regular 85.209.241.103:9999 - ok sync - - - regular 85.209.241.104:9999 - ok sync - - - regular 85.209.241.105:9999 - ok sync - - - -regular 85.209.241.121:9999 - ok sync - - - -regular 85.209.241.127:9999 - ok sync - - - +regular 85.209.241.106:9999 - ok sync - - - +regular 85.209.241.107:9999 - ok sync - - - regular 85.209.241.146:9999 - ok sync - - - regular 85.209.241.150:9999 - ok sync - - - regular 85.209.241.156:9999 - ok sync - - - regular 85.209.241.164:9999 - ok sync - - - +regular 85.209.241.165:9999 - ok sync - - - +regular 85.209.241.166:9999 - ok sync - - - regular 85.209.241.167:9999 - ok sync - - - regular 85.209.241.172:9999 - ok sync - - - regular 85.209.241.177:9999 - ok sync - - - @@ -655,17 +673,18 @@ regular 85.209.241.215:9999 - ok sync - - - regular 85.209.241.216:9999 - ok sync - - - regular 85.209.241.217:9999 - ok sync - - - regular 85.209.241.218:9999 - ok sync - - - -regular 85.209.241.219:9999 - ok sync - - - regular 85.209.241.220:9999 - ok sync - - - regular 85.209.241.221:9999 - ok sync - - - +regular 85.209.241.222:9999 - ok sync - - - regular 85.209.241.223:9999 - ok sync - - - regular 85.209.241.224:9999 - ok sync - - - regular 85.209.241.225:9999 - ok sync - - - regular 85.209.241.226:9999 - ok sync - - - regular 85.209.241.228:9999 - ok sync - - - regular 85.209.241.229:9999 - ok sync - - - +regular 85.209.241.230:9999 - ok sync - - - regular 85.209.241.231:9999 - ok sync - - - -regular 85.209.241.233:9999 - ok sync - - - +regular 85.209.241.232:9999 - ok sync - - - regular 85.209.241.234:9999 - ok sync - - - regular 85.209.241.235:9999 - ok sync - - - regular 85.209.241.236:9999 - ok sync - - - @@ -681,6 +700,9 @@ regular 85.209.241.247:9999 - ok sync - - - regular 85.209.241.248:9999 - ok sync - - - regular 85.209.241.249:9999 - ok sync - - - regular 85.209.242.250:9999 - ok sync - - - +regular 86.107.168.40:9999 - ok sync - - - +evo 86.107.168.78:9999 443 ok sync ok ok valid +regular 86.107.168.98:9999 - ok sync - - - evo 87.228.24.64:9999 443 ok sync ok ok valid regular 87.229.22.23:9999 - ok sync - - - regular 87.229.22.31:9999 - ok sync - - - @@ -692,15 +714,11 @@ regular 88.99.11.2:9999 - ok sync - - - regular 88.99.11.3:9999 - ok sync - - - regular 88.99.11.7:9999 - ok sync - - - regular 88.99.11.10:9999 - ok sync - - - -regular 88.99.11.12:9999 - ok sync - - - regular 88.99.11.14:9999 - ok sync - - - -regular 88.99.11.15:9999 - ok sync - - - regular 88.99.11.16:9999 - ok sync - - - regular 88.99.11.17:9999 - ok sync - - - -regular 88.99.11.20:9999 - ok sync - - - regular 88.99.11.21:9999 - ok sync - - - regular 88.99.11.22:9999 - ok sync - - - -regular 88.99.11.23:9999 - ok sync - - - regular 88.99.11.24:9999 - ok sync - - - regular 88.99.11.25:9999 - ok sync - - - regular 88.99.11.26:9999 - ok sync - - - @@ -708,22 +726,24 @@ regular 88.99.11.27:9999 - ok sync - - - regular 88.99.11.29:9999 - ok sync - - - regular 88.99.11.30:9999 - ok sync - - - regular 88.99.11.31:9999 - ok sync - - - -regular 88.99.38.175:9999 - ok sync - - - +regular 88.99.38.175:9999 - error ? - - - +evo 88.216.191.0:9999 443 ok sync ok ok valid +evo 88.216.191.113:9999 443 ok sync ok ok valid +evo 88.216.191.114:9999 443 ok sync ok ok valid +evo 88.216.191.115:9999 443 ok sync ok ok valid evo 89.35.131.23:9999 443 ok sync ok ok valid evo 89.35.131.39:9999 443 ok sync ok ok valid regular 89.35.131.46:9999 - ok sync - - - -evo 89.35.131.61:9999 443 ok sync ok ok expired -evo 89.35.131.149:9999 443 ok sync ok ok valid -evo 89.35.131.158:9999 443 ok sync ok ok valid -evo 89.35.131.218:9999 443 error ? ok ok valid -evo 89.35.131.219:9999 443 ok sync ok ok valid +evo 89.35.131.149:9999 443 ok sync ok ok expired +evo 89.35.131.158:9999 443 ok sync ok ok expired +evo 89.35.131.218:9999 443 ok sync ok ok expired +evo 89.35.131.219:9999 443 ok sync ok ok expired regular 89.40.15.222:9999 - ok sync - - - regular 89.46.42.216:9999 - ok sync - - - regular 89.46.42.218:9999 - ok sync - - - regular 89.47.167.131:9999 - ok sync - - - regular 89.117.3.242:9999 - ok sync - - - -evo 89.117.57.27:9999 443 ok sync ok ok expired -evo 89.125.50.14:9999 443 ok sync ok ok expired +evo 89.125.50.14:9999 443 ok sync ok ok valid evo 89.125.50.206:9999 443 ok sync ok ok valid evo 89.125.209.27:9999 443 ok sync ok ok valid evo 89.125.209.69:9999 443 ok sync ok ok valid @@ -735,13 +755,20 @@ evo 89.125.209.152:9999 443 ok sync ok ok valid evo 89.125.209.156:9999 443 ok sync ok ok valid evo 89.125.209.178:9999 443 ok sync ok ok valid evo 89.125.209.195:9999 443 ok sync ok ok valid +regular 89.125.209.226:9999 - ok sync - - - evo 89.125.209.234:9999 443 ok sync ok ok valid +regular 89.125.255.211:9999 - ok sync - - - regular 89.179.73.96:9999 - ok sync - - - regular 91.98.160.22:9999 - ok sync - - - regular 91.99.169.186:9999 - ok sync - - - +regular 91.184.249.209:9999 - ok sync - - - evo 91.198.108.35:9999 443 ok sync ok ok valid evo 91.198.108.36:9999 443 ok sync ok ok valid +evo 91.198.108.37:9999 443 ok sync ok ok valid +evo 91.198.108.38:9999 443 ok sync ok ok valid +regular 91.198.108.39:9999 - ok sync - - - regular 91.198.108.40:9999 - ok sync - - - +regular 91.198.108.41:9999 - ok sync - - - evo 91.199.149.177:9999 443 ok sync ok ok expired evo 93.115.172.36:9999 443 ok sync ok ok valid evo 93.115.172.37:9999 443 ok sync ok ok valid @@ -762,7 +789,6 @@ regular 95.85.48.49:9999 - ok sync - - - regular 95.85.54.64:9999 - ok sync - - - evo 95.111.232.18:9999 443 ok sync ok ok valid evo 95.111.233.139:9999 443 ok sync ok ok valid -evo 95.111.239.54:9999 443 ok sync ok ok valid evo 95.111.241.20:9999 443 ok sync ok ok valid evo 95.111.242.220:9999 443 ok sync ok ok valid evo 95.179.139.125:9999 443 ok sync ok ok expired @@ -770,39 +796,33 @@ regular 95.179.152.242:9999 - ok sync - - - evo 95.179.159.65:9999 443 ok sync ok ok valid regular 95.179.213.67:9999 - ok sync - - - regular 95.179.240.114:9999 - ok sync - - - -evo 95.179.241.182:9999 443 ok sync ok ok valid -regular 95.179.243.76:9999 - ok sync - - - +evo 95.179.241.182:9999 443 ok sync ok ok expired regular 95.216.79.224:9999 - ok sync - - - -regular 95.216.79.228:9999 - ok sync - - - regular 95.216.84.33:9999 - ok sync - - - regular 95.216.84.35:9999 - ok sync - - - regular 95.216.84.36:9999 - ok sync - - - regular 95.216.84.39:9999 - ok sync - - - regular 95.216.84.41:9999 - ok sync - - - -regular 95.216.84.42:9999 - ok sync - - - regular 95.216.84.43:9999 - ok sync - - - regular 95.216.84.45:9999 - ok sync - - - regular 95.216.109.130:9999 - ok sync - - - regular 95.216.109.131:9999 - ok sync - - - -regular 95.216.109.133:9999 - ok sync - - - regular 95.216.109.134:9999 - ok sync - - - regular 95.216.109.135:9999 - ok sync - - - regular 95.216.109.136:9999 - ok sync - - - -regular 95.216.126.32:9999 - ok sync - - - regular 95.216.126.34:9999 - ok sync - - - regular 95.216.126.38:9999 - ok sync - - - regular 95.216.126.39:9999 - ok sync - - - +regular 95.216.126.43:9999 - ok sync - - - regular 95.216.126.45:9999 - ok sync - - - -regular 95.216.126.46:9999 - ok sync - - - -evo 95.216.146.18:9999 443 ok sync ok ok valid +evo 95.216.146.18:9999 443 ok sync ok ok expired +regular 95.216.223.223:9999 - ok sync - - - regular 95.216.230.97:9999 - ok sync - - - regular 95.216.230.98:9999 - ok sync - - - regular 95.216.230.101:9999 - ok sync - - - regular 95.216.230.102:9999 - ok sync - - - regular 95.216.230.103:9999 - ok sync - - - regular 95.216.255.64:9999 - ok sync - - - -regular 95.216.255.67:9999 - ok sync - - - -regular 95.216.255.68:9999 - ok sync - - - regular 95.216.255.69:9999 - ok sync - - - regular 95.216.255.70:9999 - ok sync - - - regular 95.216.255.71:9999 - ok sync - - - @@ -821,10 +841,6 @@ regular 95.217.71.207:9999 - ok sync - - - regular 95.217.71.209:9999 - ok sync - - - regular 95.217.71.210:9999 - ok sync - - - regular 95.217.71.211:9999 - ok sync - - - -evo 95.217.83.2:9999 443 ok sync ok ok valid -regular 95.217.83.5:9999 - ok sync - - - -evo 95.217.83.14:9999 443 ok sync ok ok valid -regular 95.217.125.97:9999 - ok sync - - - regular 95.217.125.98:9999 - ok sync - - - regular 95.217.125.101:9999 - ok sync - - - regular 95.217.125.103:9999 - ok sync - - - @@ -849,19 +865,18 @@ regular 104.168.46.54:9999 - ok sync - - - regular 104.207.130.246:9999 - ok sync - - - regular 104.207.130.255:9999 - ok sync - - - regular 104.207.131.17:9999 - error ? - - - -regular 104.234.167.115:9999 - ok sync - - - regular 104.236.58.131:9999 - ok sync - - - regular 104.236.183.29:9999 - ok sync - - - regular 104.238.159.13:9999 - ok sync - - - regular 104.238.167.102:9999 - ok sync - - - regular 104.238.176.125:9999 - ok sync - - - regular 104.238.176.179:9999 - ok sync - - - +regular 104.238.179.122:9999 - ok sync - - - regular 104.248.40.20:9999 - ok sync - - - regular 104.248.46.23:9999 - ok sync - - - regular 104.248.137.183:9999 - ok sync - - - regular 104.248.138.204:9999 - ok sync - - - regular 104.248.159.169:9999 - ok sync - - - -regular 104.248.167.243:9999 - ok sync - - - regular 104.248.172.106:9999 - ok sync - - - regular 106.55.9.22:9999 - ok sync - - - regular 107.170.7.146:9999 - ok sync - - - @@ -890,26 +905,23 @@ regular 108.160.131.192:9999 - ok sync - - - regular 108.160.134.116:9999 - ok sync - - - regular 108.160.140.103:9999 - ok sync - - - evo 109.123.244.131:9999 443 ok sync ok ok valid -regular 109.173.240.233:9999 - ok sync - - - -regular 109.199.96.61:9999 - ok sync - - - +evo 109.173.198.67:9999 443 ok sync ok ok valid evo 109.199.97.144:9999 443 ok sync ok ok valid evo 109.199.105.14:9999 443 ok sync ok ok valid regular 109.199.105.203:9999 - ok sync - - - +regular 109.199.106.117:9999 - ok sync - - - evo 109.199.107.155:9999 443 ok sync ok ok valid -evo 109.199.108.223:9999 443 ok sync ok ok valid evo 109.199.108.234:9999 443 ok sync ok ok valid evo 109.199.109.39:9999 443 ok sync ok ok valid -evo 109.199.119.192:9999 443 ok sync ok ok valid +regular 109.199.111.47:9999 - ok sync - - - evo 109.199.120.79:9999 443 ok sync ok ok valid evo 109.199.124.30:9999 443 ok sync ok ok valid evo 109.199.124.135:9999 443 ok sync ok ok valid regular 109.235.67.246:9999 - ok sync - - - evo 114.132.172.215:9999 443 ok sync ok ok valid -regular 115.120.238.240:9999 - error ? - - - -regular 115.190.248.57:9999 - error ? - - - +regular 115.190.248.57:9999 - ok sync - - - regular 116.203.244.222:9999 - ok sync - - - -regular 123.193.1.112:9999 - ok sync - - - -regular 128.140.7.186:9999 - ok sync - - - +regular 123.193.64.166:9999 - ok sync - - - regular 128.140.58.30:9999 - ok sync - - - regular 128.199.17.79:9999 - ok sync - - - regular 128.199.22.78:9999 - ok sync - - - @@ -921,7 +933,7 @@ regular 128.199.134.235:9999 - ok sync - - - regular 128.199.137.10:9999 - ok sync - - - regular 128.199.232.212:9999 - ok sync - - - regular 129.212.233.191:9999 - ok sync - - - -evo 130.162.233.186:9999 443 ok sync ok ok valid +evo 130.162.233.186:9999 443 ok sync ok ok expired evo 130.185.118.3:9999 443 ok sync ok ok valid regular 130.255.155.163:9999 - ok sync - - - regular 132.243.197.2:9999 - ok sync - - - @@ -1034,6 +1046,7 @@ regular 132.243.197.115:9999 - ok sync - - - regular 132.243.197.116:9999 - ok sync - - - regular 132.243.197.117:9999 - ok sync - - - regular 132.243.197.118:9999 - ok sync - - - +regular 132.243.197.119:9999 - ok sync - - - regular 132.243.197.127:9999 - ok sync - - - regular 132.243.197.130:9999 - ok sync - - - regular 132.243.197.131:9999 - ok sync - - - @@ -1110,19 +1123,15 @@ regular 134.209.197.15:9999 - ok sync - - - regular 134.209.207.71:9999 - ok sync - - - evo 134.255.182.185:9999 443 ok sync ok ok valid evo 134.255.182.186:9999 443 ok sync ok ok valid -regular 135.181.8.66:9999 - ok sync - - - regular 135.181.8.68:9999 - ok sync - - - regular 135.181.8.69:9999 - ok sync - - - regular 135.181.8.70:9999 - ok sync - - - -regular 135.181.8.72:9999 - ok sync - - - regular 135.181.8.73:9999 - ok sync - - - regular 135.181.8.74:9999 - ok sync - - - regular 135.181.8.81:9999 - ok sync - - - -regular 135.181.8.82:9999 - ok sync - - - regular 135.181.8.83:9999 - ok sync - - - regular 135.181.15.224:9999 - ok sync - - - regular 135.181.15.228:9999 - ok sync - - - -regular 135.181.15.229:9999 - ok sync - - - regular 135.181.15.233:9999 - ok sync - - - regular 135.181.15.234:9999 - ok sync - - - regular 135.181.15.235:9999 - ok sync - - - @@ -1130,6 +1139,7 @@ regular 135.181.50.36:9999 - ok sync - - - regular 135.181.50.38:9999 - ok sync - - - regular 135.181.50.45:9999 - ok sync - - - regular 135.181.52.129:9999 - ok sync - - - +regular 135.181.52.139:9999 - ok sync - - - regular 135.181.52.140:9999 - ok sync - - - regular 135.181.52.145:9999 - ok sync - - - regular 135.181.52.148:9999 - ok sync - - - @@ -1149,7 +1159,6 @@ regular 135.181.53.227:9999 - ok sync - - - regular 135.181.53.228:9999 - ok sync - - - regular 135.181.53.229:9999 - ok sync - - - regular 135.181.53.230:9999 - ok sync - - - -regular 135.181.53.231:9999 - ok sync - - - regular 135.181.54.96:9999 - ok sync - - - regular 135.181.54.98:9999 - ok sync - - - regular 135.181.54.101:9999 - ok sync - - - @@ -1158,8 +1167,8 @@ regular 135.181.54.104:9999 - ok sync - - - evo 135.181.110.216:9999 443 ok sync ok ok valid regular 135.181.174.222:9999 - ok sync - - - regular 135.181.177.53:9999 - ok sync - - - -evo 135.181.229.49:9999 443 ok sync ok ok valid -regular 136.243.29.193:9999 - ok sync - - - +evo 135.181.229.49:9999 443 ok sync ok ok expired +regular 136.243.29.192:9999 - ok sync - - - regular 136.243.29.194:9999 - ok sync - - - regular 136.243.29.195:9999 - ok sync - - - regular 136.243.29.196:9999 - ok sync - - - @@ -1177,17 +1186,16 @@ regular 136.243.29.220:9999 - ok sync - - - regular 136.243.115.128:9999 - ok sync - - - regular 136.243.115.130:9999 - ok sync - - - regular 136.243.115.133:9999 - ok sync - - - -regular 136.243.115.134:9999 - ok sync - - - regular 136.243.115.136:9999 - ok sync - - - regular 136.243.115.138:9999 - ok sync - - - regular 136.243.115.139:9999 - ok sync - - - regular 136.243.115.140:9999 - ok sync - - - regular 136.243.115.141:9999 - ok sync - - - regular 136.243.115.142:9999 - ok sync - - - -regular 136.243.138.106:9999 - ok sync - - - regular 136.243.142.32:9999 - ok sync - - - regular 136.243.142.33:9999 - ok sync - - - regular 136.243.142.36:9999 - ok sync - - - +regular 136.243.142.37:9999 - ok sync - - - regular 136.243.142.41:9999 - ok sync - - - evo 136.244.99.17:9999 443 ok sync ok ok valid regular 136.244.103.110:9999 - ok sync - - - @@ -1195,11 +1203,13 @@ regular 137.184.5.78:9999 - ok sync - - - regular 137.184.175.173:9999 - ok sync - - - regular 137.220.38.255:9999 - error ? - - - regular 138.68.68.160:9999 - ok sync - - - +regular 138.68.146.83:9999 - ok sync - - - regular 138.68.149.239:9999 - ok sync - - - +regular 138.68.155.23:9999 - ok sync - - - regular 138.68.157.15:9999 - ok sync - - - regular 138.197.128.77:9999 - ok sync - - - -regular 138.197.130.116:9999 - ok sync - - - regular 138.197.131.126:9999 - ok sync - - - +regular 138.197.141.70:9999 - ok sync - - - regular 138.197.147.28:9999 - ok sync - - - regular 138.197.161.208:9999 - ok sync - - - regular 139.28.97.2:9999 - ok sync - - - @@ -1211,16 +1221,15 @@ regular 139.59.29.173:9999 - ok sync - - - regular 139.59.30.197:9999 - ok sync - - - regular 139.59.39.142:9999 - ok sync - - - regular 139.59.56.7:9999 - ok sync - - - -regular 139.59.102.0:9999 - ok sync - - - regular 139.59.120.106:9999 - ok sync - - - regular 139.59.137.155:9999 - ok sync - - - regular 139.59.138.149:9999 - ok sync - - - regular 139.59.144.169:9999 - ok sync - - - regular 139.59.156.27:9999 - ok sync - - - regular 139.59.178.169:9999 - ok sync - - - -regular 139.59.179.170:9999 - ok sync - - - regular 139.59.231.57:9999 - ok sync - - - regular 139.59.243.139:9999 - ok sync - - - +regular 139.59.243.194:9999 - ok sync - - - evo 139.84.137.143:9999 443 ok sync ok ok valid evo 139.84.170.10:9999 443 ok sync ok ok valid evo 139.84.231.221:9999 443 ok sync ok ok valid @@ -1239,6 +1248,7 @@ regular 142.93.154.186:9999 - ok sync - - - regular 142.93.157.112:9999 - ok sync - - - regular 142.93.215.115:9999 - ok sync - - - regular 142.93.216.91:9999 - ok sync - - - +regular 143.20.33.10:9999 - ok sync - - - regular 143.110.156.147:9999 - ok sync - - - regular 143.110.183.99:9999 - ok sync - - - regular 143.110.189.48:9999 - ok sync - - - @@ -1252,13 +1262,7 @@ evo 143.198.145.184:9999 443 ok sync ok ok valid regular 143.244.131.179:9999 - ok sync - - - regular 144.24.238.178:9999 - ok sync - - - regular 144.31.188.42:9999 - ok sync - - - -evo 144.91.87.82:9999 443 ok sync ok ok valid -evo 144.91.99.105:9999 443 ok sync ok ok valid -evo 144.91.105.209:9999 443 ok sync ok ok valid -evo 144.91.106.188:9999 443 ok sync ok ok valid -evo 144.91.121.81:9999 443 ok sync ok ok valid evo 144.126.141.62:9999 443 ok sync ok ok valid -regular 144.126.143.197:9999 - ok sync - - - regular 144.126.203.253:9999 - ok sync - - - regular 144.202.19.42:9999 - ok sync - - - regular 144.202.37.1:9999 - error ? - - - @@ -1275,15 +1279,14 @@ regular 146.185.180.40:9999 - ok sync - - - regular 146.185.182.189:9999 - ok sync - - - regular 146.190.33.135:9999 - ok sync - - - regular 147.45.41.210:9999 - ok sync - - - -regular 147.93.131.52:9999 - ok sync - - - -regular 147.93.132.12:9999 - ok sync - - - regular 147.182.146.51:9999 - ok sync - - - +regular 147.182.157.129:9999 - ok sync - - - regular 148.113.202.169:9999 - ok sync - - - -regular 148.113.209.97:9999 - ok sync - - - regular 148.251.73.224:9999 - ok sync - - - regular 148.251.73.232:9999 - ok sync - - - regular 148.251.73.234:9999 - ok sync - - - regular 148.251.73.235:9999 - ok sync - - - +regular 148.251.73.237:9999 - ok sync - - - regular 148.251.73.238:9999 - ok sync - - - regular 149.28.85.41:9999 - ok sync - - - regular 149.28.112.102:9999 - error ? - - - @@ -1294,51 +1297,95 @@ evo 149.28.241.190:9999 443 ok sync ok ok expired evo 149.28.247.165:9999 443 ok sync ok ok expired evo 149.102.140.101:9999 443 ok sync ok ok valid regular 149.104.110.119:9999 - ok sync - - - -regular 149.248.3.156:9999 - ok sync - - - regular 149.248.53.48:9999 - ok sync - - - regular 151.244.85.0:9999 - ok sync - - - regular 151.244.85.1:9999 - ok sync - - - regular 151.244.85.2:9999 - ok sync - - - +regular 151.244.85.4:9999 - ok sync - - - regular 151.244.85.7:9999 - ok sync - - - regular 151.244.85.8:9999 - ok sync - - - regular 151.244.85.9:9999 - ok sync - - - +regular 151.244.85.14:9999 - ok sync - - - regular 151.244.85.15:9999 - ok sync - - - +regular 151.244.85.17:9999 - ok sync - - - +regular 151.244.85.21:9999 - ok sync - - - +regular 151.244.85.23:9999 - ok sync - - - +regular 151.244.85.25:9999 - ok sync - - - +regular 151.244.85.29:9999 - ok sync - - - regular 151.244.85.31:9999 - ok sync - - - +regular 151.244.85.33:9999 - ok sync - - - +regular 151.244.85.34:9999 - ok sync - - - regular 151.244.85.36:9999 - ok sync - - - +regular 151.244.85.43:9999 - ok sync - - - regular 151.244.85.45:9999 - ok sync - - - +regular 151.244.85.46:9999 - ok sync - - - regular 151.244.85.47:9999 - ok sync - - - regular 151.244.85.48:9999 - ok sync - - - +regular 151.244.85.50:9999 - ok sync - - - +regular 151.244.85.52:9999 - ok sync - - - +regular 151.244.85.54:9999 - ok sync - - - +regular 151.244.85.58:9999 - ok sync - - - regular 151.244.85.64:9999 - ok sync - - - regular 151.244.85.65:9999 - ok sync - - - regular 151.244.85.67:9999 - ok sync - - - -regular 151.244.85.68:9999 - ok sync - - - regular 151.244.85.69:9999 - ok sync - - - regular 151.244.85.70:9999 - ok sync - - - +regular 151.244.85.71:9999 - ok sync - - - regular 151.244.85.76:9999 - ok sync - - - +regular 151.244.85.77:9999 - ok sync - - - regular 151.244.85.80:9999 - ok sync - - - regular 151.244.85.81:9999 - ok sync - - - +regular 151.244.85.82:9999 - ok sync - - - +regular 151.244.85.85:9999 - ok sync - - - +regular 151.244.85.86:9999 - ok sync - - - regular 151.244.85.88:9999 - ok sync - - - +regular 151.244.85.89:9999 - ok sync - - - regular 151.244.85.96:9999 - ok sync - - - regular 151.244.85.98:9999 - ok sync - - - +regular 151.244.85.100:9999 - ok sync - - - +regular 151.244.85.106:9999 - ok sync - - - regular 151.244.85.109:9999 - ok sync - - - +regular 151.244.85.110:9999 - ok sync - - - regular 151.244.85.111:9999 - ok sync - - - +regular 151.244.85.114:9999 - ok sync - - - +regular 151.244.85.120:9999 - ok sync - - - +regular 151.244.85.121:9999 - ok sync - - - regular 151.244.85.123:9999 - ok sync - - - +regular 151.244.85.130:9999 - ok sync - - - regular 151.244.85.131:9999 - ok sync - - - regular 151.244.85.134:9999 - ok sync - - - regular 151.244.85.135:9999 - ok sync - - - +regular 151.244.85.136:9999 - ok sync - - - +regular 151.244.85.137:9999 - ok sync - - - regular 151.244.85.141:9999 - ok sync - - - +regular 151.244.85.142:9999 - ok sync - - - regular 151.244.85.143:9999 - ok sync - - - +regular 151.244.85.146:9999 - ok sync - - - regular 151.244.85.147:9999 - ok sync - - - +regular 151.244.85.150:9999 - ok sync - - - +regular 151.244.85.151:9999 - ok sync - - - +regular 151.244.85.153:9999 - ok sync - - - +regular 151.244.85.155:9999 - ok sync - - - +regular 151.244.85.162:9999 - ok sync - - - +regular 151.244.85.165:9999 - ok sync - - - regular 151.244.85.166:9999 - ok sync - - - +regular 151.244.85.169:9999 - ok sync - - - +regular 151.244.85.173:9999 - ok sync - - - +regular 151.244.85.177:9999 - ok sync - - - regular 151.244.85.178:9999 - ok sync - - - regular 151.244.85.179:9999 - ok sync - - - regular 151.244.85.181:9999 - ok sync - - - +regular 151.244.85.183:9999 - ok sync - - - +regular 151.244.85.186:9999 - ok sync - - - regular 151.244.85.187:9999 - ok sync - - - +regular 151.244.85.189:9999 - ok sync - - - regular 151.244.85.190:9999 - ok sync - - - regular 151.244.85.195:9999 - ok sync - - - regular 151.244.85.197:9999 - ok sync - - - +regular 151.244.85.200:9999 - ok sync - - - regular 151.244.85.203:9999 - ok sync - - - regular 151.244.85.205:9999 - ok sync - - - +regular 151.244.85.209:9999 - ok sync - - - regular 151.244.85.212:9999 - ok sync - - - regular 151.244.85.213:9999 - ok sync - - - regular 151.244.85.214:9999 - ok sync - - - @@ -1349,15 +1396,22 @@ regular 151.244.85.220:9999 - ok sync - - - regular 151.244.85.228:9999 - ok sync - - - regular 151.244.85.229:9999 - ok sync - - - regular 151.244.85.236:9999 - ok sync - - - +regular 151.244.85.238:9999 - ok sync - - - +regular 151.244.85.239:9999 - ok sync - - - +regular 151.244.85.243:9999 - ok sync - - - regular 151.244.85.244:9999 - ok sync - - - regular 151.244.85.245:9999 - ok sync - - - +regular 151.244.85.250:9999 - ok sync - - - regular 151.244.85.251:9999 - ok sync - - - +regular 151.244.85.252:9999 - ok sync - - - +regular 151.244.85.253:9999 - ok sync - - - regular 151.244.85.255:9999 - ok sync - - - regular 152.42.253.148:9999 - ok sync - - - regular 152.67.95.184:9999 - ok sync - - - regular 154.16.63.112:9999 - ok sync - - - evo 155.133.26.122:9999 443 ok sync ok ok valid regular 155.138.154.140:9999 - error ? - - - +evo 155.138.159.73:9999 443 ok sync ok ok valid evo 156.67.29.45:9999 443 ok sync ok ok valid regular 157.90.150.75:9999 - ok sync - - - evo 157.90.238.161:9999 443 ok sync ok ok valid @@ -1366,6 +1420,7 @@ evo 157.173.122.21:9999 443 ok sync ok ok valid evo 157.173.122.25:9999 443 ok sync ok ok valid evo 157.173.122.26:9999 443 ok sync ok ok valid regular 157.180.91.78:9999 - ok sync - - - +regular 157.230.22.29:9999 - ok sync - - - regular 157.230.24.176:9999 - ok sync - - - regular 157.230.113.158:9999 - ok sync - - - regular 157.230.119.39:9999 - ok sync - - - @@ -1399,6 +1454,7 @@ regular 159.203.33.3:9999 - ok sync - - - regular 159.203.33.254:9999 - ok sync - - - regular 159.203.39.201:9999 - ok sync - - - regular 159.203.61.192:9999 - ok sync - - - +regular 159.223.12.172:9999 - ok sync - - - regular 159.223.60.121:9999 - ok sync - - - regular 159.223.218.14:9999 - ok sync - - - regular 159.223.218.50:9999 - ok sync - - - @@ -1408,74 +1464,58 @@ regular 161.35.69.134:9999 - ok sync - - - regular 161.35.94.177:9999 - ok sync - - - regular 161.35.150.124:9999 - ok sync - - - regular 161.35.168.34:9999 - ok sync - - - -regular 161.35.207.170:9999 - ok sync - - - regular 161.35.213.210:9999 - ok sync - - - regular 161.35.247.75:9999 - ok sync - - - evo 161.97.66.31:9999 443 ok sync ok ok valid -regular 161.97.69.219:9999 - ok sync - - - evo 161.97.70.66:9999 443 ok sync ok ok valid evo 161.97.74.173:9999 443 ok sync ok ok valid evo 161.97.75.36:9999 443 ok sync ok ok valid -evo 161.97.83.102:9999 443 ok sync ok ok valid evo 161.97.85.159:9999 443 ok sync ok ok valid evo 161.97.85.182:9999 443 ok sync ok ok valid -evo 161.97.88.199:9999 443 ok sync ok ok valid evo 161.97.88.219:9999 443 ok sync ok ok valid evo 161.97.91.68:9999 443 ok sync ok ok valid evo 161.97.91.217:9999 443 ok sync ok ok valid evo 161.97.96.120:9999 443 ok sync ok ok valid evo 161.97.102.156:9999 443 ok sync ok ok valid -evo 161.97.104.37:9999 443 ok sync ok ok valid evo 161.97.106.14:9999 443 ok sync ok ok valid -evo 161.97.106.164:9999 443 ok sync ok ok valid evo 161.97.117.125:9999 443 ok sync ok ok valid -evo 161.97.120.156:9999 443 ok sync ok ok valid -regular 161.97.124.111:9999 - ok sync - - - evo 161.97.142.210:9999 443 ok sync ok ok valid evo 161.97.153.122:9999 443 ok sync ok ok valid evo 161.97.159.172:9999 443 ok sync ok ok valid evo 161.97.163.75:9999 443 ok sync ok ok valid -evo 161.97.165.115:9999 443 ok sync ok ok valid -evo 161.97.166.49:9999 443 ok sync ok ok valid -evo 161.97.173.67:9999 443 ok sync ok ok valid +evo 161.97.165.115:9999 443 error ? ok ok valid evo 161.97.175.233:9999 443 ok sync ok ok valid -evo 161.97.176.38:9999 443 ok sync ok ok valid -evo 161.97.179.214:9999 443 ok sync ok ok valid -evo 161.97.180.105:9999 443 ok sync ok ok valid evo 161.97.180.182:9999 443 ok sync ok ok valid regular 162.212.35.99:9999 - ok sync - - - evo 162.212.35.100:9999 443 ok sync ok ok valid regular 162.212.35.101:9999 - ok sync - - - regular 162.212.35.102:9999 - ok sync - - - -regular 162.212.35.103:9999 - ok sync - - - +regular 162.212.35.103:9999 - error ? - - - regular 162.212.35.104:9999 - ok sync - - - -regular 162.212.35.105:9999 - ok sync - - - +regular 162.212.35.105:9999 - error ? - - - regular 162.212.35.106:9999 - ok sync - - - regular 162.212.35.107:9999 - ok sync - - - -regular 162.212.35.108:9999 - ok sync - - - -regular 162.212.35.109:9999 - ok sync - - - +regular 162.212.35.108:9999 - error ? - - - +regular 162.212.35.109:9999 - error ? - - - regular 162.243.59.230:9999 - ok sync - - - regular 162.243.76.23:9999 - ok sync - - - regular 162.243.136.66:9999 - ok sync - - - regular 162.243.205.212:9999 - ok sync - - - regular 162.243.219.25:9999 - ok sync - - - regular 163.172.66.41:9999 - ok sync - - - -evo 164.68.114.36:9999 443 ok sync ok ok valid evo 164.68.118.37:9999 443 ok sync ok ok valid regular 164.90.221.233:9999 - ok sync - - - regular 164.92.160.130:9999 - ok sync - - - regular 164.92.164.29:9999 - ok sync - - - regular 164.92.170.229:9999 - ok sync - - - regular 164.92.175.210:9999 - ok sync - - - -regular 164.92.204.214:9999 - ok sync - - - -regular 164.92.249.64:9999 - ok sync - - - +regular 165.22.20.20:9999 - ok sync - - - regular 165.22.30.195:9999 - ok sync - - - regular 165.22.62.28:9999 - ok sync - - - +regular 165.22.62.245:9999 - ok sync - - - regular 165.22.65.68:9999 - ok sync - - - -regular 165.22.107.61:9999 - ok sync - - - regular 165.22.194.80:9999 - ok sync - - - regular 165.22.208.146:9999 - ok sync - - - -regular 165.22.221.97:9999 - ok sync - - - regular 165.22.227.208:9999 - ok sync - - - regular 165.22.234.135:9999 - ok sync - - - regular 165.22.236.90:9999 - ok sync - - - @@ -1483,10 +1523,12 @@ regular 165.227.35.129:9999 - ok sync - - - regular 165.227.43.54:9999 - ok sync - - - regular 165.227.46.145:9999 - ok sync - - - regular 165.227.166.196:9999 - ok sync - - - +regular 165.227.238.23:9999 - ok sync - - - regular 165.232.33.54:9999 - ok sync - - - regular 165.232.95.72:9999 - ok sync - - - regular 165.232.114.131:9999 - ok sync - - - regular 165.232.187.22:9999 - ok sync - - - +regular 167.71.40.198:9999 - ok sync - - - regular 167.71.75.173:9999 - ok sync - - - regular 167.71.76.44:9999 - ok sync - - - regular 167.71.133.157:9999 - ok sync - - - @@ -1497,7 +1539,8 @@ regular 167.71.229.90:9999 - ok sync - - - evo 167.86.77.58:9999 443 ok sync ok ok valid evo 167.86.93.21:9999 443 ok sync ok ok valid evo 167.86.94.138:9999 443 ok sync ok ok valid -evo 167.88.169.16:9999 443 ok sync ok ok valid +evo 167.88.169.16:9999 443 ok sync ok ok expired +regular 167.99.65.122:9999 - ok sync - - - regular 167.99.73.247:9999 - ok sync - - - regular 167.99.74.32:9999 - ok sync - - - regular 167.99.177.115:9999 - ok sync - - - @@ -1507,13 +1550,13 @@ regular 167.99.185.82:9999 - ok sync - - - regular 167.99.199.59:9999 - ok sync - - - regular 167.99.205.145:9999 - ok sync - - - regular 167.114.185.96:9999 - ok sync - - - -regular 167.172.45.235:9999 - ok sync - - - regular 167.172.65.155:9999 - ok sync - - - regular 167.172.67.55:9999 - ok sync - - - regular 167.172.70.187:9999 - ok sync - - - regular 167.172.87.128:9999 - ok sync - - - regular 167.172.94.148:9999 - ok sync - - - regular 167.172.183.190:9999 - ok sync - - - +evo 167.233.98.163:9999 443 ok sync ok ok valid evo 167.235.102.194:9999 443 ok sync ok ok valid regular 167.235.132.126:9999 - ok sync - - - regular 168.119.57.12:9999 - ok sync - - - @@ -1525,9 +1568,9 @@ regular 168.119.80.5:9999 - ok sync - - - regular 168.119.80.6:9999 - ok sync - - - regular 168.119.80.7:9999 - ok sync - - - regular 168.119.80.10:9999 - ok sync - - - -regular 168.119.80.12:9999 - ok sync - - - regular 168.119.80.13:9999 - ok sync - - - regular 168.119.80.15:9999 - ok sync - - - +regular 168.119.80.18:9999 - ok sync - - - regular 168.119.83.0:9999 - ok sync - - - regular 168.119.83.2:9999 - ok sync - - - regular 168.119.83.3:9999 - ok sync - - - @@ -1558,7 +1601,6 @@ regular 168.119.87.152:9999 - ok sync - - - regular 168.119.87.193:9999 - ok sync - - - regular 168.119.87.194:9999 - ok sync - - - regular 168.119.87.195:9999 - ok sync - - - -regular 168.119.87.196:9999 - ok sync - - - regular 168.119.87.197:9999 - ok sync - - - regular 168.119.87.198:9999 - ok sync - - - regular 168.119.87.199:9999 - ok sync - - - @@ -1566,9 +1608,12 @@ regular 168.119.87.202:9999 - ok sync - - - regular 168.119.87.203:9999 - ok sync - - - regular 168.119.87.204:9999 - ok sync - - - regular 168.119.87.206:9999 - ok sync - - - -evo 168.119.102.10:9999 443 ok sync ok ok valid +evo 168.119.102.10:9999 443 ok sync ok ok expired +regular 169.58.66.232:9999 - ok sync - - - +regular 169.58.103.141:9999 - ok sync - - - +evo 169.58.206.63:9999 443 ok sync ok ok valid regular 172.86.73.160:9999 - ok sync - - - -regular 172.86.91.158:9999 - ok sync - - - +regular 172.86.91.158:9999 - error ? - - - regular 172.104.145.166:9999 - ok sync - - - evo 172.236.244.81:9999 443 ok sync ok ok valid evo 172.238.7.25:9999 443 ok sync ok ok valid @@ -1586,13 +1631,13 @@ regular 173.249.33.131:9999 - ok sync - - - regular 173.249.35.151:9999 - ok sync - - - regular 173.249.40.168:9999 - ok sync - - - regular 174.138.11.164:9999 - ok sync - - - +regular 174.138.14.99:9999 - ok sync - - - regular 174.138.24.25:9999 - ok sync - - - regular 176.9.145.240:9999 - ok sync - - - regular 176.9.210.0:9999 - ok sync - - - regular 176.9.210.3:9999 - ok sync - - - regular 176.9.210.4:9999 - ok sync - - - regular 176.9.210.6:9999 - ok sync - - - -regular 176.9.210.9:9999 - ok sync - - - regular 176.9.210.13:9999 - ok sync - - - regular 176.9.210.14:9999 - ok sync - - - regular 176.9.210.15:9999 - ok sync - - - @@ -1601,7 +1646,7 @@ regular 176.9.210.21:9999 - ok sync - - - regular 176.9.210.22:9999 - ok sync - - - regular 176.9.210.23:9999 - ok sync - - - regular 176.102.65.145:9999 - ok sync - - - -evo 177.93.141.60:9999 443 ok sync ok ok expired +evo 177.93.141.60:9999 443 ok sync ok ok valid regular 177.93.141.86:9999 - ok sync - - - evo 178.18.244.255:9999 443 ok sync ok ok valid evo 178.18.254.136:9999 443 ok sync ok ok valid @@ -1644,7 +1689,6 @@ regular 178.62.192.42:9999 - ok sync - - - regular 178.62.196.119:9999 - ok sync - - - regular 178.62.200.153:9999 - ok sync - - - regular 178.62.207.128:9999 - ok sync - - - -regular 178.62.209.128:9999 - ok sync - - - regular 178.62.215.116:9999 - ok sync - - - regular 178.62.220.199:9999 - ok sync - - - regular 178.62.230.209:9999 - ok sync - - - @@ -1662,6 +1706,7 @@ regular 178.63.121.136:9999 - ok sync - - - regular 178.63.121.137:9999 - ok sync - - - regular 178.63.121.140:9999 - ok sync - - - regular 178.63.121.141:9999 - ok sync - - - +regular 178.63.121.144:9999 - ok sync - - - regular 178.63.121.147:9999 - ok sync - - - regular 178.63.235.192:9999 - ok sync - - - regular 178.63.235.193:9999 - ok sync - - - @@ -1679,6 +1724,7 @@ regular 178.63.236.103:9999 - ok sync - - - regular 178.63.236.104:9999 - ok sync - - - regular 178.63.236.105:9999 - ok sync - - - regular 178.63.236.107:9999 - ok sync - - - +regular 178.63.236.108:9999 - ok sync - - - regular 178.63.236.109:9999 - ok sync - - - regular 178.63.236.110:9999 - ok sync - - - regular 178.63.236.112:9999 - ok sync - - - @@ -1687,6 +1733,7 @@ regular 178.63.236.115:9999 - ok sync - - - regular 178.63.236.116:9999 - ok sync - - - regular 178.63.236.117:9999 - ok sync - - - regular 178.63.236.118:9999 - ok sync - - - +regular 178.63.236.119:9999 - ok sync - - - regular 178.104.155.5:9999 - ok sync - - - regular 178.128.40.87:9999 - ok sync - - - regular 178.128.49.188:9999 - ok sync - - - @@ -1735,7 +1782,6 @@ regular 185.164.163.218:9999 - ok sync - - - regular 185.164.163.220:9999 - ok sync - - - regular 185.165.171.117:9999 - ok sync - - - evo 185.166.217.154:9999 443 ok sync ok ok valid -regular 185.175.56.212:9999 - ok sync - - - regular 185.185.40.14:9999 - ok sync - - - regular 185.185.40.104:9999 - ok sync - - - regular 185.185.40.115:9999 - ok sync - - - @@ -1771,7 +1817,6 @@ regular 188.40.21.243:9999 - ok sync - - - regular 188.40.21.244:9999 - ok sync - - - regular 188.40.21.245:9999 - ok sync - - - regular 188.40.21.246:9999 - ok sync - - - -regular 188.40.21.247:9999 - ok sync - - - regular 188.40.21.248:9999 - ok sync - - - regular 188.40.163.0:9999 - ok sync - - - regular 188.40.163.1:9999 - ok sync - - - @@ -1785,6 +1830,7 @@ regular 188.40.163.8:9999 - ok sync - - - regular 188.40.163.9:9999 - ok sync - - - regular 188.40.163.12:9999 - ok sync - - - regular 188.40.163.15:9999 - ok sync - - - +regular 188.40.163.16:9999 - ok sync - - - regular 188.40.163.18:9999 - ok sync - - - regular 188.40.163.21:9999 - ok sync - - - regular 188.40.163.24:9999 - ok sync - - - @@ -1794,14 +1840,16 @@ regular 188.40.163.27:9999 - ok sync - - - regular 188.40.163.28:9999 - ok sync - - - regular 188.40.163.30:9999 - ok sync - - - regular 188.40.163.31:9999 - ok sync - - - -regular 188.40.169.39:9999 - ok sync - - - +regular 188.40.169.35:9999 - ok sync - - - regular 188.40.169.42:9999 - ok sync - - - regular 188.40.169.44:9999 - ok sync - - - regular 188.40.175.65:9999 - ok sync - - - regular 188.40.175.67:9999 - ok sync - - - regular 188.40.175.68:9999 - ok sync - - - regular 188.40.175.69:9999 - ok sync - - - +regular 188.40.175.70:9999 - ok sync - - - regular 188.40.175.71:9999 - ok sync - - - +regular 188.40.175.73:9999 - ok sync - - - regular 188.40.178.64:9999 - ok sync - - - regular 188.40.178.66:9999 - ok sync - - - regular 188.40.178.67:9999 - ok sync - - - @@ -1811,13 +1859,11 @@ regular 188.40.178.71:9999 - ok sync - - - regular 188.40.178.72:9999 - ok sync - - - regular 188.40.178.73:9999 - ok sync - - - regular 188.40.180.128:9999 - ok sync - - - -regular 188.40.180.129:9999 - ok sync - - - regular 188.40.180.130:9999 - ok sync - - - regular 188.40.180.132:9999 - ok sync - - - regular 188.40.180.135:9999 - ok sync - - - regular 188.40.180.138:9999 - ok sync - - - regular 188.40.180.139:9999 - ok sync - - - -regular 188.40.180.140:9999 - ok sync - - - regular 188.40.180.142:9999 - ok sync - - - regular 188.40.182.192:9999 - ok sync - - - regular 188.40.182.193:9999 - ok sync - - - @@ -1841,6 +1887,7 @@ regular 188.40.182.222:9999 - ok sync - - - regular 188.40.184.65:9999 - ok sync - - - regular 188.40.184.67:9999 - ok sync - - - regular 188.40.184.68:9999 - ok sync - - - +regular 188.40.184.69:9999 - ok sync - - - regular 188.40.184.72:9999 - ok sync - - - regular 188.40.185.128:9999 - ok sync - - - regular 188.40.185.129:9999 - ok sync - - - @@ -1852,12 +1899,12 @@ regular 188.40.185.134:9999 - ok sync - - - regular 188.40.185.137:9999 - ok sync - - - regular 188.40.185.144:9999 - ok sync - - - regular 188.40.185.147:9999 - ok sync - - - +regular 188.40.190.32:9999 - ok sync - - - regular 188.40.190.38:9999 - ok sync - - - regular 188.40.190.39:9999 - ok sync - - - regular 188.40.190.40:9999 - ok sync - - - regular 188.40.190.41:9999 - ok sync - - - regular 188.40.190.46:9999 - ok sync - - - -regular 188.40.190.49:9999 - ok sync - - - regular 188.40.190.52:9999 - ok sync - - - regular 188.40.190.55:9999 - ok sync - - - regular 188.40.190.56:9999 - ok sync - - - @@ -1869,7 +1916,6 @@ regular 188.40.205.10:9999 - ok sync - - - regular 188.40.205.11:9999 - ok sync - - - regular 188.40.205.13:9999 - ok sync - - - regular 188.40.205.18:9999 - ok sync - - - -regular 188.40.205.19:9999 - ok sync - - - regular 188.40.231.1:9999 - ok sync - - - regular 188.40.231.2:9999 - ok sync - - - regular 188.40.231.3:9999 - ok sync - - - @@ -1905,7 +1951,6 @@ regular 188.40.251.198:9999 - ok sync - - - regular 188.40.251.200:9999 - ok sync - - - regular 188.40.251.202:9999 - ok sync - - - regular 188.40.251.203:9999 - ok sync - - - -regular 188.40.251.204:9999 - ok sync - - - regular 188.40.251.205:9999 - ok sync - - - regular 188.40.251.206:9999 - ok sync - - - regular 188.40.251.207:9999 - ok sync - - - @@ -1928,7 +1973,6 @@ regular 188.166.5.202:9999 - ok sync - - - regular 188.166.8.181:9999 - ok sync - - - regular 188.166.21.185:9999 - ok sync - - - regular 188.166.30.248:9999 - ok sync - - - -regular 188.166.33.180:9999 - ok sync - - - regular 188.166.54.247:9999 - ok sync - - - regular 188.166.56.220:9999 - ok sync - - - regular 188.166.60.137:9999 - ok sync - - - @@ -1952,7 +1996,6 @@ regular 188.226.210.144:9999 - ok sync - - - regular 188.226.224.233:9999 - ok sync - - - regular 188.226.228.88:9999 - ok sync - - - evo 188.245.90.255:9999 443 ok sync ok ok valid -regular 191.101.2.231:9999 - ok sync - - - evo 192.95.32.205:9999 443 ok sync ok ok expired evo 192.99.153.216:9999 443 ok sync ok ok expired regular 192.169.6.87:9999 - ok sync - - - @@ -1961,63 +2004,60 @@ regular 192.241.194.154:9999 - ok sync - - - regular 192.241.217.126:9999 - ok sync - - - regular 192.248.169.180:9999 - ok sync - - - evo 192.248.175.198:9999 443 ok sync ok ok valid -evo 192.248.178.237:9999 443 ok sync ok ok valid +evo 192.248.178.237:9999 443 ok sync ok ok expired regular 192.248.189.66:9999 - ok sync - - - -evo 193.70.43.172:9999 443 ok sync ok ok valid -regular 193.160.119.129:9999 - ok sync - - - -regular 194.5.157.214:9999 - ok sync - - - +evo 193.70.43.172:9999 443 ok sync ok ok expired regular 194.58.47.132:9999 - ok sync - - - regular 194.58.47.228:9999 - ok sync - - - regular 194.135.80.33:9999 - ok sync - - - regular 194.135.84.100:9999 - ok sync - - - regular 194.135.94.228:9999 - ok sync - - - -evo 194.163.156.190:9999 443 ok sync ok ok valid evo 194.163.166.76:9999 443 ok sync ok ok valid -evo 194.163.172.206:9999 443 ok sync ok ok valid regular 194.163.183.132:9999 - ok sync - - - regular 194.238.25.199:9999 - ok sync - - - evo 195.26.254.228:9999 443 ok sync ok ok valid regular 195.98.95.210:9999 - ok sync - - - regular 195.201.35.7:9999 - ok sync - - - -evo 195.201.238.55:9999 443 ok sync ok ok valid +evo 195.201.238.55:9999 443 ok sync ok ok expired regular 198.12.92.214:9999 - ok sync - - - +regular 198.166.15.224:9999 - ok sync - - - regular 198.199.119.50:9999 - ok sync - - - regular 198.199.124.50:9999 - ok sync - - - regular 198.211.119.126:9999 - ok sync - - - regular 198.251.79.146:9999 - ok sync - - - +regular 199.85.8.8:9999 - ok sync - - - +regular 199.85.8.16:9999 - ok sync - - - +regular 199.85.8.190:9999 - ok sync - - - +regular 199.85.8.191:9999 - ok sync - - - +regular 199.91.220.189:9999 - ok sync - - - regular 202.5.16.33:9999 - ok sync - - - evo 202.71.14.79:9999 443 ok sync ok ok valid regular 202.71.14.128:9999 - ok sync - - - regular 202.182.105.95:9999 - ok sync - - - regular 202.182.115.161:9999 - ok sync - - - -regular 205.206.173.159:9999 - ok sync - - - regular 205.206.254.21:9999 - ok sync - - - regular 205.206.254.210:9999 - ok sync - - - +regular 206.189.20.59:9999 - ok sync - - - regular 206.189.21.120:9999 - ok sync - - - regular 206.189.28.109:9999 - ok sync - - - regular 206.189.112.246:9999 - ok sync - - - +regular 206.189.132.224:9999 - ok sync - - - regular 206.189.134.126:9999 - ok sync - - - -regular 206.189.136.218:9999 - ok sync - - - regular 206.189.143.107:9999 - ok sync - - - evo 206.245.167.74:9999 443 ok sync ok ok valid evo 206.245.167.76:9999 443 ok sync ok ok valid regular 207.148.5.10:9999 - ok sync - - - +regular 207.148.76.230:9999 - ok sync - - - regular 207.148.78.131:9999 - ok sync - - - regular 207.154.204.134:9999 - ok sync - - - regular 207.154.207.223:9999 - ok sync - - - regular 207.154.214.156:9999 - ok sync - - - -regular 207.154.226.228:9999 - ok sync - - - regular 207.154.229.81:9999 - ok sync - - - -regular 207.154.235.44:9999 - ok sync - - - evo 207.180.213.141:9999 443 ok sync ok ok valid -evo 207.180.218.245:9999 443 ok sync ok ok valid -regular 207.180.224.34:9999 - ok sync - - - -evo 207.180.224.96:9999 443 ok sync ok ok valid regular 207.180.227.146:9999 - ok sync - - - evo 207.180.233.6:9999 443 ok sync ok ok valid evo 207.180.233.246:9999 443 ok sync ok ok valid regular 207.180.237.66:9999 - ok sync - - - -evo 207.180.241.242:9999 443 ok sync ok ok valid evo 207.244.247.40:9999 443 ok sync ok ok valid regular 208.87.98.19:9999 - ok sync - - - regular 208.87.98.21:9999 - ok sync - - - @@ -2028,14 +2068,12 @@ regular 208.87.102.91:9999 - ok sync - - - regular 208.87.102.95:9999 - ok sync - - - regular 209.38.120.93:9999 - ok sync - - - regular 209.38.217.42:9999 - ok sync - - - -regular 209.38.226.76:9999 - ok sync - - - regular 209.97.142.65:9999 - ok sync - - - regular 209.141.36.206:9999 - ok sync - - - regular 209.250.240.73:9999 - ok sync - - - regular 212.24.104.225:9999 - ok sync - - - regular 212.24.109.253:9999 - ok sync - - - regular 213.136.91.123:9999 - ok sync - - - -evo 213.159.77.221:9999 443 ok sync ok ok valid evo 213.199.44.112:9999 443 ok sync ok ok valid evo 213.199.53.161:9999 443 ok sync ok ok valid evo 213.199.54.34:9999 443 ok sync ok ok valid @@ -2053,12 +2091,9 @@ regular 216.189.157.22:9999 - ok sync - - - regular 216.189.157.131:9999 - ok sync - - - regular 216.189.157.210:9999 - ok sync - - - regular 216.230.232.125:9999 - ok sync - - - -regular 216.238.73.67:9999 - ok sync - - - evo 216.238.75.46:9999 443 ok sync ok ok valid regular 216.238.82.102:9999 - ok sync - - - -regular 216.238.82.108:9999 - ok sync - - - -regular 216.238.85.93:9999 - ok sync - - - -regular 216.238.85.238:9999 - ok sync - - - evo 216.238.99.9:9999 443 ok sync ok ok valid regular 217.69.5.141:9999 - ok sync - - - evo 217.76.54.175:9999 443 ok sync ok ok valid +regular 217.154.231.20:9999 - ok sync - - - diff --git a/dash-network-seeds/seeds/testnet.txt b/dash-network-seeds/seeds/testnet.txt index f9426f81b..15a2447f4 100644 --- a/dash-network-seeds/seeds/testnet.txt +++ b/dash-network-seeds/seeds/testnet.txt @@ -1,10 +1,10 @@ -# Auto-generated by masternode-seeds-fetcher on 2026-05-18T07:39:39Z for testnet +# Auto-generated by masternode-seeds-fetcher on 2026-08-24T06:28:28Z for testnet # Source: Dash P2P network (mnlistdiff) -# Primary peer: 68.67.122.32:19999 -# Tip block hash: 0000007ca418eb769dee64c6ad8e31a3b38702bfb33b2b304b124306f00e2025 -# Tip block height: 1478969 -# 87 seeds (56 regular + 31 evo) of 542 total masternodes, valid-only -# Probe summary: core_reachable=87/87 platform_reachable=31/31 ssl_valid=29/31 +# Primary peer: 68.67.122.65:19999 +# Tip block hash: 000000ae66b60069f72d88fae1c1941d0c8d66ddf3093af1f06d19036b8ec9c8 +# Tip block height: 1540197 +# 85 seeds (55 regular + 30 evo) of 371 total masternodes, valid-only +# Probe summary: core_reachable=85/85 platform_reachable=30/30 ssl_valid=30/30 # Columns: # Values: core_reach/plat_reach=ok|timeout|refused|error|?, core_sync=sync|-N|+N|?, plat_live=ok|none|?|-, plat_ssl=valid|expired|self-signed|untrusted|no-handshake|?|- # Do not edit manually — refreshed weekly by .github/workflows/update-masternode-seeds.yml @@ -91,7 +91,5 @@ regular 68.67.122.79:19999 - ok sync - - - regular 68.67.122.80:19999 - ok sync - - - regular 68.67.122.81:19999 - ok sync - - - regular 68.67.122.82:19999 - ok sync - - - -regular 147.93.132.9:19999 - ok sync - - - -regular 185.187.169.193:19999 - ok sync - - - -evo 206.245.167.63:19999 1443 ok sync ok ok self-signed -evo 212.47.68.199:19999 1443 ok sync ok ok expired +regular 78.17.70.175:19999 - ok sync - - - +evo 206.245.167.63:19999 1443 ok sync ok ok valid diff --git a/dash-network/Cargo.toml b/dash-network/Cargo.toml index e8c5fa407..9aeaea805 100644 --- a/dash-network/Cargo.toml +++ b/dash-network/Cargo.toml @@ -22,8 +22,8 @@ ffi = [] [dependencies] serde = { version = "1.0.219", default-features = false, features = ["derive"], optional = true } -bincode = { version = "2.0.1", optional = true } -bincode_derive = { version = "2.0.1", optional = true } +bincode = { workspace = true, optional = true } +bincode_derive = { workspace = true, optional = true } [build-dependencies] cbindgen = "0.29" diff --git a/dash-spv-bench/.gitignore b/dash-spv-bench/.gitignore new file mode 100644 index 000000000..fb948c9f4 --- /dev/null +++ b/dash-spv-bench/.gitignore @@ -0,0 +1,24 @@ +bench-results/ +bench-storage/ +profiles/ + +*.lock + +# Persistent testnet chain snapshot grown by snapshot-chain.sh (multi-GB; never committed). +chain-data/ + +# Transient per-peer CoW clones created by run.sh + the file that remembers their dir. +.bench-clones.* +.clonedir + +# Scenario matrix outputs (bench-scenarios.sh): generated reports/logs/compose, and the +# bootstrapped yq binary. Reports are meant to be copied out, not committed. +.scenario.*.yml +results/ +.bin/ + +# FlameGraph tooling clone (run.sh --flame), fetched on demand. +.flamegraph/ + +# Wallet mnemonics (one BIP39 phrase per line) — secrets, never committed. +wallets.txt diff --git a/dash-spv-bench/Cargo.toml b/dash-spv-bench/Cargo.toml new file mode 100644 index 000000000..0cf11a8b9 --- /dev/null +++ b/dash-spv-bench/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "dash-spv-bench" +version = { workspace = true } +edition = "2021" +license = "MIT" +publish = false + +[dependencies] +dash-spv = { path = "../dash-spv" } +dashcore = { path = "../dash", features = ["serde", "core-block-hash-use-x11", "message_verification", "bls", "quorum_validation"] } +key-wallet = { path = "../key-wallet" } +key-wallet-manager = { path = "../key-wallet-manager", features = ["parallel-filters"] } + +tokio = { version = "1.0", features = ["full"] } + +anyhow = "1.0" +tempfile = "3.0" + +tracing = "0.1" +tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } +indicatif = "0.18" + +[[bin]] +name = "dash-spv-bench" +path = "src/main.rs" diff --git a/dash-spv-bench/Dockerfile b/dash-spv-bench/Dockerfile new file mode 100644 index 000000000..65b6fde46 --- /dev/null +++ b/dash-spv-bench/Dockerfile @@ -0,0 +1,24 @@ +FROM debian:bookworm-slim + +ARG DASHVERSION=23.1.7 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl iproute2 \ + && rm -rf /var/lib/apt/lists/* + +# Download the Linux dashd matching the host CPU arch (amd64 -> x86_64, arm64 -> aarch64). +# The version is pinned so benchmark numbers stay comparable; run.sh and snapshot-chain.sh tag the +# image with it and pass it back as --build-arg, so bumping it there rebuilds instead of reusing. +RUN set -eux; \ + arch="$(dpkg --print-architecture)"; \ + case "$arch" in \ + amd64) dlarch=x86_64 ;; \ + arm64) dlarch=aarch64 ;; \ + *) echo "unsupported arch $arch" >&2; exit 1 ;; \ + esac; \ + url="https://github.com/dashpay/dash/releases/download/v${DASHVERSION}/dashcore-${DASHVERSION}-${dlarch}-linux-gnu.tar.gz"; \ + curl -fsSL "$url" -o /tmp/dashcore.tgz; \ + tar -xzf /tmp/dashcore.tgz -C /tmp; \ + cp "/tmp/dashcore-${DASHVERSION}/bin/dashd" /usr/local/bin/dashd; \ + rm -rf /tmp/dashcore*; \ + dashd --version | head -1 diff --git a/dash-spv-bench/Dockerfile.client b/dash-spv-bench/Dockerfile.client new file mode 100644 index 000000000..d0f1f96c9 --- /dev/null +++ b/dash-spv-bench/Dockerfile.client @@ -0,0 +1,10 @@ +# Runtime image for the measured client. +# +# Only exists so the client's own link can be shaped +FROM debian:bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates iproute2 \ + && rm -rf /var/lib/apt/lists/* + +ENTRYPOINT ["/bin/sh", "-ec"] diff --git a/dash-spv-bench/bench-scenarios.sh b/dash-spv-bench/bench-scenarios.sh new file mode 100755 index 000000000..7e1ffd744 --- /dev/null +++ b/dash-spv-bench/bench-scenarios.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# dash-spv-bench scenario matrix (LOCAL mode only). +# +# ./bench-scenarios.sh [-d scenarios] +# +# Thin loop over scenario files: for each scenarios/*.yml it calls `run.sh scenario ` +# once per its `repeats:` count, captures the timings run.sh prints, and writes an organized +# Markdown report (median over repeats). All the real work — compose generation, bring-up, +# CPU pinning, teardown — lives in run.sh; this only orchestrates and aggregates. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +cd "${SCRIPT_DIR}" + +SCN_DIR="${SCRIPT_DIR}/scenarios" +RUN_CMD="${RUN_CMD:-${SCRIPT_DIR}/run.sh}" # overridable for testing the loop without docker +while [ $# -gt 0 ]; do + case "$1" in + -d | --dir) SCN_DIR="$2"; shift 2 ;; + -h | --help) sed -n '3,10p' "$0"; exit 0 ;; + *) echo "unknown arg: $1" >&2; exit 1 ;; + esac +done + +shopt -s nullglob +FILES=("${SCN_DIR}"/*.yml) +[ "${#FILES[@]}" -gt 0 ] || { echo "Error: no scenarios (*.yml) in ${SCN_DIR}" >&2; exit 1; } + +RUN_TS="$(date +%Y%m%d-%H%M%S)" +OUT_DIR="${SCRIPT_DIR}/results/${RUN_TS}" +mkdir -p "${OUT_DIR}" +REPORT="${OUT_DIR}/report.md" +TSV="${OUT_DIR}/results.tsv" +: >"${TSV}" + +metric() { awk -F':[[:space:]]*' -v k="$2" '$1==k {gsub(/[[:space:]]+$/,"",$2); print $2; exit}' "$1"; } +median() { sort -n | awk '{a[NR]=$1} END{if(NR==0){print"-";exit} print (NR%2)?a[(NR+1)/2]:int((a[NR/2]+a[NR/2+1])/2)}'; } + +for f in "${FILES[@]}"; do + name="$(basename "${f}" .yml)" + repeats="$(awk '/^repeats:/{print $2; exit}' "${f}")"; repeats="${repeats:-1}" + echo "===== scenario '${name}' × ${repeats} =====" + for r in $(seq 1 "${repeats}"); do + echo "-- ${name} repeat ${r}/${repeats}" + log="${OUT_DIR}/${name}.r${r}.log" + "${RUN_CMD}" "${f}" >"${log}" 2>&1 || echo " (run.sh exited non-zero; see ${log})" + + # Scenario metadata comes straight from run.sh's own echo lines, so we never re-parse the YAML. + meta="$(grep -m1 "==> scenario '" "${log}" || true)" + cpus="$(sed -n "s/.*cpus=\([^,]*\),.*/\1/p" <<<"${meta}")" + blocks="$(sed -n "s/.*blocks=\([0-9]*\).*/\1/p" <<<"${meta}")" + peers="$(sed -n "s/.*\[\(.*\)\], cpus=.*/\1/p" <<<"${meta}")" + desc="$(grep -m1 "==> description:" "${log}" | sed 's/.*==> description: //' || true)" + + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${name}" "${cpus}" "${blocks}" "${peers}" "${r}" \ + "$(metric "${log}" completed)" "$(metric "${log}" total_ms)" \ + "$(metric "${log}" block_headers_ms)" "$(metric "${log}" filter_headers_ms)" \ + "$(metric "${log}" filters_ms)" "$(metric "${log}" filter_hdr_lag_ms)" "${desc}" \ + >>"${TSV}" + done +done + +# --- report --------------------------------------------------------------------------------- +{ + echo "# dash-spv-bench scenario report" + echo + echo "- **Date:** $(date -u '+%Y-%m-%d %H:%M:%SZ')" + echo "- **Host:** $(nproc) cores — $(grep -m1 'model name' /proc/cpuinfo 2>/dev/null | cut -d: -f2 | sed 's/^ //' || uname -m)" + echo "- **git:** $(git -C "${REPO_ROOT}" rev-parse --abbrev-ref HEAD 2>/dev/null)@$(git -C "${REPO_ROOT}" rev-parse --short HEAD 2>/dev/null)" + echo "- **Scenarios:** \`$(basename "${SCN_DIR}")/\` (${#FILES[@]} files)" + echo + echo "Timings are the **median** over each scenario's repeats (ms)." + echo + echo "| scenario | description | cpus | blocks | peers | completed | total | block_hdrs | filter_hdrs | filters | fh_lag |" + echo "|---|---|---|---|---|---|---|---|---|---|---|" + for name in $(cut -f1 "${TSV}" | awk '!seen[$0]++'); do + row() { awk -F'\t' -v n="${name}" '$1==n{print $'"$1"'; exit}' "${TSV}"; } + med() { awk -F'\t' -v n="${name}" '$1==n && $'"$1"'!="" && $'"$1"'!="-"{print $'"$1"'}' "${TSV}" | median; } + echo "| ${name} | $(row 12) | $(row 2) | $(row 3) | $(row 4) | $(row 6) | $(med 7) | $(med 8) | $(med 9) | $(med 10) | $(med 11) |" + done + echo + echo "Raw per-repeat data: \`results.tsv\`; per-run logs alongside this report." +} >"${REPORT}" + +echo "==> wrote ${REPORT}" diff --git a/dash-spv-bench/run.sh b/dash-spv-bench/run.sh new file mode 100755 index 000000000..a3770b607 --- /dev/null +++ b/dash-spv-bench/run.sh @@ -0,0 +1,663 @@ +#!/bin/bash +# +# dash-spv benchmark driver — runs ONE scenario end to end (build, bring up peers, sync, report). +# +# Usage: +# ./run.sh Run the scenario +# ./run.sh --flame Same, under the sampler (perf on Linux, sample on macOS) +# -> profiles/flamegraph.svg +# --wallets Wallets for this run: a file with one BIP39 mnemonic per +# line. No file => the run has no wallet. +# +# ./run.sh 'scenarios/mainnet.*' Several scenarios: any argument that is not a file is +# ./run.sh scenarios/local.*.yml treated as a glob (quote it to let run.sh expand it, or +# ./run.sh scenarios/*.yml let your shell do it). Patterns resolve against the +# current directory and then against scenarios/, so +# 'mainnet.*' works from anywhere. +# +# Every invocation, one scenario or twenty, writes results// containing per scenario +# .log stdout of the run (build, bring-up, summary) +# .run.log the sync trace, kept because bench-storage/ is wiped by the next run +# .summary.txt the metrics block +# plus results.tsv and report.md over all of them. A scenario that fails is recorded and the +# rest still run. +# +# RUST_LOG can be set to tweak logging, e.g. RUST_LOG=info ./run.sh scenarios/local.1ideal.yml +# It is the tracing filter for BOTH log sinks and overrides the defaults, which are +# terminal = "warn,dash_spv_bench=info" (kept light so the live bars stay readable) +# file = "warn,dash_spv=debug,dash_spv_bench=debug" (debug, for offline analysis) +# +# Outputs land in bench-storage/ (gitignored, wiped at the start of each run): +# run.log full trace of the sync (debug by default) for offline analysis +# summary.txt metrics + per-wallet tx/balance fingerprint (also printed to stdout) +# +# bench-storage/ is wiped at the start of every run, so both are archived into results/ too. +# plus the SPV storage the run produced (block_headers/, filters/, ...) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +COMPOSE_FILE="" # local mode generates one here; deleted on exit +DASH_VERSION="23.1.7" +IMAGE="dash-spv-bench/dashd:${DASH_VERSION}" +CLIENT_IMAGE="dash-spv-bench/client:1" +# Rust image for the Linux build, taken from the workspace's own pin so the two +# cannot drift: with a mismatched tag the image spends every run having rustup +# fetch the pinned toolchain before it can compile anything. +RUST_CHANNEL="$(sed -n 's/^channel *= *"\(.*\)"/\1/p' "${REPO_ROOT}/rust-toolchain.toml" 2>/dev/null || true)" +RUST_IMAGE="rust:${RUST_CHANNEL:-1.89}-bookworm" +# Separate from the host's `target/`: a different triple, and sharing one +# directory across both would make every switch a full rebuild. +PROJECT="spv-bench" +STATE="${SCRIPT_DIR}/.clonedir" +FLAME_SVG="${SCRIPT_DIR}/profiles/flamegraph.svg" +FLAMEGRAPH_DIR="${SCRIPT_DIR}/.flamegraph" # FlameGraph tooling clone (gitignored, inside the package) +CHAIN_DIR="${SCRIPT_DIR}/chain-data" + +# Absolute, because the script `cd`s to its own directory below: a relative `$0` +# stops resolving after that, which broke both `--help` and re-invoking self. +SELF="${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")" + +INVOCATION_DIR="${PWD}" +abspath() { case "$1" in /*) printf '%s\n' "$1" ;; *) printf '%s\n' "${INVOCATION_DIR%/}/$1" ;; esac; } + +cd "${SCRIPT_DIR}" + +FLAME=0 +SCN_ARGS=() +WALLETS_ARG="" +while [ $# -gt 0 ]; do + case "$1" in + --flame) FLAME=1; shift ;; + --wallets) WALLETS_ARG="${2:?--wallets needs a file path}"; shift 2 ;; + -h | --help) sed -n '3,24p' "${SELF}"; exit 0 ;; + -*) echo "unknown flag: $1" >&2; exit 1 ;; + *) SCN_ARGS+=("$1"); shift ;; + esac +done +[ "${#SCN_ARGS[@]}" -gt 0 ] || { echo "usage: $0 ... [--flame] [--wallets ]" >&2; exit 1; } + +# Resolve each argument to scenario files. +# +# A bare path stays a bare path, so the original single-file invocation is +# untouched. Anything that is not a file is treated as a glob — which covers +# both a quoted pattern (`'scenarios/mainnet.*'`, expanded here) and one the +# caller's shell already expanded into several arguments. Patterns are tried +# against the invocation directory first and then against `scenarios/`, so +# `mainnet.*` works from anywhere. +SCN_FILES=() +for arg in "${SCN_ARGS[@]}"; do + if [ -f "${arg}" ] || [ -f "$(abspath "${arg}")" ]; then + SCN_FILES+=("$(abspath "${arg}")") + continue + fi + matched=0 + while IFS= read -r hit; do + [ -f "${hit}" ] || continue + SCN_FILES+=("${hit}") + matched=1 + done < <(cd "${INVOCATION_DIR}" 2>/dev/null && shopt -s nullglob && printf '%s\n' ${arg} | while IFS= read -r m; do abspath "${m}"; done + cd "${SCRIPT_DIR}/scenarios" 2>/dev/null && shopt -s nullglob && printf '%s\n' ${arg} | while IFS= read -r m; do printf '%s\n' "${SCRIPT_DIR}/scenarios/${m}"; done) + [ "${matched}" -eq 1 ] || { echo "Error: no scenario matched: ${arg}" >&2; exit 1; } +done + +# Run each scenario through a fresh invocation of this script and collect the +# numbers. Re-invoking rather than looping in place keeps the per-scenario path +# — exports, compose, traps, teardown — byte for byte what a single-file run has +# always done, so the extension cannot change the thing it is measuring. +# +# Taken for one scenario as much as for twenty: naming a file and naming a glob +# that happens to match one file are the same request, and having them leave +# their results in different shapes is a trap for anyone scripting on top. The +# child is marked so it runs the scenario instead of wrapping it again. +if [ -z "${BENCH_BATCH:-}" ]; then + RUN_TS="$(date +%Y%m%d-%H%M%S)" + OUT_DIR="${SCRIPT_DIR}/results/${RUN_TS}" + mkdir -p "${OUT_DIR}" + TSV="${OUT_DIR}/results.tsv" + REPORT="${OUT_DIR}/report.md" + printf 'scenario\tcompleted\ttotal_ms\tblock_headers_ms\tfilter_headers_ms\tfilters_ms\ttransactions\tconfirmed_sat\n' >"${TSV}" + + child_flags=() + [ "${FLAME}" -eq 1 ] && child_flags+=(--flame) + [ -n "${WALLETS_ARG}" ] && child_flags+=(--wallets "${WALLETS_ARG}") + + metric() { awk -F':[[:space:]]*' -v k="$2" '$1==k {gsub(/[[:space:]]+$/,"",$2); print $2; exit}' "$1"; } + + echo "==> ${#SCN_FILES[@]} scenario(s); results in ${OUT_DIR}" + for f in "${SCN_FILES[@]}"; do + name="$(basename "${f}" .yml)" + log="${OUT_DIR}/${name}.log" + echo "===== ${name} =====" + # A scenario that fails must not take the batch with it: record it and move + # on, or one bad run costs every result after it. + BENCH_BATCH=1 BENCH_ARCHIVE_DIR="${OUT_DIR}" BENCH_ARCHIVE_NAME="${name}" \ + "${SELF}" "${f}" "${child_flags[@]+"${child_flags[@]}"}" 2>&1 | tee "${log}" || true + # Read the metrics from the summary the child archived, not from its stdout: + # the summary is written by the binary and is plain text either way, while a + # pty-captured log carries the bars' escape codes and carriage returns. + src="${log}" + if [ -s "${OUT_DIR}/${name}.summary.txt" ]; then src="${OUT_DIR}/${name}.summary.txt"; fi + wallet_line="$(grep -m1 -o 'confirmed_sat=[0-9]*' "${src}" || true)" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${name}" \ + "$(metric "${src}" completed)" \ + "$(metric "${src}" total_ms)" \ + "$(metric "${src}" block_headers_ms)" \ + "$(metric "${src}" filter_headers_ms)" \ + "$(metric "${src}" filters_ms)" \ + "$(metric "${src}" transactions)" \ + "$(sed -n 's/.*confirmed_sat=\([0-9]*\).*/\1/p' <<<"${wallet_line}")" \ + >>"${TSV}" + done + + { + echo "# dash-spv bench — ${RUN_TS}" + echo + echo "| scenario | completed | total_ms | headers_ms | filter_headers_ms | filters_ms | transactions | confirmed_sat |" + echo "|---|---|---|---|---|---|---|---|" + tail -n +2 "${TSV}" | awk -F'\t' '{printf "| %s | %s | %s | %s | %s | %s | %s | %s |\n", $1,$2,$3,$4,$5,$6,$7,$8}' + } >"${REPORT}" + + echo + echo "==> report: ${REPORT}" + cat "${REPORT}" + exit 0 +fi + +SCN_FILE="${SCN_FILES[0]}" +[ -f "${SCN_FILE}" ] || { echo "Error: scenario file not found: ${SCN_FILE}" >&2; exit 1; } + +YQ_VERSION="v4.44.6" +ensure_yq() { + if command -v yq >/dev/null 2>&1; then YQ=yq; return 0; fi + local bin="${SCRIPT_DIR}/.bin/yq" + if [ ! -x "${bin}" ]; then + mkdir -p "${SCRIPT_DIR}/.bin" + local os arch + os="$(uname -s | tr '[:upper:]' '[:lower:]')"; arch="$(uname -m)" + case "${arch}" in x86_64 | amd64) arch=amd64 ;; aarch64 | arm64) arch=arm64 ;; esac + echo "==> fetching yq ${YQ_VERSION} (${os}/${arch}) into .bin/yq" + curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_${os}_${arch}" \ + -o "${bin}" || { echo "Error: could not download yq; install it manually." >&2; exit 1; } + chmod +x "${bin}" + fi + YQ="${bin}" +} +ensure_yq +scn() { "${YQ}" "$1" "${SCN_FILE}"; } # evaluate a yq expression against the scenario file + +_peer_group() { + "${YQ}" ".peers[$1] | [.count, .latency_ms // 0, .jitter_ms // 0, .loss_pct // 0, .rate_kbit // 0, .corrupt_pct // 0, .reorder_pct // 0] | @tsv" "${SCN_FILE}" +} + +build_netem() { + local lat="$1" jit="$2" loss="$3" rate="$4" corrupt="$5" reorder="$6" a="" + [ "${lat}" != 0 ] && { a="delay ${lat}ms"; [ "${jit}" != 0 ] && a="${a} ${jit}ms"; } + [ "${loss}" != 0 ] && a="${a} loss ${loss}%" + [ "${rate}" != 0 ] && a="${a} rate ${rate}kbit" + [ "${corrupt}" != 0 ] && a="${a} corrupt ${corrupt}%" + [ "${reorder}" != 0 ] && a="${a} reorder ${reorder}%" + echo "${a# }" +} + +# The measured client's own link shaping, if the scenario asks for one +client_netem() { + local lat jit loss rate corrupt reorder + read -r lat jit loss rate corrupt reorder <<<"$("${YQ}" \ + '[.client.latency_ms // 0, .client.jitter_ms // 0, .client.loss_pct // 0, .client.rate_kbit // 0, .client.corrupt_pct // 0, .client.reorder_pct // 0] | @tsv' \ + "${SCN_FILE}")" + build_netem "${lat}" "${jit}" "${loss}" "${rate}" "${corrupt}" "${reorder}" +} + +# The client's bandwidth cap on its own, so the container can mirror it onto +# ingress +client_rate_kbit() { + "${YQ}" '.client.rate_kbit // 0' "${SCN_FILE}" +} + +peers_summary() { + local ng g count lat jit loss rate corrupt reorder tag out="" + ng="$(scn '.peers | length')" + case "${ng}" in ''|null|*[!0-9]*) ng=0 ;; esac + for ((g = 0; g < ng; g++)); do + read -r count lat jit loss rate corrupt reorder <<<"$(_peer_group "${g}")" + tag="${lat}ms" + [ "${jit}" != 0 ] && tag="${tag}±${jit}" + [ "${loss}" != 0 ] && tag="${tag}/${loss}%loss" + [ "${rate}" != 0 ] && tag="${tag}/${rate}kbit" + out="${out}, ${count}×${tag}" + done + echo "${out#, }" +} + +emit_compose() { + local out="$1" peer_cpus="$2" + cat >"${out}" <
>"${out}" <<'ANCHOR' + build: + context: . + dockerfile: Dockerfile + cap_add: [NET_ADMIN] + entrypoint: ["/bin/sh", "-ec"] + command: + - | + if [ -n "$${NETEM_ARGS:-}" ]; then + tc qdisc add dev eth0 root netem $${NETEM_ARGS} \ + && echo "netem: $${NETEM_ARGS}" || echo "WARNING: netem failed (NET_ADMIN/sch_netem?)" + fi + exec dashd -testnet -datadir=/data -port=19400 -rpcport=19500 -server=1 -daemon=0 \ + -connect=0 -bind=0.0.0.0 -listen=1 -rpcbind=0.0.0.0 -rpcallowip=0.0.0.0/0 \ + -whitelist=0.0.0.0/0 -disablewallet=1 -peerbloomfilters=1 \ + -dbcache=64 -fallbackfee=0.00001 -txindex=0 -addressindex=0 $${FILTER_FLAGS} + +services: +ANCHOR + local ng g count lat jit loss rate corrupt reorder netem n peer=0 + # `null` when the scenario has no `peers:` at all, which is the normal shape + # for testnet/mainnet: there the peers are the real network and the only + # service to emit is the client. + ng="$(scn '.peers | length')" + # Arithmetic loops rather than `seq`: BSD seq (macOS) reads `seq 0 -1` as a + # descending range and prints "0 -1", where GNU seq prints nothing. An empty + # `peers:` — the normal shape for testnet/mainnet — hit exactly that and + # emitted two garbage peer services. + case "${ng}" in ''|null|*[!0-9]*) ng=0 ;; esac + for ((g = 0; g < ng; g++)); do + read -r count lat jit loss rate corrupt reorder <<<"$(_peer_group "${g}")" + netem="$(build_netem "${lat}" "${jit}" "${loss}" "${rate}" "${corrupt}" "${reorder}")" + case "${count}" in ''|null|*[!0-9]*) count=0 ;; esac + for ((n = 1; n <= count; n++)); do + peer=$((peer + 1)) + cat >>"${out}" <>"${out}" </dev/null \ + && ip link set ifb0 up \ + && tc qdisc add dev eth0 handle ffff: ingress \ + && tc filter add dev eth0 parent ffff: protocol all prio 1 u32 \ + match u32 0 0 action mirred egress redirect dev ifb0 \ + && tc qdisc add dev ifb0 root netem rate \$\${INGRESS_RATE_KBIT}kbit; then + echo "client netem (ingress): rate \$\${INGRESS_RATE_KBIT}kbit via ifb0" + else + tc qdisc del dev eth0 ingress 2>/dev/null || true + if tc qdisc add dev eth0 handle ffff: ingress 2>/dev/null \ + && tc filter add dev eth0 parent ffff: protocol all prio 1 u32 \ + match u32 0 0 action police rate \$\${INGRESS_RATE_KBIT}kbit \ + burst \$\${INGRESS_BURST_K}k mtu 64k conform-exceed drop; then + echo "client police (ingress): rate \$\${INGRESS_RATE_KBIT}kbit burst \$\${INGRESS_BURST_K}k" + echo "NOTE: no ifb, so ingress is POLICED (drops) not shaped (queues) — throughput lands within ~3% but the loss pattern differs; do not compare against ifb runs" + else + echo "WARNING: ingress shaping failed (NET_ADMIN? no ifb and no act_police?)" + echo "WARNING: DOWNLOADS ARE UNSHAPED — rate_kbit is NOT enforced and this run is not comparable" + fi + fi + fi + exec /usr/local/bin/dash-spv-bench +SERVICE +} + + +MODE="$(scn '.mode // "local"')" +case "${MODE}" in local | testnet | mainnet) ;; *) echo "Error: mode must be 'local', 'testnet' or 'mainnet' (got '${MODE}')" >&2; exit 1 ;; esac +export BENCH_MODE="${MODE}" +export CLONE_DIR="${CLONE_DIR:-/nonexistent}" + +CLIENT_NETEM="$(client_netem)" +CLIENT_RATE_KBIT="$(client_rate_kbit)" +case "${CLIENT_RATE_KBIT}" in + '' | *[!0-9]*) echo "Error: client.rate_kbit must be a whole number of kbit (got '${CLIENT_RATE_KBIT}')" >&2; exit 1 ;; +esac +# Burst for the policer fallback: ~0.5s of the rate (kbit/16 => kbytes), floored +# so a very small rate still gets a workable bucket. Unused on the ifb path. +CLIENT_INGRESS_BURST_K=$(( CLIENT_RATE_KBIT / 16 )) +if [ "${CLIENT_INGRESS_BURST_K}" -lt 32 ]; then CLIENT_INGRESS_BURST_K=32; fi +BENCH_CPUS="$(scn '.cpus // ""')" +BENCH_MAX_PEERS="$(scn '.max_peers // ""')" # only if set; else the ClientConfig default +export BENCH_MAX_PEERS + +BENCH_WALLET_FILE="${SCRIPT_DIR}/wallets.txt" +[ -n "${WALLETS_ARG}" ] && BENCH_WALLET_FILE="$(abspath "${WALLETS_ARG}")" +export BENCH_WALLET_FILE +export BENCH_STORAGE_DIR="${SCRIPT_DIR}/bench-storage" + +DESC="$(scn '.description // ""')" +[ -n "${DESC}" ] && echo "==> description: ${DESC}" + +BLOCKS="" +if [ "${MODE}" = local ]; then + BLOCKS="$(scn '.blocks // 1000000')" + export BENCH_HEIGHT="${BLOCKS}" + unset BENCH_START_HEIGHT # local always syncs from genesis +else + BENCH_PEERS="$(scn '.peers // [] | join(",")')" + export BENCH_PEERS + sh="$(scn '.start_height // ""')"; [ -n "${sh}" ] && export BENCH_START_HEIGHT="${sh}" +fi + +expand_cpus() { + local part lo hi + local IFS=, + for part in $1; do + case "${part}" in + *-*) lo="${part%-*}"; hi="${part#*-}"; seq "${lo}" "${hi}" ;; + *) echo "${part}" ;; + esac + done +} + +CPU_PREFIX=() +if [ -n "${BENCH_CPUS}" ]; then + ncpu="$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 0)" + bench_cores="$(expand_cpus "${BENCH_CPUS}" | sort -nu)" + if [ "${ncpu}" -gt 0 ]; then + peer_cores="" + for i in $(seq 0 $((ncpu - 1))); do + grep -qxF "${i}" <<<"${bench_cores}" || peer_cores="${peer_cores},${i}" + done + BENCH_PEER_CPUS="${peer_cores#,}" + fi + + if [ -n "${CLIENT_NETEM}" ]; then + # The client is a container: `cpuset` on the service does the pinning that + # `taskset` does for a host run, and it works on macOS where taskset does + # not exist at all. Emitted by `emit_client_service` from BENCH_CPUS. + echo "==> pinning the measured client container to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" + elif command -v taskset >/dev/null 2>&1; then + CPU_PREFIX=(taskset -c "${BENCH_CPUS}") + echo "==> pinning the measured run to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" + else + echo "==> note: 'taskset' not found (e.g. macOS); running the measured binary UNPINNED${BENCH_PEER_CPUS:+ (docker peers still pinned to ${BENCH_PEER_CPUS})}" >&2 + fi +fi + +# Where the measured binary will be, decided before the compose that mounts +# it is written. Everything lands in the workspace `target/`: a cross build +# gets cargo's own per-triple subdirectory, so it cannot collide with a host +# build, and neither duplicates the dependency graph. +if [ -n "${CLIENT_NETEM}" ] && [ "$(uname -s)" != Linux ]; then + CROSS_TRIPLE="$(docker run --rm "${RUST_IMAGE}" rustc -vV | sed -n 's/^host: //p')" + [ -n "${CROSS_TRIPLE}" ] || { echo "could not read the build image's target triple" >&2; exit 1; } + BIN="${REPO_ROOT}/target/${CROSS_TRIPLE}/release/dash-spv-bench" +else + CROSS_TRIPLE="" + BIN="${REPO_ROOT}/target/release/dash-spv-bench" +fi + +# A compose file is needed for the peers (local mode) and for the client +# container (any mode with a `client:` section) — testnet/mainnet shape the +# client against the real network, with no peer services at all. +if [ "${MODE}" = local ] || [ -n "${CLIENT_NETEM}" ]; then + COMPOSE_FILE="$(mktemp "${SCRIPT_DIR}/.scenario.XXXXXX")" + mv "${COMPOSE_FILE}" "${COMPOSE_FILE}.yml" + COMPOSE_FILE="${COMPOSE_FILE}.yml" + npeers="$(emit_compose "${COMPOSE_FILE}" "${BENCH_PEER_CPUS:-}")" + if [ "${MODE}" = local ]; then + echo "==> scenario '$(basename "${SCN_FILE}" .yml)': ${npeers} peers [$(peers_summary)], cpus=${BENCH_CPUS:-}, blocks=${BLOCKS}" + fi +fi +[ -n "${CLIENT_NETEM}" ] && echo "==> client link shaped: ${CLIENT_NETEM}" + +compose() { docker compose -p "${PROJECT}" -f "${COMPOSE_FILE}" "$@"; } + +wait_loaded() { + local c logs + for _ in $(seq 1 400); do + local all=1 + for c in "$@"; do + logs="$(docker logs "${c}" 2>&1)" || { all=0; break; } + case "${logs}" in *"init message: Done loading"*) ;; *) all=0; break ;; esac + done + [ "${all}" -eq 1 ] && return 0 + echo -n "."; sleep 3 + done + return 1 +} + +teardown() { + [ -n "${COMPOSE_FILE}" ] && compose down --remove-orphans >/dev/null 2>&1 || true + [ -f "${STATE}" ] && { rm -rf "$(cat "${STATE}")" 2>/dev/null || true; rm -f "${STATE}"; } + rm -rf "${SCRIPT_DIR}"/.bench-clones.* 2>/dev/null || true +} + +arm_teardown() { + # On exit: tear down, THEN delete the generated compose (down needs it to still exist). + trap 'teardown; [ -n "${COMPOSE_FILE}" ] && rm -f "${COMPOSE_FILE}"' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + trap 'exit 129' HUP +} + +bring_up() { + echo "==> ensuring a clean network" + teardown + docker image inspect "${IMAGE}" >/dev/null 2>&1 || { echo "==> building peer image"; compose build; } + + # The client is in the same compose file but is not a peer: it must not be + # counted, started here, or waited on for "Done loading". + local services; services="$(compose config --services | grep -v '^client$' || true)" + local n; n="$(echo ${services} | wc -w | tr -d ' ')" + # BENCH_PEERS is derived from the generated peers (host ports 19401..). + local peers_csv="" + for svc in ${services}; do peers_csv="${peers_csv},127.0.0.1:$((19400 + ${svc#dashd}))"; done + export BENCH_PEERS="${peers_csv#,}" + + echo "==> CoW-cloning ${CHAIN_DIR} for ${n} peers (instant)" + local clone_dir; clone_dir="$(mktemp -d "${SCRIPT_DIR}/.bench-clones.XXXXXX")" + echo "${clone_dir}" > "${STATE}" + for svc in ${services}; do + local dst="${clone_dir}/peer${svc#dashd}" + cp -c -R "${CHAIN_DIR}" "${dst}" 2>/dev/null \ + || cp --reflink=auto -R "${CHAIN_DIR}" "${dst}" 2>/dev/null \ + || cp -R "${CHAIN_DIR}" "${dst}" + done + + local batch=4 + echo "==> starting ${n} peers in batches of ${batch}" + local started="" count=0 + for svc in ${services}; do + CLONE_DIR="${clone_dir}" compose up -d "${svc}" >/dev/null 2>&1 + started="${started} spv-bench-${svc}"; count=$((count + 1)) + if [ $((count % batch)) -eq 0 ]; then + echo -n " loaded ${count}/${n} " + wait_loaded ${started} || { echo " timeout loading batch" >&2; exit 1; } + echo " ok" + fi + done + if [ $((count % batch)) -ne 0 ]; then # wait on the trailing partial batch too + echo -n " loaded ${count}/${n} " + wait_loaded ${started} || { echo " timeout loading batch" >&2; exit 1; } + echo " ok" + fi + + echo "==> ${n} peers started" + + # A containerised client shares the compose network with the peers, so it must + # reach them at their container addresses; the published host ports only exist + # for a client running on the host. Read the addresses back from the running + # containers rather than pinning a subnet in the compose file: a pinned subnet + # is one more thing that can collide with whatever else the machine has up, + # VPNs included, and it buys nothing a lookup does not. + if [ -n "${CLIENT_NETEM}" ]; then + local ip csv="" + for svc in ${services}; do + ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "spv-bench-${svc}")" + [ -n "${ip}" ] || { echo "Error: could not resolve address of spv-bench-${svc}" >&2; exit 1; } + csv="${csv},${ip}:19400" + done + export BENCH_PEERS="${csv#,}" + echo "==> client will reach peers at ${BENCH_PEERS}" + fi +} + +build_bin() { + if [ -n "${CROSS_TRIPLE}" ]; then + # The client runs in a Linux container, so the binary has to be a Linux + # one. On a Linux host the ordinary build already is — same triple, and + # glibc is forward compatible — so only a non-Linux host comes here. + # Bind-mounts the workspace and builds into its `target/`, which cargo + # keeps under the triple, so this stays incremental — an image layer build + # would replay the whole workspace on every source change, which makes A/B + # runs unusable. + echo "==> cross-building bench binary (${RUST_IMAGE}, ${CROSS_TRIPLE})" + # As the invoking user, with CARGO_HOME inside the workspace: writing + # into the shared `target/` as root leaves artifacts the host build then + # cannot overwrite. + docker run --rm \ + -v "${REPO_ROOT}:/src" \ + -w /src \ + --user "$(id -u):$(id -g)" \ + -e CARGO_HOME=/src/target/.cross-cargo-home \ + -e CARGO_TARGET_DIR=/src/target \ + -e CARGO_NET_GIT_FETCH_WITH_CLI=true \ + -e CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ + "${RUST_IMAGE}" \ + cargo build --release --target "${CROSS_TRIPLE}" -p dash-spv-bench + else + echo "==> building bench binary (release + line-table symbols)" + ( cd "${REPO_ROOT}" && CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ + cargo build --release -p dash-spv-bench ) + fi + [ -x "${BIN}" ] || { echo "build produced no binary at ${BIN}" >&2; exit 1; } +} + +FLAME_TOOL="" +if [ "${FLAME}" -eq 1 ]; then # fail fast on missing profiler deps, before building + if command -v perf >/dev/null; then FLAME_TOOL=perf # Linux + elif command -v /usr/bin/sample >/dev/null; then FLAME_TOOL=sample # macOS + else echo "flame mode needs 'perf' (Linux) or '/usr/bin/sample' (macOS)" >&2; exit 1; fi + [ -f "${FLAMEGRAPH_DIR}/flamegraph.pl" ] || \ + git clone --depth 1 https://github.com/brendangregg/FlameGraph "${FLAMEGRAPH_DIR}" +fi + +build_bin + +if [ "${MODE}" = local ]; then + arm_teardown + bash "${SCRIPT_DIR}/snapshot-chain.sh" # builds ./chain-data to BENCH_HEIGHT via docker + bring_up +else + echo "==> ${MODE} mode, peers: ${BENCH_PEERS:-}" +fi + +# Run the sync with its live output straight on the terminal instead of through +# the batch wrapper's pipe +run_live() { + # Not `[ -w /dev/tty ]`: the device node is writable even with no + # controlling terminal, and the redirect then fails with ENXIO. Only + # actually opening it answers the question. + if { : >/dev/tty; } 2>/dev/null; then + "$@" >/dev/tty 2>&1 + else + "$@" + fi +} + +if [ -n "${CLIENT_NETEM}" ]; then + # `run` rather than `up`: it streams the client's own stdout and gives back + # its exit code, which is what the report is read from. Profiling is not + # wired through the container, so `--flame` is refused rather than silently + # producing a graph of the host doing nothing. + [ "${FLAME}" -eq 0 ] || { echo "Error: --flame is not supported with a containerised client" >&2; exit 1; } + arm_teardown + echo "==> running sync in the client container" + run_live compose run --rm --build client +elif [ "${FLAME}" -eq 0 ]; then + echo "==> running sync" + run_live "${CPU_PREFIX[@]+"${CPU_PREFIX[@]}"}" "${BIN}" +elif [ "${FLAME_TOOL}" = perf ]; then + echo "==> running sync under perf" + mkdir -p "$(dirname "${FLAME_SVG}")" + perf record -F 499 -g -o /tmp/bench-perf.data -- "${CPU_PREFIX[@]+"${CPU_PREFIX[@]}"}" "${BIN}" + perf script -i /tmp/bench-perf.data \ + | "${FLAMEGRAPH_DIR}/stackcollapse-perf.pl" \ + | "${FLAMEGRAPH_DIR}/flamegraph.pl" --title "dash-spv sync" --colors hot > "${FLAME_SVG}" + echo "==> wrote ${FLAME_SVG}" +else + echo "==> running sync under the sampler" + "${CPU_PREFIX[@]+"${CPU_PREFIX[@]}"}" "${BIN}" & bpid=$! + sleep 3 + /usr/bin/sample "${bpid}" 2000 1 -file /tmp/bench.sample.txt -mayDie >/dev/null 2>&1 || true + wait "${bpid}" 2>/dev/null || true + mkdir -p "$(dirname "${FLAME_SVG}")" + "${FLAMEGRAPH_DIR}/stackcollapse-sample.awk" /tmp/bench.sample.txt \ + | sed -E 's/^Thread_[^;]*;//' \ + | "${FLAMEGRAPH_DIR}/flamegraph.pl" --title "dash-spv sync" --colors hot > "${FLAME_SVG}" + echo "==> wrote ${FLAME_SVG}" +fi + +# Keep this run's outputs, which `bench-storage/` does not: it is wiped at the +# start of every run, so without this the trace of anything but the most recent +# scenario is gone — and in a batch that meant 15 of 16 syncs left nothing to +# look at afterwards. +# +# The batch passes its own directory in, so a child archives straight into it +# and there is one copy, named after the scenario. A lone run makes its own. +ARCHIVE_DIR="${BENCH_ARCHIVE_DIR:-${SCRIPT_DIR}/results/$(date +%Y%m%d-%H%M%S)-$(basename "${SCN_FILE}" .yml)}" +# Prefixed with the scenario only inside a batch, where one directory holds +# every scenario. A lone run's directory is already named after it. +ARCHIVE_NAME="${BENCH_ARCHIVE_NAME:-}" +mkdir -p "${ARCHIVE_DIR}" +for out in run.log summary.txt; do + [ -s "${BENCH_STORAGE_DIR}/${out}" ] || continue + cp "${BENCH_STORAGE_DIR}/${out}" "${ARCHIVE_DIR}/${ARCHIVE_NAME:+${ARCHIVE_NAME}.}${out}" +done +echo "==> logs archived to ${ARCHIVE_DIR}" diff --git a/dash-spv-bench/scenario.example.yml b/dash-spv-bench/scenario.example.yml new file mode 100644 index 000000000..d9b7dc2b2 --- /dev/null +++ b/dash-spv-bench/scenario.example.yml @@ -0,0 +1,52 @@ +description: "One-line summary of this scenario" # optional; shown in the report table. + +mode: local # "local" (docker peers you control), "testnet" or "mainnet" (real network). Default: local. + +cpus: "0-3" # optional. taskset CPU list pinning the MEASURED client (here cores 0,1,2,3). + # In local mode the docker peers get every OTHER core. Omit => no pinning. + +max_peers: 3 # optional. Max simultaneous peers. Omit => the ClientConfig default. + + +# --------------------------------------------------------------------------------------------- +# CLIENT link shaping (any mode) — shapes the MEASURED client's own connection. +# --------------------------------------------------------------------------------------------- +# Its presence moves the client into a container so `tc netem` can be applied to it; omit the +# whole section and the client runs on the host. +# client: +# latency_ms: 200 # added on the way out, so ~+200ms RTT against a real peer +# jitter_ms: 20 +# loss_pct: 0 +# rate_kbit: 8000 # the client's total bandwidth, shared across all peers +# corrupt_pct: 0 +# reorder_pct: 0 + + +# --------------------------------------------------------------------------------------------- +# LOCAL mode (mode: local) — docker dashd peers serving the frozen ./chain-data snapshot +# --------------------------------------------------------------------------------------------- +blocks: 1000000 # optional (default 1000000). The target HEIGHT: snapshot-chain.sh builds + # ./chain-data up to here (via docker) and the client syncs the full + # range genesis..blocks. Lower it for a shorter run. + +# Peer groups: one entry per group; each becomes `count` dashd containers sharing a connection +# quality (via `tc netem`). List several groups to mix good and bad peers in one network. +peers: + - count: 3 # required: how many peers in this group + latency_ms: 50 # optional: delay added on this peer's egress (netem delay) + jitter_ms: 0 # optional: delay jitter (netem delay ) + loss_pct: 0 # optional: packet loss % (netem loss) + rate_kbit: 0 # optional: bandwidth cap in kbit (netem rate) + corrupt_pct: 0 # optional: bit corruption % (netem corrupt) + reorder_pct: 0 # optional: packet reordering % (netem reorder) + # A second, degraded group ("bad peers"): + # - { count: 1, latency_ms: 800, loss_pct: 8, rate_kbit: 500 } + +# --------------------------------------------------------------------------------------------- +# TESTNET / MAINNET mode — connect to the real Dash network instead of docker peers. +# Set `mode: testnet` or `mode: mainnet` above, then use these instead of blocks/peers: +# --------------------------------------------------------------------------------------------- +# peers: # explicit nodes as "host:port" strings; omit for DNS discovery. +# - "1.2.3.4:9999" # (testnet uses :19999, mainnet :9999) +# - "5.6.7.8:9999" +# start_height: 2000000 # optional checkpoint: skip genesis..start and sync a bounded range. diff --git a/dash-spv-bench/scenarios/local.1ideal.yml b/dash-spv-bench/scenarios/local.1ideal.yml new file mode 100644 index 000000000..8fad585fd --- /dev/null +++ b/dash-spv-bench/scenarios/local.1ideal.yml @@ -0,0 +1,9 @@ +description: "Ideal link, single peer: 4 CPUs, 1 peer @ 50ms, full 1M sync." +cpus: "0-3" +repeats: 3 +peers: + - { count: 1, latency_ms: 50 } + +client: + latency_ms: 5 + rate_kbit: 150000 diff --git a/dash-spv-bench/scenarios/local.1slow-2ideal.yml b/dash-spv-bench/scenarios/local.1slow-2ideal.yml new file mode 100644 index 000000000..3363d349e --- /dev/null +++ b/dash-spv-bench/scenarios/local.1slow-2ideal.yml @@ -0,0 +1,10 @@ +description: "Two good peers plus one slow/lossy/capped peer (a bad peer in the set)." +cpus: "0-3" +repeats: 3 +peers: + - { count: 2, latency_ms: 50 } + - { count: 1, latency_ms: 800, loss_pct: 8, rate_kbit: 500 } + +client: + latency_ms: 5 + rate_kbit: 150000 diff --git a/dash-spv-bench/scenarios/local.3ideal.yml b/dash-spv-bench/scenarios/local.3ideal.yml new file mode 100644 index 000000000..c79ee67c2 --- /dev/null +++ b/dash-spv-bench/scenarios/local.3ideal.yml @@ -0,0 +1,9 @@ +description: "Ideal link: 4 CPUs, 3 peers @ 50ms, full 1M sync." +cpus: "0-3" +repeats: 3 +peers: + - { count: 3, latency_ms: 50 } + +client: + latency_ms: 5 + rate_kbit: 150000 diff --git a/dash-spv-bench/scenarios/local.5ideal.yml b/dash-spv-bench/scenarios/local.5ideal.yml new file mode 100644 index 000000000..8a256d33e --- /dev/null +++ b/dash-spv-bench/scenarios/local.5ideal.yml @@ -0,0 +1,9 @@ +description: "Ideal link, more peers: 4 CPUs, 5 peers @ 50ms, full 1M sync." +cpus: "0-3" +repeats: 3 +peers: + - { count: 5, latency_ms: 50 } + +client: + latency_ms: 5 + rate_kbit: 150000 diff --git a/dash-spv-bench/scenarios/local.5slow-congested.yml b/dash-spv-bench/scenarios/local.5slow-congested.yml new file mode 100644 index 000000000..97b596cf0 --- /dev/null +++ b/dash-spv-bench/scenarios/local.5slow-congested.yml @@ -0,0 +1,9 @@ +description: "CPU-starved client (2 cores), 5 peers on a jittery 300ms link." +cpus: "0-1" +repeats: 3 +peers: + - { count: 5, latency_ms: 300, jitter_ms: 100 } + +client: + latency_ms: 5 + rate_kbit: 150000 diff --git a/dash-spv-bench/scenarios/local.8slow.yml b/dash-spv-bench/scenarios/local.8slow.yml new file mode 100644 index 000000000..ccedf4947 --- /dev/null +++ b/dash-spv-bench/scenarios/local.8slow.yml @@ -0,0 +1,15 @@ +description: "Many slow peers: 8 peers @ 300ms/30ms jitter, 8 Mbit each, full 1M sync. Stresses per-peer cap growth — saturating a high-latency rate-capped link needs several in-flight requests per peer, so caps must climb well above the floor." +mode: local +cpus: "0-5" +max_peers: 8 +repeats: 2 +blocks: 1000000 +peers: + - count: 8 + latency_ms: 300 + jitter_ms: 30 + rate_kbit: 8000 + +client: + latency_ms: 5 + rate_kbit: 150000 diff --git a/dash-spv-bench/scenarios/mainnet.100mbi.1000ms.yml b/dash-spv-bench/scenarios/mainnet.100mbi.1000ms.yml new file mode 100644 index 000000000..0619b92ed --- /dev/null +++ b/dash-spv-bench/scenarios/mainnet.100mbi.1000ms.yml @@ -0,0 +1,7 @@ +description: "Mainnet from genesis with the client's own link capped at 100 Mbit each way and 1000ms of one-way delay (~+1000ms RTT). One cell of the bandwidth x latency matrix." +mode: mainnet +max_peers: 3 + +client: + latency_ms: 1000 + rate_kbit: 100000 diff --git a/dash-spv-bench/scenarios/mainnet.100mbi.100ms.yml b/dash-spv-bench/scenarios/mainnet.100mbi.100ms.yml new file mode 100644 index 000000000..be8f45b1e --- /dev/null +++ b/dash-spv-bench/scenarios/mainnet.100mbi.100ms.yml @@ -0,0 +1,7 @@ +description: "Mainnet from genesis with the client's own link capped at 100 Mbit each way and 100ms of one-way delay (~+100ms RTT). One cell of the bandwidth x latency matrix." +mode: mainnet +max_peers: 3 + +client: + latency_ms: 100 + rate_kbit: 100000 diff --git a/dash-spv-bench/scenarios/mainnet.100mbi.500ms.yml b/dash-spv-bench/scenarios/mainnet.100mbi.500ms.yml new file mode 100644 index 000000000..35114cfcd --- /dev/null +++ b/dash-spv-bench/scenarios/mainnet.100mbi.500ms.yml @@ -0,0 +1,7 @@ +description: "Mainnet from genesis with the client's own link capped at 100 Mbit each way and 500ms of one-way delay (~+500ms RTT). One cell of the bandwidth x latency matrix." +mode: mainnet +max_peers: 3 + +client: + latency_ms: 500 + rate_kbit: 100000 diff --git a/dash-spv-bench/scenarios/mainnet.50mbi.1000ms.yml b/dash-spv-bench/scenarios/mainnet.50mbi.1000ms.yml new file mode 100644 index 000000000..237f662cf --- /dev/null +++ b/dash-spv-bench/scenarios/mainnet.50mbi.1000ms.yml @@ -0,0 +1,7 @@ +description: "Mainnet from genesis with the client's own link capped at 50 Mbit each way and 1000ms of one-way delay (~+1000ms RTT). One cell of the bandwidth x latency matrix." +mode: mainnet +max_peers: 3 + +client: + latency_ms: 1000 + rate_kbit: 50000 diff --git a/dash-spv-bench/scenarios/mainnet.50mbi.100ms.yml b/dash-spv-bench/scenarios/mainnet.50mbi.100ms.yml new file mode 100644 index 000000000..657a3a93f --- /dev/null +++ b/dash-spv-bench/scenarios/mainnet.50mbi.100ms.yml @@ -0,0 +1,7 @@ +description: "Mainnet from genesis with the client's own link capped at 50 Mbit each way and 100ms of one-way delay (~+100ms RTT). One cell of the bandwidth x latency matrix." +mode: mainnet +max_peers: 3 + +client: + latency_ms: 100 + rate_kbit: 50000 diff --git a/dash-spv-bench/scenarios/mainnet.50mbi.500ms.yml b/dash-spv-bench/scenarios/mainnet.50mbi.500ms.yml new file mode 100644 index 000000000..8fdeaee46 --- /dev/null +++ b/dash-spv-bench/scenarios/mainnet.50mbi.500ms.yml @@ -0,0 +1,7 @@ +description: "Mainnet from genesis with the client's own link capped at 50 Mbit each way and 500ms of one-way delay (~+500ms RTT). One cell of the bandwidth x latency matrix." +mode: mainnet +max_peers: 3 + +client: + latency_ms: 500 + rate_kbit: 50000 diff --git a/dash-spv-bench/scenarios/mainnet.8mbi.1000ms.yml b/dash-spv-bench/scenarios/mainnet.8mbi.1000ms.yml new file mode 100644 index 000000000..8fcb87d4c --- /dev/null +++ b/dash-spv-bench/scenarios/mainnet.8mbi.1000ms.yml @@ -0,0 +1,7 @@ +description: "Mainnet from genesis with the client's own link capped at 8 Mbit each way and 1000ms of one-way delay (~+1000ms RTT). One cell of the bandwidth x latency matrix." +mode: mainnet +max_peers: 3 + +client: + latency_ms: 1000 + rate_kbit: 8000 diff --git a/dash-spv-bench/scenarios/mainnet.8mbi.100ms.yml b/dash-spv-bench/scenarios/mainnet.8mbi.100ms.yml new file mode 100644 index 000000000..64d8d557d --- /dev/null +++ b/dash-spv-bench/scenarios/mainnet.8mbi.100ms.yml @@ -0,0 +1,7 @@ +description: "Mainnet from genesis with the client's own link capped at 8 Mbit each way and 100ms of one-way delay (~+100ms RTT). One cell of the bandwidth x latency matrix." +mode: mainnet +max_peers: 3 + +client: + latency_ms: 100 + rate_kbit: 8000 diff --git a/dash-spv-bench/scenarios/mainnet.8mbi.500ms.yml b/dash-spv-bench/scenarios/mainnet.8mbi.500ms.yml new file mode 100644 index 000000000..d76b0eee2 --- /dev/null +++ b/dash-spv-bench/scenarios/mainnet.8mbi.500ms.yml @@ -0,0 +1,7 @@ +description: "Mainnet from genesis with the client's own link capped at 8 Mbit each way and 500ms of one-way delay (~+500ms RTT). One cell of the bandwidth x latency matrix." +mode: mainnet +max_peers: 3 + +client: + latency_ms: 500 + rate_kbit: 8000 diff --git a/dash-spv-bench/scenarios/testnet.yml b/dash-spv-bench/scenarios/testnet.yml new file mode 100644 index 000000000..fa0348c5b --- /dev/null +++ b/dash-spv-bench/scenarios/testnet.yml @@ -0,0 +1,3 @@ +description: "Testnet sync from genesis via DNS peer discovery." +max_peers: 3 +mode: testnet diff --git a/dash-spv-bench/snapshot-chain.sh b/dash-spv-bench/snapshot-chain.sh new file mode 100755 index 000000000..c6a873554 --- /dev/null +++ b/dash-spv-bench/snapshot-chain.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHAIN_DIR="${SCRIPT_DIR}/chain-data" +SUBDIR="testnet3" +DASH_VERSION="23.1.7" # keep in sync with run.sh +IMAGE="dash-spv-bench/dashd:${DASH_VERSION}" + +cd "${SCRIPT_DIR}" + +: "${BENCH_HEIGHT:?BENCH_HEIGHT must be set — run.sh exports it from the scenario blocks}" +command -v docker >/dev/null 2>&1 || { echo "Error: docker is required to build the snapshot." >&2; exit 1; } + +docker image inspect "${IMAGE}" >/dev/null 2>&1 \ + || { echo "==> building peer image" + docker build --build-arg "DASHVERSION=${DASH_VERSION}" \ + -t "${IMAGE}" -f "${SCRIPT_DIR}/Dockerfile" "${SCRIPT_DIR}"; } + +mkdir -p "${CHAIN_DIR}" + +echo "==> extending snapshot to height ${BENCH_HEIGHT} via docker (downloads only the missing blocks)..." + +docker run --rm --user "$(id -u):$(id -g)" -v "${CHAIN_DIR}:/data" "${IMAGE}" \ + dashd -testnet -datadir=/data -daemon=0 -server=1 \ + -blockfilterindex=1 -peerblockfilters=1 -stopatheight="${BENCH_HEIGHT}" \ + -txindex=0 -prune=0 -disablewallet=1 + +echo "==> done. Snapshot is at ${CHAIN_DIR}/${SUBDIR} (height ${BENCH_HEIGHT})." diff --git a/dash-spv-bench/src/dashboard.rs b/dash-spv-bench/src/dashboard.rs new file mode 100644 index 000000000..d8f0aa7ce --- /dev/null +++ b/dash-spv-bench/src/dashboard.rs @@ -0,0 +1,217 @@ +use std::io::{self, IsTerminal, Write}; + +use dash_spv::sync::{ProgressPercentage, SyncProgress}; +use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; +use tracing_subscriber::fmt::MakeWriter; + +const REDRAW_HZ: u8 = 10; +const BAR_WIDTH: usize = 32; +const NOT_ENABLED: &str = "not enabled"; + +pub struct Dashboard { + multi: MultiProgress, + block_headers: ProgressBar, + filter_headers: ProgressBar, + filters: ProgressBar, + masternodes: ProgressBar, + blocks: ProgressBar, + chainlocks: ProgressBar, + instantsend: ProgressBar, + mempool: ProgressBar, +} + +impl Dashboard { + pub fn new() -> Self { + let target = if io::stderr().is_terminal() { + ProgressDrawTarget::stderr_with_hz(REDRAW_HZ) + } else { + ProgressDrawTarget::hidden() + }; + let multi = MultiProgress::with_draw_target(target); + + let bar_style = ProgressStyle::with_template(&format!( + " {{prefix:<15}} [{{bar:{BAR_WIDTH}}}] {{percent:>3}}% {{msg}}" + )) + .expect("static template") + .progress_chars("=> "); + + let info_style = ProgressStyle::with_template(&format!( + " {{prefix:<15}} {:BAR_WIDTH$} {{msg}}", + "" + )) + .expect("static template"); + + let mk = |label: &'static str, style: &ProgressStyle| { + let pb = multi.add(ProgressBar::new(1)); + pb.set_style(style.clone()); + pb.set_prefix(label); + pb.set_message(NOT_ENABLED); + pb + }; + + Self { + block_headers: mk("block headers", &bar_style), + filter_headers: mk("filter headers", &bar_style), + filters: mk("filters", &bar_style), + masternodes: mk("masternodes", &bar_style), + blocks: mk("matched blocks", &bar_style), + chainlocks: mk("chainlocks", &info_style), + instantsend: mk("instantsend", &info_style), + mempool: mk("mempool", &info_style), + multi, + } + } + + pub fn log_writer(&self) -> BarWriter { + BarWriter(self.multi.clone()) + } + + pub fn render(&self, progress: &SyncProgress) { + fn set(pb: &ProgressBar, p: &impl ProgressPercentage) { + let (current, target) = (p.current_height(), p.target_height()); + pb.set_length(target.max(1) as u64); + pb.set_position(current.min(target) as u64); + pb.set_message(format!("{} / {}", current, target)); + } + + if let Ok(p) = progress.headers() { + set(&self.block_headers, p); + } + if let Ok(p) = progress.filter_headers() { + let (current, target) = (p.current_height(), p.target_height()); + self.filter_headers.set_length(target.max(1) as u64); + self.filter_headers.set_position(current.min(target) as u64); + self.filter_headers.set_message(format!( + "{} / {} (verified {})", + current, + target, + p.current_height() + )); + } + if let Ok(p) = progress.filters() { + set(&self.filters, p); + } + if let Ok(p) = progress.masternodes() { + let (current, target) = (p.current_height(), p.target_height()); + self.masternodes.set_length(target.max(1) as u64); + self.masternodes.set_position(current.min(target) as u64); + self.masternodes.set_message(format!( + "{} / {} ({} diffs, {} cycles)", + current, + target, + p.diffs_processed(), + p.validated_cycles() + )); + } + if let Ok(p) = progress.blocks() { + let tip = progress.headers().map(|h| h.target_height()).unwrap_or(0); + let done = p.last_processed(); + self.blocks.set_length(tip.max(1) as u64); + self.blocks.set_position(done.min(tip) as u64); + self.blocks.set_message(format!( + "{} / {} ({} blocks, {} txs)", + done, + tip, + p.processed(), + p.transactions() + )); + } + if let Ok(p) = progress.chainlocks() { + self.chainlocks.set_message(format!( + "{:?} best {} ({} valid, {} invalid)", + p.state(), + p.best_validated_height(), + p.valid(), + p.invalid() + )); + } + if let Ok(p) = progress.instantsend() { + self.instantsend.set_message(format!( + "{:?} {} valid, {} invalid, {} pending", + p.state(), + p.valid(), + p.invalid(), + p.pending() + )); + } + if let Ok(p) = progress.mempool() { + self.mempool.set_message(format!( + "{:?} {} tracked ({} relevant of {} seen)", + p.state(), + p.tracked(), + p.relevant(), + p.received() + )); + } + } + + pub fn finish(&self) { + for pb in [ + &self.block_headers, + &self.filter_headers, + &self.filters, + &self.masternodes, + &self.blocks, + &self.chainlocks, + &self.instantsend, + &self.mempool, + ] { + if pb.message() == NOT_ENABLED { + pb.abandon(); + } else { + pb.finish(); + } + } + } +} + +impl Default for Dashboard { + fn default() -> Self { + Self::new() + } +} + +#[derive(Clone)] +pub struct BarWriter(MultiProgress); + +impl<'a> MakeWriter<'a> for BarWriter { + type Writer = BarWriterGuard; + + fn make_writer(&'a self) -> Self::Writer { + BarWriterGuard { + multi: self.0.clone(), + buf: Vec::new(), + } + } +} + +pub struct BarWriterGuard { + multi: MultiProgress, + buf: Vec, +} + +impl Write for BarWriterGuard { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.buf.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + if self.buf.is_empty() { + return Ok(()); + } + let line = std::mem::take(&mut self.buf); + self.multi.suspend(|| { + let mut err = io::stderr().lock(); + let _ = err.write_all(&line); + let _ = err.flush(); + }); + Ok(()) + } +} + +impl Drop for BarWriterGuard { + fn drop(&mut self) { + let _ = self.flush(); + } +} diff --git a/dash-spv-bench/src/main.rs b/dash-spv-bench/src/main.rs new file mode 100644 index 000000000..2ee5d533a --- /dev/null +++ b/dash-spv-bench/src/main.rs @@ -0,0 +1,223 @@ +mod dashboard; +mod metrics; + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{anyhow, Context, Result}; +use dash_spv::storage::DiskStorageManager; +use dash_spv::{ClientConfig, DashSpvClient, EventHandler}; +use dashcore::Network; +use key_wallet::wallet::initialization::WalletAccountCreationOptions; +use key_wallet::wallet::managed_wallet_info::wallet_info_interface::WalletInfoInterface; +use key_wallet::wallet::ManagedWalletInfo; +use key_wallet_manager::WalletManager; +use tokio::sync::RwLock; +use tracing_subscriber::prelude::*; + +use crate::dashboard::Dashboard; +use crate::metrics::BenchEventHandler; + +fn env_or(key: &str, default: &str) -> String { + std::env::var(key).unwrap_or_else(|_| default.to_string()) +} + +fn load_mnemonics() -> Vec { + std::env::var("BENCH_WALLET_FILE") + .ok() + .filter(|p| !p.trim().is_empty()) + .and_then(|path| std::fs::read_to_string(&path).ok()) + .map(|contents| { + contents + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(String::from) + .collect() + }) + .unwrap_or_default() +} + +#[tokio::main] +async fn main() -> Result<()> { + let dashboard = Arc::new(Dashboard::new()); + + let (output_dir, _tempdir_guard): (PathBuf, Option) = + match std::env::var("BENCH_STORAGE_DIR") { + Ok(dir) if !dir.trim().is_empty() => { + let path = PathBuf::from(dir.trim()); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).context("create bench storage dir")?; + (path, None) + } + _ => { + let td = tempfile::tempdir().context("temp storage dir")?; + (td.path().to_path_buf(), Some(td)) + } + }; + + let log_file = std::fs::File::create(output_dir.join("run.log")).context("create run.log")?; + let terminal_filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn,dash_spv_bench=info")); + let file_filter = tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + tracing_subscriber::EnvFilter::new("warn,dash_spv=debug,dash_spv_bench=debug") + }); + tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::layer() + .with_writer(dashboard.log_writer()) + .with_filter(terminal_filter), + ) + .with( + tracing_subscriber::fmt::layer() + .with_ansi(false) + .with_writer(std::sync::Mutex::new(log_file)) + .with_filter(file_filter), + ) + .init(); + + let mode = env_or("BENCH_MODE", "local").trim().to_ascii_lowercase(); + let (network, remote) = match mode.as_str() { + "local" => (Network::Testnet, false), + "testnet" => (Network::Testnet, true), + "mainnet" => (Network::Mainnet, true), + other => { + return Err(anyhow!( + "BENCH_MODE must be 'local', 'testnet' or 'mainnet', got '{other}'" + )) + } + }; + + let peers: Vec = std::env::var("BENCH_PEERS") + .unwrap_or_default() + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|p| p.parse::().with_context(|| format!("bad peer {p}"))) + .collect::>()?; + + if !remote && peers.is_empty() { + return Err(anyhow!("BENCH_MODE=local requires BENCH_PEERS (the docker peer addresses)")); + } + + let dns_mode = remote && peers.is_empty(); + + let start_height: Option = + std::env::var("BENCH_START_HEIGHT").ok().and_then(|v| v.trim().parse().ok()); + + let mnemonics = load_mnemonics(); + let timeout = Duration::from_secs(1800); + + tracing::info!( + "dash-spv-bench: mode={} ({}) on {:?}, wallets={}", + mode, + if dns_mode { + "DNS discovery".to_string() + } else { + format!("{} peers", peers.len()) + }, + network, + mnemonics.len(), + ); + + let mut config = ClientConfig::new(network).with_storage_path(output_dir.clone()); + config.start_from_height = start_height; + config.restrict_to_configured_peers = !dns_mode; + if let Ok(v) = std::env::var("BENCH_MAX_PEERS") { + if let Ok(n) = v.trim().parse() { + config.max_peers = n; + } + } + for addr in &peers { + config.add_peer(*addr); + } + + let storage_manager = + DiskStorageManager::new(&config).await.map_err(|e| anyhow!("storage manager: {e}"))?; + + let mut wallet_manager = WalletManager::::new(network); + let birth_height = start_height.unwrap_or(0); + let mut wallet_ids = Vec::with_capacity(mnemonics.len()); + for mnemonic in &mnemonics { + let id = wallet_manager + .create_wallet_from_mnemonic( + mnemonic, + birth_height, + WalletAccountCreationOptions::default(), + ) + .map_err(|e| anyhow!("wallet from mnemonic: {e}"))?; + wallet_ids.push(id); + } + let wallet = Arc::new(RwLock::new(wallet_manager)); + let wallet_probe = wallet.clone(); + + let handler = Arc::new(BenchEventHandler::new(dashboard.clone())); + let network = dash_spv::network::PeerNetworkManager::new(&config) + .await + .map_err(|e| anyhow!("network new: {e}"))?; + + let client = DashSpvClient::new( + config, + network, + storage_manager, + wallet, + vec![handler.clone() as Arc], + ) + .await + .map_err(|e| anyhow!("client new: {e}"))?; + + let run_client = client.clone(); + let run_handle = tokio::spawn(async move { + if let Err(e) = run_client.run().await { + tracing::error!("client run() exited with error: {e}"); + } + }); + + tokio::select! { + _ = handler.wait_done() => {} + _ = tokio::time::sleep(timeout) => {} + _ = tokio::signal::ctrl_c() => eprintln!("ctrl-c received, shutting down..."), + } + let m = handler.snapshot(); + + let _ = client.shutdown().await; + run_handle.abort(); + let _ = run_handle.await; + + use std::fmt::Write as _; + let mut report = format!("{m}\n"); + + if !wallet_ids.is_empty() { + use key_wallet_manager::WalletInterface; + let w = wallet_probe.read().await; + let _ = writeln!(report, "wallet_synced_height: {}", w.synced_height()); + let _ = writeln!(report, "wallet_addresses: {}", w.monitored_addresses().len()); + + for (i, id) in wallet_ids.iter().enumerate() { + let txs: usize = w + .get_wallet_info(id) + .map(|info| info.accounts().all_accounts().iter().map(|a| a.tx_count()).sum()) + .unwrap_or(0); + + let (confirmed, total) = + w.get_wallet_balance(id).map(|b| (b.confirmed(), b.total())).unwrap_or((0, 0)); + + let _ = writeln!( + report, + "wallet[{i}] transactions={txs} confirmed_sat={confirmed} total_sat={total}" + ); + } + + let _ = writeln!(report, "wallet_describe:\n{}", w.describe().await); + } + + print!("\n{report}"); + if let Err(e) = std::fs::write(output_dir.join("summary.txt"), &report) { + tracing::warn!("could not write summary.txt: {e}"); + } + tracing::info!("run outputs written to {}", output_dir.display()); + + Ok(()) +} diff --git a/dash-spv-bench/src/metrics.rs b/dash-spv-bench/src/metrics.rs new file mode 100644 index 000000000..6b8ea8341 --- /dev/null +++ b/dash-spv-bench/src/metrics.rs @@ -0,0 +1,167 @@ +use std::collections::BTreeMap; +use std::fmt; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use dash_spv::sync::{SyncEvent, SyncProgress}; +use dash_spv::EventHandler; +use tokio::sync::Notify; + +use crate::dashboard::Dashboard; + +pub const MILESTONE_BLOCK_HEADERS: &str = "block_headers_complete"; +pub const MILESTONE_FILTER_HEADERS: &str = "filter_headers_complete"; +pub const MILESTONE_FILTERS: &str = "filters_complete"; +pub const MILESTONE_SYNC: &str = "sync_complete"; + +#[derive(Debug)] +pub struct RunMetrics { + total_ms: u64, + completed: bool, + block_headers_ms: Option, + filter_headers_ms: Option, + filters_ms: Option, + transactions: u32, + error: Option, +} + +impl fmt::Display for RunMetrics { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let ms = |o: Option| o.map(|v| v.to_string()).unwrap_or_else(|| "-".to_string()); + + writeln!(f, "=== dash-spv sync ===")?; + writeln!( + f, + "completed: {}{}", + self.completed, + if self.completed { + "" + } else { + " (TIMED OUT)" + } + )?; + writeln!(f, "total_ms: {}", self.total_ms)?; + writeln!(f, "block_headers_ms: {}", ms(self.block_headers_ms))?; + writeln!(f, "filter_headers_ms: {}", ms(self.filter_headers_ms))?; + writeln!(f, "filters_ms: {}", ms(self.filters_ms))?; + write!(f, "transactions: {}", self.transactions)?; + if let Some(e) = &self.error { + write!(f, "\nerror: {e}")?; + } + + Ok(()) + } +} + +pub struct BenchEventHandler { + start: Instant, + inner: Mutex, + done: Notify, + dashboard: Arc, +} + +struct Inner { + milestones: BTreeMap<&'static str, u64>, + error: Option, + completed: bool, + transactions: u32, +} + +impl BenchEventHandler { + pub fn new(dashboard: Arc) -> Self { + Self { + start: Instant::now(), + inner: Mutex::new(Inner { + milestones: BTreeMap::new(), + error: None, + completed: false, + transactions: 0, + }), + done: Notify::new(), + dashboard, + } + } + + fn record(&self, label: &'static str) { + let elapsed = self.start.elapsed().as_millis() as u64; + let mut inner = self.inner.lock().unwrap(); + inner.milestones.entry(label).or_insert(elapsed); + } + + pub async fn wait_done(&self) { + let notified = self.done.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + + if self.inner.lock().unwrap().completed { + return; + } + + notified.await; + } + + pub fn snapshot(&self) -> RunMetrics { + let inner = self.inner.lock().unwrap(); + let bh = inner.milestones.get(MILESTONE_BLOCK_HEADERS).copied(); + let fh = inner.milestones.get(MILESTONE_FILTER_HEADERS).copied(); + let fl = inner.milestones.get(MILESTONE_FILTERS).copied(); + let sc = inner.milestones.get(MILESTONE_SYNC).copied(); + + RunMetrics { + total_ms: sc.unwrap_or_else(|| self.start.elapsed().as_millis() as u64), + completed: inner.completed, + block_headers_ms: bh, + filter_headers_ms: fh, + filters_ms: fl, + transactions: inner.transactions, + error: inner.error.clone(), + } + } +} + +impl EventHandler for BenchEventHandler { + fn on_sync_event(&self, event: &SyncEvent) { + match event { + SyncEvent::BlockHeaderSyncComplete { + .. + } => self.record(MILESTONE_BLOCK_HEADERS), + SyncEvent::FilterHeadersSyncComplete { + .. + } => self.record(MILESTONE_FILTER_HEADERS), + SyncEvent::FiltersSyncComplete { + .. + } => self.record(MILESTONE_FILTERS), + SyncEvent::SyncComplete { + .. + } => { + self.record(MILESTONE_SYNC); + self.dashboard.finish(); + let mut inner = self.inner.lock().unwrap(); + inner.completed = true; + drop(inner); + self.done.notify_waiters(); + } + _ => {} + } + } + + /// Every progress change repaints the bars (throttled inside). + fn on_progress(&self, progress: &SyncProgress) { + if let Ok(blocks) = progress.blocks() { + let seen = blocks.transactions(); + let mut inner = self.inner.lock().unwrap(); + // Highest seen rather than last: the final repaint is whichever + // progress change happened to land last, not necessarily the one + // carrying the highest count. + inner.transactions = inner.transactions.max(seen); + } + self.dashboard.render(progress); + } + + fn on_error(&self, error: &str) { + let mut inner = self.inner.lock().unwrap(); + if inner.error.is_none() { + inner.error = Some(error.to_string()); + } + } +} diff --git a/dash-spv-ffi/FFI_API.md b/dash-spv-ffi/FFI_API.md index 5b708ea8b..2e81eea7f 100644 --- a/dash-spv-ffi/FFI_API.md +++ b/dash-spv-ffi/FFI_API.md @@ -4,7 +4,7 @@ This document provides a comprehensive reference for all FFI (Foreign Function I **Auto-generated**: This documentation is automatically generated from the source code. Do not edit manually. -**Total Functions**: 39 +**Total Functions**: 44 ## Table of Contents @@ -31,7 +31,7 @@ Functions: 3 ### Configuration -Functions: 15 +Functions: 19 | Function | Description | Module | |----------|-------------|--------| @@ -41,6 +41,10 @@ Functions: 15 | `dash_spv_ffi_config_get_network` | Gets the network type from the configuration # Safety - `config` must be a... | config | | `dash_spv_ffi_config_mainnet` | No description | config | | `dash_spv_ffi_config_new` | No description | config | +| `dash_spv_ffi_config_set_broadcast_acceptance_threshold` | Sets how many distinct non-recipient peers must announce a broadcast txid... | config | +| `dash_spv_ffi_config_set_broadcast_acceptance_timeout_secs` | Sets the timeout (in seconds) after which a pending broadcast with no... | config | +| `dash_spv_ffi_config_set_broadcast_holdout_count` | Withholds broadcast transactions from a fixed number of peers (clamped so... | config | +| `dash_spv_ffi_config_set_broadcast_holdout_half` | Withholds broadcast transactions from half of the connected peers (the... | config | | `dash_spv_ffi_config_set_data_dir` | Sets the data directory for storing blockchain data # Safety - `config`... | config | | `dash_spv_ffi_config_set_fetch_mempool_transactions` | Sets whether to fetch full mempool transaction data # Safety - `config`... | config | | `dash_spv_ffi_config_set_masternode_sync_enabled` | Enables or disables masternode synchronization # Safety - `config` must be... | config | @@ -63,11 +67,12 @@ Functions: 3 ### Transaction Management -Functions: 1 +Functions: 2 | Function | Description | Module | |----------|-------------|--------| | `dash_spv_ffi_client_broadcast_transaction` | Broadcasts a transaction to the Dash network via connected peers | client | +| `dash_spv_ffi_client_broadcast_transaction_and_wait` | Broadcasts a transaction and waits for its network-level outcome | client | ### Mempool Operations @@ -252,6 +257,70 @@ dash_spv_ffi_config_new(network: FFINetwork) -> *mut FFIClientConfig --- +#### `dash_spv_ffi_config_set_broadcast_acceptance_threshold` + +```c +dash_spv_ffi_config_set_broadcast_acceptance_threshold(config: *mut FFIClientConfig, threshold: u32,) -> i32 +``` + +**Description:** +Sets how many distinct non-recipient peers must announce a broadcast txid back before it is reported as accepted. Must be > 0. # Safety - `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet - The caller must ensure the config pointer remains valid for the duration of this call + +**Safety:** +- `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet - The caller must ensure the config pointer remains valid for the duration of this call + +**Module:** `config` + +--- + +#### `dash_spv_ffi_config_set_broadcast_acceptance_timeout_secs` + +```c +dash_spv_ffi_config_set_broadcast_acceptance_timeout_secs(config: *mut FFIClientConfig, seconds: u32,) -> i32 +``` + +**Description:** +Sets the timeout (in seconds) after which a pending broadcast with no acceptance signal is reported as uncertain. Must be > 0. # Safety - `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet - The caller must ensure the config pointer remains valid for the duration of this call + +**Safety:** +- `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet - The caller must ensure the config pointer remains valid for the duration of this call + +**Module:** `config` + +--- + +#### `dash_spv_ffi_config_set_broadcast_holdout_count` + +```c +dash_spv_ffi_config_set_broadcast_holdout_count(config: *mut FFIClientConfig, count: u32,) -> i32 +``` + +**Description:** +Withholds broadcast transactions from a fixed number of peers (clamped so that at least one peer always receives the transaction). # Safety - `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet - The caller must ensure the config pointer remains valid for the duration of this call + +**Safety:** +- `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet - The caller must ensure the config pointer remains valid for the duration of this call + +**Module:** `config` + +--- + +#### `dash_spv_ffi_config_set_broadcast_holdout_half` + +```c +dash_spv_ffi_config_set_broadcast_holdout_half(config: *mut FFIClientConfig,) -> i32 +``` + +**Description:** +Withholds broadcast transactions from half of the connected peers (the default policy). The withheld peers are the source of the `inv` echo that proves a broadcast propagated through the network. # Safety - `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet - The caller must ensure the config pointer remains valid for the duration of this call + +**Safety:** +- `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet - The caller must ensure the config pointer remains valid for the duration of this call + +**Module:** `config` + +--- + #### `dash_spv_ffi_config_set_data_dir` ```c @@ -458,6 +527,22 @@ Broadcasts a transaction to the Dash network via connected peers. # Safety - ` --- +#### `dash_spv_ffi_client_broadcast_transaction_and_wait` + +```c +dash_spv_ffi_client_broadcast_transaction_and_wait(client: *mut FFIDashSpvClient, tx_bytes: *const u8, length: usize, timeout_secs: u32, out_result: *mut FFIBroadcastResult,) -> i32 +``` + +**Description:** +Broadcasts a transaction and waits for its network-level outcome. Blocks until the network accepts the transaction (non-recipient peers announce it back, it is InstantSend-locked, or confirmed) or the timeout elapses (outcome `Uncertain`). `timeout_secs == 0` uses the configured acceptance timeout plus a small grace period. Requires mempool tracking to be enabled in the client config. # Safety - `client` must be a valid, non-null pointer to an initialized FFIDashSpvClient - `tx_bytes` must be a valid, non-null pointer to the transaction data - `length` must be the length of the transaction data in bytes - `out_result` must be a valid, non-null pointer to an FFIBroadcastResult + +**Safety:** +- `client` must be a valid, non-null pointer to an initialized FFIDashSpvClient - `tx_bytes` must be a valid, non-null pointer to the transaction data - `length` must be the length of the transaction data in bytes - `out_result` must be a valid, non-null pointer to an FFIBroadcastResult + +**Module:** `client` + +--- + ### Mempool Operations - Detailed #### `dash_spv_ffi_mempool_progress_destroy` diff --git a/dash-spv-ffi/src/bin/ffi_cli.rs b/dash-spv-ffi/src/bin/ffi_cli.rs index fa32ddb87..a9715bbf5 100644 --- a/dash-spv-ffi/src/bin/ffi_cli.rs +++ b/dash-spv-ffi/src/bin/ffi_cli.rs @@ -133,6 +133,27 @@ extern "C" fn on_sync_complete(header_tip: u32, cycle: u32, _user_data: *mut c_v println!("[Sync] Sync complete at height: {} (cycle {})", header_tip, cycle); } +extern "C" fn on_transaction_broadcast_result( + txid: *const [u8; 32], + status: FFIBroadcastStatus, + relayed_by: u32, + _user_data: *mut c_void, +) { + let txid_hex = unsafe { &*txid }.iter().rev().fold(String::new(), |mut acc, b| { + use std::fmt::Write; + let _ = write!(acc, "{:02x}", b); + acc + }); + match status { + FFIBroadcastStatus::Accepted => { + println!("[Broadcast] {} accepted ({} peer(s) relayed it back)", txid_hex, relayed_by) + } + FFIBroadcastStatus::Uncertain => { + println!("[Broadcast] {} outcome uncertain (no network signal)", txid_hex) + } + } +} + // ============================================================================ // Network Event Callbacks // ============================================================================ @@ -205,6 +226,46 @@ extern "C" fn on_transaction_detected( ); } +extern "C" fn on_transactions_swept( + wallet_id: *const c_char, + txids: *const [u8; 32], + txids_count: usize, + superseded_by: *const [u8; 32], + released_outpoints: *const dash_spv_ffi::FFIOutPoint, + released_outpoints_count: usize, + balance: *const FFIBalance, + _account_balances: *const dash_spv_ffi::FFIAccountBalance, + _account_balances_count: u32, + _user_data: *mut c_void, +) { + let wallet_short = short_wallet(wallet_id); + if txids.is_null() || superseded_by.is_null() { + println!("[Wallet] TXs swept: wallet={}..., null payload", wallet_short); + return; + } + let list = unsafe { std::slice::from_raw_parts(txids, txids_count) }; + let winner = unsafe { &*superseded_by }; + let released = if released_outpoints.is_null() { + &[][..] + } else { + unsafe { std::slice::from_raw_parts(released_outpoints, released_outpoints_count) } + }; + let b = read_balance(balance); + println!( + "[Wallet] TXs swept: wallet={}..., removed=[{}], superseded_by={}, released=[{}], balance[confirmed={}, unconfirmed={}]", + wallet_short, + list.iter().map(hex::encode).collect::>().join(","), + hex::encode(winner), + released + .iter() + .map(|o| format!("{}:{}", hex::encode(o.txid), o.vout)) + .collect::>() + .join(","), + b.confirmed, + b.unconfirmed, + ); +} + extern "C" fn on_transaction_instant_locked( wallet_id: *const c_char, txid: *const [u8; 32], @@ -511,6 +572,7 @@ fn main() { on_instantlock_received: Some(on_instantlock_received), on_manager_error: Some(on_manager_error), on_sync_complete: Some(on_sync_complete), + on_transaction_broadcast_result: Some(on_transaction_broadcast_result), user_data: ptr::null_mut(), }, network: FFINetworkEventCallbacks { @@ -526,6 +588,7 @@ fn main() { wallet: FFIWalletEventCallbacks { on_transaction_detected: Some(on_transaction_detected), on_transaction_instant_locked: Some(on_transaction_instant_locked), + on_transactions_swept: Some(on_transactions_swept), on_block_processed: Some(on_wallet_block_processed), on_sync_height_advanced: Some(on_sync_height_advanced), on_chain_lock_processed: Some(on_wallet_chain_lock_processed), diff --git a/dash-spv-ffi/src/callbacks.rs b/dash-spv-ffi/src/callbacks.rs index ff1d1c765..bb01d1d05 100644 --- a/dash-spv-ffi/src/callbacks.rs +++ b/dash-spv-ffi/src/callbacks.rs @@ -226,6 +226,35 @@ pub type OnManagerErrorCallback = pub type OnSyncCompleteCallback = Option; +/// Network-level outcome of a transaction broadcast. +/// +/// There is no rejected state: modern Dash Core removed the BIP61 `reject` +/// message, so the p2p network provides no negative signal — a transaction +/// the network refuses surfaces as `Uncertain` (no echo within the timeout). +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FFIBroadcastStatus { + /// Non-recipient peers announced the txid back (or it was + /// InstantSend-locked/confirmed) — the network accepted it. + Accepted = 0, + /// No acceptance signal arrived within the acceptance timeout. + Uncertain = 1, +} + +/// Callback for SyncEvent::TransactionBroadcastResult +/// +/// The `txid` pointer is borrowed and only valid for the duration of the +/// callback. `relayed_by` is the number of distinct non-recipient peers that +/// announced the txid back (meaningful for `Accepted`). +pub type OnTransactionBroadcastResultCallback = Option< + extern "C" fn( + txid: *const [u8; 32], + status: FFIBroadcastStatus, + relayed_by: u32, + user_data: *mut c_void, + ), +>; + /// Sync event callbacks - one callback per SyncEvent variant. /// /// Set only the callbacks you're interested in; unset callbacks will be ignored. @@ -250,6 +279,7 @@ pub struct FFISyncEventCallbacks { pub on_instantlock_received: OnInstantLockReceivedCallback, pub on_manager_error: OnManagerErrorCallback, pub on_sync_complete: OnSyncCompleteCallback, + pub on_transaction_broadcast_result: OnTransactionBroadcastResultCallback, pub user_data: *mut c_void, } @@ -282,6 +312,7 @@ impl Default for FFISyncEventCallbacks { on_instantlock_received: None, on_manager_error: None, on_sync_complete: None, + on_transaction_broadcast_result: None, user_data: std::ptr::null_mut(), } } @@ -438,6 +469,22 @@ impl FFISyncEventCallbacks { cb(*header_tip, *cycle, self.user_data); } } + SyncEvent::TransactionBroadcastResult { + txid, + result, + } => { + if let Some(cb) = self.on_transaction_broadcast_result { + use dash_spv::BroadcastResult; + let txid_bytes = txid.as_byte_array(); + let (status, relayed_by) = match result { + BroadcastResult::Accepted { + relayed_by, + } => (FFIBroadcastStatus::Accepted, *relayed_by as u32), + BroadcastResult::Uncertain => (FFIBroadcastStatus::Uncertain, 0), + }; + cb(txid_bytes as *const [u8; 32], status, relayed_by, self.user_data); + } + } } } } @@ -705,6 +752,78 @@ pub type OnTransactionDetectedCallback = Option< ), >; +/// C representation of a Core [`OutPoint`](dashcore::OutPoint): the parent +/// txid and output index of a coin. +#[repr(C)] +pub struct FFIOutPoint { + /// Parent transaction id. + pub txid: [u8; 32], + /// Output index within the parent transaction. + pub vout: u32, +} + +impl FFIOutPoint { + fn from_slice(outpoints: &[dashcore::OutPoint]) -> Vec { + outpoints + .iter() + .map(|outpoint| FFIOutPoint { + txid: *outpoint.txid.as_byte_array(), + vout: outpoint.vout, + }) + .collect() + } +} + +/// Callback for `WalletEvent::TransactionsSwept`. +/// +/// Fires when the wallet removes transactions that a later, final transaction +/// provably beat to one of their inputs: they can never confirm, so their +/// outputs are gone from the UTXO set and their records deleted. +/// +/// **The only removal-shaped wallet callback.** Every other one is additive, +/// so a consumer mirroring wallet state to disk must act on this — delete the +/// named transactions and any UTXO they created. Ignoring it leaves the dead +/// rows in the mirror, which replays them on the next load and re-creates a +/// balance the wallet has already corrected. +/// +/// `txids` points to `txids_count` consecutive 32-byte txids. +/// `superseded_by` is the transaction whose arrival settled the inputs. +/// `released_outpoints` points to `released_outpoints_count` outpoints freed +/// by the removal: inputs the removed transactions claimed to spend that no +/// surviving record spends too. Mark these coins spendable again. This is +/// not the same set as `txids`' inputs — a loser spending A+B against a +/// winner spending only A leaves A marked and frees only B — and it cannot +/// be recomputed from `txids` on the consumer side: `superseded_by` need not +/// be wallet-relevant at all (it can spend our coin while paying only +/// external addresses), so it may never appear in any other callback. Null +/// with a zero count when the removal released nothing. +/// +/// The Rust `WalletEvent::TransactionsSwept` additionally carries +/// `winner_mined_height` (the winner's block height, `None` for an +/// InstantSend-locked winner not yet mined). It is deliberately not +/// forwarded here: this signature has no safely-degrading insertion point +/// for hand-declared C consumers (see the released-outpoints addition), +/// so growing it is a flagged breaking change deferred until a C consumer +/// needs the field. +/// All pointer parameters are borrowed and only valid for the duration of the +/// callback. `balance` is the wallet's balance *after* the removal; +/// `account_balances` follows the same contract as on +/// [`OnTransactionDetectedCallback`]. +pub type OnTransactionsSweptCallback = Option< + extern "C" fn( + wallet_id: *const c_char, + txids: *const [u8; 32], + txids_count: usize, + superseded_by: *const [u8; 32], + released_outpoints: *const FFIOutPoint, + released_outpoints_count: usize, + balance: *const FFIBalance, + account_balances: *const FFIAccountBalance, + account_balances_count: u32, + user_data: *mut c_void, + ), +>; + /// Callback for `WalletEvent::TransactionInstantLocked`. /// /// Fires when an InstantSend lock is applied to a previously-seen off-chain @@ -862,6 +981,7 @@ pub type OnWalletChainLockProcessedCallback = Option< pub struct FFIWalletEventCallbacks { pub on_transaction_detected: OnTransactionDetectedCallback, pub on_transaction_instant_locked: OnTransactionInstantLockedCallback, + pub on_transactions_swept: OnTransactionsSweptCallback, pub on_block_processed: OnWalletBlockProcessedCallback, pub on_sync_height_advanced: OnSyncHeightAdvancedCallback, pub on_chain_lock_processed: OnWalletChainLockProcessedCallback, @@ -877,6 +997,7 @@ impl Default for FFIWalletEventCallbacks { Self { on_transaction_detected: None, on_transaction_instant_locked: None, + on_transactions_swept: None, on_block_processed: None, on_sync_height_advanced: None, on_chain_lock_processed: None, @@ -974,6 +1095,72 @@ impl FFIWalletEventCallbacks { /// Dispatch a WalletEvent to the appropriate callback. pub fn dispatch(&self, event: &WalletEvent) { match event { + WalletEvent::TransactionsSwept { + wallet_id, + txids, + superseded_by, + // Deliberately not forwarded across the C ABI. #962 + // established that `OnTransactionsSweptCallback` has no + // safely-degrading insertion point: a C consumer declaring + // the function pointer by hand keeps compiling against a + // grown parameter list and reads shifted arguments. Rust + // consumers of `WalletEvent` get the field directly; the C + // surface stays byte-for-byte identical until a consumer + // needs it there, at which point the addition must be a + // flagged breaking change like #962's. + winner_mined_height: _, + released_outpoints, + balance, + account_balances, + } => { + if let Some(cb) = self.on_transactions_swept { + let wallet_id_hex = hex::encode(wallet_id); + let c_wallet_id = CString::new(wallet_id_hex).unwrap_or_default(); + let raw_txids: Vec<[u8; 32]> = + txids.iter().map(|t| t.to_byte_array()).collect(); + let raw_superseded_by = superseded_by.to_byte_array(); + let ffi_released_outpoints = FFIOutPoint::from_slice(released_outpoints); + let released_outpoints_ptr = if ffi_released_outpoints.is_empty() { + ptr::null() + } else { + ffi_released_outpoints.as_ptr() + }; + let ffi_balance = FFIBalance::from(*balance); + let ffi_account_balances = FFIAccountBalance::from_map(account_balances); + let account_balances_ptr = if ffi_account_balances.is_empty() { + ptr::null() + } else { + ffi_account_balances.as_ptr() + }; + + cb( + c_wallet_id.as_ptr(), + raw_txids.as_ptr(), + raw_txids.len(), + &raw_superseded_by as *const [u8; 32], + released_outpoints_ptr, + ffi_released_outpoints.len(), + &ffi_balance as *const FFIBalance, + account_balances_ptr, + ffi_account_balances.len() as u32, + self.user_data, + ); + + drop(ffi_account_balances); + drop(ffi_released_outpoints); + } else { + // Deliberately loud: every other wallet callback is + // additive, so a consumer that leaves this one unset keeps + // transactions the wallet has already dropped. + tracing::warn!( + wallet_id = %hex::encode(wallet_id), + swept = txids.len(), + %superseded_by, + "no on_transactions_swept callback set; the consumer will keep \ + mirroring transactions the wallet removed" + ); + } + } WalletEvent::TransactionDetected { wallet_id, record, @@ -1190,6 +1377,117 @@ mod tests { use std::collections::{BTreeMap, BTreeSet}; use std::sync::atomic::{AtomicU32, Ordering}; + /// A sweep with nothing released must hand the callback `null` and `0`, + /// not a dangling pointer into an empty `Vec`. + /// + /// This is the layer where getting it wrong does not fail an assertion: + /// a consumer reading `count` first sees zero and stops, but one that + /// dereferences the pointer defensively would read whatever the empty + /// allocation points at. The ordinary resend — the winner takes every + /// input the loser named — releases nothing, so this is the common case, + /// not the edge one. + #[test] + fn test_transactions_swept_dispatch_passes_null_when_nothing_released() { + static RELEASED_PTR_WAS_NULL: AtomicU32 = AtomicU32::new(u32::MAX); + static RELEASED_COUNT: AtomicU32 = AtomicU32::new(u32::MAX); + + extern "C" fn cb( + _wallet_id: *const c_char, + _txids: *const [u8; 32], + _txids_count: usize, + _superseded_by: *const [u8; 32], + released: *const FFIOutPoint, + released_count: usize, + _balance: *const FFIBalance, + _account_balances: *const FFIAccountBalance, + _account_balances_count: u32, + _user: *mut c_void, + ) { + RELEASED_PTR_WAS_NULL.store(u32::from(released.is_null()), Ordering::SeqCst); + RELEASED_COUNT.store(released_count as u32, Ordering::SeqCst); + } + + let callbacks = FFIWalletEventCallbacks { + on_transactions_swept: Some(cb), + ..FFIWalletEventCallbacks::default() + }; + + callbacks.dispatch(&WalletEvent::TransactionsSwept { + wallet_id: [7u8; 32], + txids: vec![Txid::from_byte_array([1u8; 32])], + superseded_by: Txid::from_byte_array([2u8; 32]), + winner_mined_height: Some(1_000), + released_outpoints: Vec::new(), + balance: WalletCoreBalance::default(), + account_balances: BTreeMap::new(), + }); + + assert_eq!(RELEASED_PTR_WAS_NULL.load(Ordering::SeqCst), 1, "expected a null pointer"); + assert_eq!(RELEASED_COUNT.load(Ordering::SeqCst), 0); + } + + /// The released outpoints must arrive intact and in order, and the + /// parameters after them must not be shifted by their insertion — the + /// balance a consumer reads has to be the balance, not an outpoint. + #[test] + fn test_transactions_swept_dispatch_marshals_released_outpoints() { + static FIRST_TXID: std::sync::Mutex<[u8; 32]> = std::sync::Mutex::new([0u8; 32]); + static VOUTS: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + static CONFIRMED: AtomicU32 = AtomicU32::new(u32::MAX); + + extern "C" fn cb( + _wallet_id: *const c_char, + _txids: *const [u8; 32], + _txids_count: usize, + _superseded_by: *const [u8; 32], + released: *const FFIOutPoint, + released_count: usize, + balance: *const FFIBalance, + _account_balances: *const FFIAccountBalance, + _account_balances_count: u32, + _user: *mut c_void, + ) { + assert!(!released.is_null()); + let entries = unsafe { std::slice::from_raw_parts(released, released_count) }; + *FIRST_TXID.lock().expect("txid") = entries[0].txid; + *VOUTS.lock().expect("vouts") = entries.iter().map(|e| e.vout).collect(); + CONFIRMED.store(unsafe { (*balance).confirmed } as u32, Ordering::SeqCst); + } + + let callbacks = FFIWalletEventCallbacks { + on_transactions_swept: Some(cb), + ..FFIWalletEventCallbacks::default() + }; + + let parent = Txid::from_byte_array([9u8; 32]); + callbacks.dispatch(&WalletEvent::TransactionsSwept { + wallet_id: [7u8; 32], + txids: vec![Txid::from_byte_array([1u8; 32])], + superseded_by: Txid::from_byte_array([2u8; 32]), + winner_mined_height: None, + released_outpoints: vec![ + dashcore::OutPoint { + txid: parent, + vout: 3, + }, + dashcore::OutPoint { + txid: parent, + vout: 7, + }, + ], + balance: WalletCoreBalance::new(123_456, 0, 0, 0), + account_balances: BTreeMap::new(), + }); + + assert_eq!(*FIRST_TXID.lock().expect("txid"), *parent.as_byte_array()); + assert_eq!(*VOUTS.lock().expect("vouts"), vec![3, 7]); + assert_eq!( + CONFIRMED.load(Ordering::SeqCst), + 123_456, + "the balance parameter must not be shifted by the released-outpoint insertion" + ); + } + /// `BlocksNeeded` dispatch must pass exactly one entry per /// `FilterMatchKey` to the FFI callback (i.e. iterate keys, not /// inflated by the per-block wallet attribution). @@ -1370,4 +1668,47 @@ mod tests { }); assert_eq!(FIRED.load(Ordering::SeqCst), 0); } + + /// `TransactionBroadcastResult` dispatch must marshal the outcome fields + /// (status, relayed_by) for each variant. + #[test] + fn test_transaction_broadcast_result_dispatch() { + use dash_spv::BroadcastResult; + + static STATUS: AtomicU32 = AtomicU32::new(u32::MAX); + static RELAYED: AtomicU32 = AtomicU32::new(u32::MAX); + + extern "C" fn cb( + txid: *const [u8; 32], + status: FFIBroadcastStatus, + relayed_by: u32, + _user: *mut c_void, + ) { + assert!(!txid.is_null()); + STATUS.store(status as u32, Ordering::SeqCst); + RELAYED.store(relayed_by, Ordering::SeqCst); + } + + let callbacks = FFISyncEventCallbacks { + on_transaction_broadcast_result: Some(cb), + ..FFISyncEventCallbacks::default() + }; + let txid = Txid::from_byte_array([7u8; 32]); + + callbacks.dispatch(&SyncEvent::TransactionBroadcastResult { + txid, + result: BroadcastResult::Accepted { + relayed_by: 3, + }, + }); + assert_eq!(STATUS.load(Ordering::SeqCst), FFIBroadcastStatus::Accepted as u32); + assert_eq!(RELAYED.load(Ordering::SeqCst), 3); + + callbacks.dispatch(&SyncEvent::TransactionBroadcastResult { + txid, + result: BroadcastResult::Uncertain, + }); + assert_eq!(STATUS.load(Ordering::SeqCst), FFIBroadcastStatus::Uncertain as u32); + assert_eq!(RELAYED.load(Ordering::SeqCst), 0); + } } diff --git a/dash-spv-ffi/src/client.rs b/dash-spv-ffi/src/client.rs index 6cb94c8e2..33b334ef9 100644 --- a/dash-spv-ffi/src/client.rs +++ b/dash-spv-ffi/src/client.rs @@ -348,6 +348,89 @@ pub unsafe extern "C" fn dash_spv_ffi_client_broadcast_transaction( } } +/// Network-level outcome of a broadcast, as returned by +/// `dash_spv_ffi_client_broadcast_transaction_and_wait`. +/// +/// Plain value type — nothing to free. +#[repr(C)] +pub struct FFIBroadcastResult { + /// The determined outcome. + pub status: crate::FFIBroadcastStatus, + /// Distinct non-recipient peers that announced the txid back + /// (meaningful for `Accepted`). + pub relayed_by: u32, +} + +/// Broadcasts a transaction and waits for its network-level outcome. +/// +/// Blocks until the network accepts the transaction (non-recipient peers +/// announce it back, it is InstantSend-locked, or confirmed) or the timeout +/// elapses (outcome `Uncertain`). `timeout_secs == 0` uses the configured +/// acceptance timeout plus a small grace period. +/// +/// Requires mempool tracking to be enabled in the client config. +/// +/// # Safety +/// +/// - `client` must be a valid, non-null pointer to an initialized FFIDashSpvClient +/// - `tx_bytes` must be a valid, non-null pointer to the transaction data +/// - `length` must be the length of the transaction data in bytes +/// - `out_result` must be a valid, non-null pointer to an FFIBroadcastResult +#[no_mangle] +pub unsafe extern "C" fn dash_spv_ffi_client_broadcast_transaction_and_wait( + client: *mut FFIDashSpvClient, + tx_bytes: *const u8, + length: usize, + timeout_secs: u32, + out_result: *mut FFIBroadcastResult, +) -> i32 { + null_check!(client); + null_check!(tx_bytes); + null_check!(out_result); + + let tx_bytes = std::slice::from_raw_parts(tx_bytes, length); + + let tx = match dashcore::consensus::deserialize::(tx_bytes) { + Ok(t) => t, + Err(e) => { + set_last_error(&format!("Invalid transaction: {}", e)); + return FFIErrorCode::InvalidArgument as i32; + } + }; + + let client = &(*client); + let spv_client = client.inner.clone(); + let timeout = (timeout_secs > 0).then(|| std::time::Duration::from_secs(timeout_secs as u64)); + + let result = client + .runtime + .block_on(async { spv_client.broadcast_transaction_and_wait(&tx, timeout).await }); + + match result { + Ok(outcome) => { + use dash_spv::BroadcastResult; + let ffi = match outcome { + BroadcastResult::Accepted { + relayed_by, + } => FFIBroadcastResult { + status: crate::FFIBroadcastStatus::Accepted, + relayed_by: relayed_by as u32, + }, + BroadcastResult::Uncertain => FFIBroadcastResult { + status: crate::FFIBroadcastStatus::Uncertain, + relayed_by: 0, + }, + }; + std::ptr::write(out_result, ffi); + FFIErrorCode::Success as i32 + } + Err(e) => { + set_last_error(&format!("Failed to broadcast transaction: {}", e)); + FFIErrorCode::from(e) as i32 + } + } +} + /// Destroy the client and free associated resources. /// /// # Safety diff --git a/dash-spv-ffi/src/config.rs b/dash-spv-ffi/src/config.rs index d6d7bf581..849c0a87d 100644 --- a/dash-spv-ffi/src/config.rs +++ b/dash-spv-ffi/src/config.rs @@ -325,6 +325,86 @@ pub unsafe extern "C" fn dash_spv_ffi_config_set_fetch_mempool_transactions( FFIErrorCode::Success as i32 } +/// Withholds broadcast transactions from half of the connected peers +/// (the default policy). The withheld peers are the source of the `inv` +/// echo that proves a broadcast propagated through the network. +/// +/// # Safety +/// - `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet +/// - The caller must ensure the config pointer remains valid for the duration of this call +#[no_mangle] +pub unsafe extern "C" fn dash_spv_ffi_config_set_broadcast_holdout_half( + config: *mut FFIClientConfig, +) -> i32 { + null_check!(config); + + let config = unsafe { &mut *((*config).inner as *mut ClientConfig) }; + config.broadcast_holdout = dash_spv::BroadcastHoldout::Half; + FFIErrorCode::Success as i32 +} + +/// Withholds broadcast transactions from a fixed number of peers (clamped so +/// that at least one peer always receives the transaction). +/// +/// # Safety +/// - `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet +/// - The caller must ensure the config pointer remains valid for the duration of this call +#[no_mangle] +pub unsafe extern "C" fn dash_spv_ffi_config_set_broadcast_holdout_count( + config: *mut FFIClientConfig, + count: u32, +) -> i32 { + null_check!(config); + + let config = unsafe { &mut *((*config).inner as *mut ClientConfig) }; + config.broadcast_holdout = dash_spv::BroadcastHoldout::Count(count as usize); + FFIErrorCode::Success as i32 +} + +/// Sets how many distinct non-recipient peers must announce a broadcast txid +/// back before it is reported as accepted. Must be > 0. +/// +/// # Safety +/// - `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet +/// - The caller must ensure the config pointer remains valid for the duration of this call +#[no_mangle] +pub unsafe extern "C" fn dash_spv_ffi_config_set_broadcast_acceptance_threshold( + config: *mut FFIClientConfig, + threshold: u32, +) -> i32 { + null_check!(config); + if threshold == 0 { + set_last_error("broadcast acceptance threshold must be > 0"); + return FFIErrorCode::InvalidArgument as i32; + } + + let config = unsafe { &mut *((*config).inner as *mut ClientConfig) }; + config.broadcast_acceptance_threshold = threshold as usize; + FFIErrorCode::Success as i32 +} + +/// Sets the timeout (in seconds) after which a pending broadcast with no +/// acceptance signal is reported as uncertain. Must be > 0. +/// +/// # Safety +/// - `config` must be a valid pointer to an FFIClientConfig created by dash_spv_ffi_config_new/mainnet/testnet +/// - The caller must ensure the config pointer remains valid for the duration of this call +#[no_mangle] +pub unsafe extern "C" fn dash_spv_ffi_config_set_broadcast_acceptance_timeout_secs( + config: *mut FFIClientConfig, + seconds: u32, +) -> i32 { + null_check!(config); + if seconds == 0 { + set_last_error("broadcast acceptance timeout must be > 0"); + return FFIErrorCode::InvalidArgument as i32; + } + + let config = unsafe { &mut *((*config).inner as *mut ClientConfig) }; + config.broadcast_acceptance_timeout = std::time::Duration::from_secs(seconds as u64); + FFIErrorCode::Success as i32 +} + // Checkpoint sync configuration functions /// Sets the starting block height for synchronization diff --git a/dash-spv-ffi/src/platform_integration.rs b/dash-spv-ffi/src/platform_integration.rs index ce1609788..bb323c3bf 100644 --- a/dash-spv-ffi/src/platform_integration.rs +++ b/dash-spv-ffi/src/platform_integration.rs @@ -101,53 +101,25 @@ pub unsafe extern "C" fn ffi_dash_spv_get_quorum_public_key( }; let engine_guard = engine.blocking_read(); - let (before, _after) = engine_guard.masternode_lists_around_height(core_chain_locked_height); - let ml = match before { - Some(ml) => ml, - None => { - return FFIResult::error( - FFIErrorCode::ValidationError, - &format!( - "No masternode list found at or before height {}", - core_chain_locked_height - ), - ); - } - }; - - let list_height = ml.known_height; - match ml.quorums.get(&llmq_type) { - Some(quorums) => match quorums.get(&quorum_hash) { - Some(quorum) => { - let pubkey_bytes: &[u8; 48] = quorum.quorum_entry.quorum_public_key.as_ref(); - std::ptr::copy_nonoverlapping( - pubkey_bytes.as_ptr(), - out_pubkey, - QUORUM_PUBKEY_SIZE, - ); - - FFIResult { - error_code: 0, - error_message: ptr::null(), - } + match engine_guard.quorum_entry_for_hash_at_or_before_height( + llmq_type, + quorum_hash, + core_chain_locked_height, + ) { + Some((_list_height, quorum)) => { + let pubkey_bytes: &[u8; 48] = quorum.quorum_entry.quorum_public_key.as_ref(); + std::ptr::copy_nonoverlapping(pubkey_bytes.as_ptr(), out_pubkey, QUORUM_PUBKEY_SIZE); + + FFIResult { + error_code: 0, + error_message: ptr::null(), } - None => FFIResult::error( - FFIErrorCode::ValidationError, - &format!( - "Quorum not found: type {} at list height {} (requested {}) with hash {:x} (masternode list exists with {} quorums of this type)", - quorum_type, - list_height, - core_chain_locked_height, - quorum_hash, - quorums.len() - ), - ), - }, + } None => FFIResult::error( FFIErrorCode::ValidationError, &format!( - "No quorums of type {} found at list height {} (requested {})", - quorum_type, list_height, core_chain_locked_height + "Quorum not found: type {} at or before height {} with hash {:x}", + quorum_type, core_chain_locked_height, quorum_hash ), ), } diff --git a/dash-spv-ffi/tests/dashd_sync/callbacks.rs b/dash-spv-ffi/tests/dashd_sync/callbacks.rs index a36b944ef..c9e58b108 100644 --- a/dash-spv-ffi/tests/dashd_sync/callbacks.rs +++ b/dash-spv-ffi/tests/dashd_sync/callbacks.rs @@ -593,6 +593,7 @@ pub(super) fn create_sync_callbacks(tracker: &Arc) -> FFISyncEv on_instantlock_received: Some(on_instantlock_received), on_manager_error: Some(on_manager_error), on_sync_complete: Some(on_sync_complete), + on_transaction_broadcast_result: None, user_data: Arc::as_ptr(tracker) as *mut c_void, } } @@ -621,6 +622,8 @@ pub(super) fn create_wallet_callbacks(tracker: &Arc) -> FFIWall on_block_processed: Some(on_wallet_block_processed), on_sync_height_advanced: Some(on_sync_height_advanced), on_chain_lock_processed: None, + // Not exercised by these tests: they never build a conflicting spend. + on_transactions_swept: None, user_data: Arc::as_ptr(tracker) as *mut c_void, } } diff --git a/dash-spv-ffi/tests/test_config.rs b/dash-spv-ffi/tests/test_config.rs index 651d7201d..e58b6e2f7 100644 --- a/dash-spv-ffi/tests/test_config.rs +++ b/dash-spv-ffi/tests/test_config.rs @@ -107,4 +107,78 @@ mod tests { dash_spv_ffi_config_destroy(config); } } + + #[test] + #[serial] + fn test_config_broadcast_holdout_setters() { + unsafe { + let config = dash_spv_ffi_config_new(FFINetwork::Testnet); + + let result = dash_spv_ffi_config_set_broadcast_holdout_count(config, 2); + assert_eq!(result, FFIErrorCode::Success as i32); + assert_eq!( + (*config).get_inner().broadcast_holdout, + dash_spv::BroadcastHoldout::Count(2) + ); + + let result = dash_spv_ffi_config_set_broadcast_holdout_half(config); + assert_eq!(result, FFIErrorCode::Success as i32); + assert_eq!((*config).get_inner().broadcast_holdout, dash_spv::BroadcastHoldout::Half); + + dash_spv_ffi_config_destroy(config); + } + } + + #[test] + #[serial] + fn test_config_broadcast_acceptance_threshold() { + unsafe { + let config = dash_spv_ffi_config_new(FFINetwork::Testnet); + + // Zero is rejected and leaves the default intact + let default_threshold = (*config).get_inner().broadcast_acceptance_threshold; + let result = dash_spv_ffi_config_set_broadcast_acceptance_threshold(config, 0); + assert_eq!(result, FFIErrorCode::InvalidArgument as i32); + assert_eq!( + (*config).get_inner().broadcast_acceptance_threshold, + default_threshold, + "rejected value must not modify the config" + ); + + // Nonzero maps to the ClientConfig field + let result = dash_spv_ffi_config_set_broadcast_acceptance_threshold(config, 3); + assert_eq!(result, FFIErrorCode::Success as i32); + assert_eq!((*config).get_inner().broadcast_acceptance_threshold, 3); + + dash_spv_ffi_config_destroy(config); + } + } + + #[test] + #[serial] + fn test_config_broadcast_acceptance_timeout() { + unsafe { + let config = dash_spv_ffi_config_new(FFINetwork::Testnet); + + // Zero is rejected and leaves the default intact + let default_timeout = (*config).get_inner().broadcast_acceptance_timeout; + let result = dash_spv_ffi_config_set_broadcast_acceptance_timeout_secs(config, 0); + assert_eq!(result, FFIErrorCode::InvalidArgument as i32); + assert_eq!( + (*config).get_inner().broadcast_acceptance_timeout, + default_timeout, + "rejected value must not modify the config" + ); + + // Nonzero maps to the ClientConfig field + let result = dash_spv_ffi_config_set_broadcast_acceptance_timeout_secs(config, 90); + assert_eq!(result, FFIErrorCode::Success as i32); + assert_eq!( + (*config).get_inner().broadcast_acceptance_timeout, + std::time::Duration::from_secs(90) + ); + + dash_spv_ffi_config_destroy(config); + } + } } diff --git a/dash-spv-ffi/tests/test_wallet_manager.rs b/dash-spv-ffi/tests/test_wallet_manager.rs index b6ecb0d7a..9e04908ff 100644 --- a/dash-spv-ffi/tests/test_wallet_manager.rs +++ b/dash-spv-ffi/tests/test_wallet_manager.rs @@ -93,6 +93,7 @@ mod tests { wallet_manager_ptr, serialized_wallet.as_ptr(), serialized_wallet.len(), + 0, imported_wallet_id.as_mut_ptr(), &mut error as *mut FFIError, ); diff --git a/dash-spv-ffi/tests/unit/test_async_operations.rs b/dash-spv-ffi/tests/unit/test_async_operations.rs index 0a3d9613e..ccdf9c825 100644 --- a/dash-spv-ffi/tests/unit/test_async_operations.rs +++ b/dash-spv-ffi/tests/unit/test_async_operations.rs @@ -78,6 +78,7 @@ mod tests { on_instantlock_received: None, on_manager_error: None, on_sync_complete: Some(on_sync_complete), + on_transaction_broadcast_result: None, user_data: &event_data as *const _ as *mut c_void, }; diff --git a/dash-spv/benches/storage.rs b/dash-spv/benches/storage.rs index 44452c303..8466d6b76 100644 --- a/dash-spv/benches/storage.rs +++ b/dash-spv/benches/storage.rs @@ -28,7 +28,10 @@ fn bench_disk_storage(c: &mut Criterion) { let rt = Builder::new_multi_thread().worker_threads(4).enable_all().build().unwrap(); - let headers = (0..NUM_ELEMENTS).map(create_test_header).collect::>(); + let headers = (0..NUM_ELEMENTS) + .map(create_test_header) + .map(dash_spv::types::HashedBlockHeader::from) + .collect::>(); let mut rng = StdRng::seed_from_u64(SEED); c.bench_function("storage/disk/store", |b| { @@ -75,7 +78,7 @@ fn bench_disk_storage(c: &mut Criterion) { b.to_async(&rt).iter_batched( || { let height = rand::random::() % NUM_ELEMENTS; - headers[height as usize].block_hash() + *headers[height as usize].hash() }, async |hash| { let _ = storage.get_header_height_by_hash(&hash).await.unwrap(); diff --git a/dash-spv/src/chain/checkpoints.rs b/dash-spv/src/chain/checkpoints.rs index f78621b54..2eec44d34 100644 --- a/dash-spv/src/chain/checkpoints.rs +++ b/dash-spv/src/chain/checkpoints.rs @@ -6,7 +6,7 @@ //! - Protect against deep reorganizations //! - Bootstrap masternode lists at specific heights -use dashcore::{BlockHash, CompactTarget, Target}; +use dashcore::{BlockHash, CompactTarget, Network, Target}; use dashcore_hashes::{hex, Hash}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -66,6 +66,18 @@ pub struct CheckpointManager { } impl CheckpointManager { + /// Create a checkpoint manager seeded with the hardcoded checkpoints for `network`. + /// + /// Networks without bundled checkpoints (devnet, regtest) yield an empty manager. + pub fn for_network(network: Network) -> Self { + let checkpoints = match network { + Network::Mainnet => mainnet_checkpoints(), + Network::Testnet => testnet_checkpoints(), + _ => vec![], + }; + Self::new(checkpoints) + } + /// Create a new checkpoint manager from a list of checkpoints pub fn new(checkpoints: Vec) -> Self { let mut checkpoint_map = HashMap::new(); @@ -721,6 +733,54 @@ pub fn mainnet_checkpoints() -> Vec { 972444458, Some("ML2300000__70232"), ), + // Height 2350000 + create_checkpoint( + 2350000, + "00000000000000258216a62e8c7170be1207335474ddfa667092a71e3d4162d2", + "000000000000002702e07f9f3402026e4cd5707961ce54af22c873331a329708", + 1759648416, + 0x192f2bfb, + "0x00000000000000000000000000000000000000000000ada31a5dd056c969c842", + "3e77d2ed0c24aab79096d700cdc4fe3ea9502fcec38ee114c81c9063842a6725", + 3635235496, + None, + ), + // Height 2400000 + create_checkpoint( + 2400000, + "000000000000002883c9151a37092287c1773c5a4d175a8f688373daa631f934", + "000000000000001d1d2f9edbd2792ebbfa001e42ff47e6ef81efeaef68df4fe7", + 1767528049, + 0x19377eb0, + "0x00000000000000000000000000000000000000000000b2955e92bc38ce3a61ed", + "9247dc06cf2e952b6a97d7c0057946933c1b5699ce6b76b86583775c26ae3b01", + 3220391604, + None, + ), + // Height 2450000 + create_checkpoint( + 2450000, + "00000000000000074832a3ff13968b2aa0c841fcdfced9c3619257242470f25f", + "0000000000000023c3bd45c16e76cfe33e5cac0e3c4c392b61ee8f4e9d9e345a", + 1775408495, + 0x192e7792, + "0x00000000000000000000000000000000000000000000b716ba49112c2a375b0f", + "c6b4c57362b96e300b1b3ad20aa73e3fb0c40fe67049a51bd1f0b2d034bd74dc", + 1523946232, + None, + ), + // Height 2500000 + create_checkpoint( + 2500000, + "000000000000000ac721a208c5efba67e76cb08796324de3519fefa32a5b2f44", + "000000000000003908fc91c23e32714e3d4e07254f8e2e9a0a803c12c3f11083", + 1783288591, + 0x193d61e0, + "0x00000000000000000000000000000000000000000000bb84d5645e0656eac7db", + "053838492199315a53ada9322184cef33c08ea4cbdc9e0a5c30a5a8857cd469e", + 3924859976, + None, + ), ] } @@ -1075,6 +1135,30 @@ pub fn testnet_checkpoints() -> Vec { 516400, None, ), + // Height 1450000 + create_checkpoint( + 1450000, + "000000fa4225fa022d4216eabb176848d61d03027df12e0685f82d8b13c60f03", + "000000f16d17ad5b97a6c7e1c431953cc589155a6f24b981bc542f68759d48f8", + 1775019403, + 0x1e01222f, + "0x00000000000000000000000000000000000000000000000003eb87177577500b", + "914a43b78cd4e12cac031e44f8cb0c63f8a3bfd3a238d08ebf83af61962cbb49", + 61968, + None, + ), + // Height 1500000 + create_checkpoint( + 1500000, + "000000b625f73fab6cb338c31cb656680ccfb3b574b9225022c8e90e293a0c12", + "000000376fa86df5684a69d8055daa1ddaf349844075d55e9a190f2b1899f4f0", + 1782027920, + 0x1e00ca28, + "0x00000000000000000000000000000000000000000000000003ec5aff7410d4a1", + "7ad0f11d8ed65954ce5f42e4aebbe10fd45cac880c31dee76b4b9dcc08382945", + 461766, + None, + ), ] } diff --git a/dash-spv/src/client/config.rs b/dash-spv/src/client/config.rs index ef84a75ac..00cd577de 100644 --- a/dash-spv/src/client/config.rs +++ b/dash-spv/src/client/config.rs @@ -3,11 +3,13 @@ use clap::ValueEnum; use std::net::SocketAddr; use std::path::PathBuf; +use std::time::Duration; use dashcore::Network; // Serialization removed due to complex Address types use crate::client::devnet::DevnetConfig; +use crate::sync::BroadcastHoldout; use crate::types::ValidationMode; /// Strategy for handling mempool (unconfirmed) transactions. @@ -68,10 +70,33 @@ pub struct ClientConfig { /// Whether to fetch transactions from INV messages immediately. pub fetch_mempool_transactions: bool, + /// How many peers to withhold broadcast transactions from. + /// + /// Peers never re-announce a transaction to whoever sent it to them, so + /// the withheld ("holdout") peers are the only source of the `inv` echo + /// that proves a broadcast propagated through the network. + pub broadcast_holdout: BroadcastHoldout, + + /// Distinct non-recipient peers that must announce a broadcast txid back + /// before the broadcast is reported as accepted. + pub broadcast_acceptance_threshold: usize, + + /// How long a broadcast may stay pending before its outcome is reported + /// as uncertain. + pub broadcast_acceptance_timeout: Duration, + /// Start syncing from a specific block height. /// The client will use the nearest checkpoint at or before this height. pub start_from_height: Option, + /// Time-to-live (seconds) for unfunded receive-address reservations, + /// periodically reclaimed by the sweep so a hand-out that is never paid and + /// never released does not pin gap-limit headroom forever. `Some(secs)` + /// enables the sweep with that TTL; `None` disables it. Defaults to one hour. + /// Set with [`Self::with_reservation_ttl_secs`] or clear with + /// [`Self::without_reservation_sweep`]. + pub reservation_sweep_ttl_secs: Option, + /// Devnet-only configuration. Must be `Some` iff `network == Network::Devnet`. pub devnet: Option, } @@ -86,14 +111,18 @@ impl Default for ClientConfig { validation_mode: ValidationMode::Full, enable_filters: true, enable_masternodes: true, - max_peers: 8, + max_peers: 3, user_agent: None, // Mempool defaults enable_mempool_tracking: true, mempool_strategy: MempoolStrategy::FetchAll, max_mempool_transactions: 1000, fetch_mempool_transactions: true, + broadcast_holdout: BroadcastHoldout::Half, + broadcast_acceptance_threshold: 1, + broadcast_acceptance_timeout: Duration::from_secs(60), start_from_height: None, + reservation_sweep_ttl_secs: Some(3600), devnet: None, } } @@ -180,12 +209,52 @@ impl ClientConfig { self } + /// Set the broadcast holdout policy. + pub fn with_broadcast_holdout(mut self, holdout: BroadcastHoldout) -> Self { + self.broadcast_holdout = holdout; + self + } + + /// Set how many non-recipient peer announcements mark a broadcast accepted. + pub fn with_broadcast_acceptance_threshold(mut self, threshold: usize) -> Self { + self.broadcast_acceptance_threshold = threshold; + self + } + + /// Set the timeout after which a pending broadcast is reported uncertain. + pub fn with_broadcast_acceptance_timeout(mut self, timeout: Duration) -> Self { + self.broadcast_acceptance_timeout = timeout; + self + } + + /// Bundle the broadcast acceptance settings for the mempool manager. + pub(crate) fn broadcast_config(&self) -> crate::sync::BroadcastConfig { + crate::sync::BroadcastConfig { + holdout: self.broadcast_holdout, + acceptance_threshold: self.broadcast_acceptance_threshold, + acceptance_timeout: self.broadcast_acceptance_timeout, + } + } + /// Set the starting height for synchronization. pub fn with_start_height(mut self, height: u32) -> Self { self.start_from_height = Some(height); self } + /// Disable the periodic receive-address reservation sweep. + pub fn without_reservation_sweep(mut self) -> Self { + self.reservation_sweep_ttl_secs = None; + self + } + + /// Enable the periodic receive-address reservation sweep with the given TTL + /// (seconds): a reservation older than `secs` is reclaimed. + pub fn with_reservation_ttl_secs(mut self, secs: u64) -> Self { + self.reservation_sweep_ttl_secs = Some(secs); + self + } + /// Attach a [`DevnetConfig`]. The network must be `Network::Devnet`. /// [`validate`](Self::validate) enforces the biconditional. pub fn with_devnet(mut self, devnet: DevnetConfig) -> Self { @@ -208,6 +277,17 @@ impl ClientConfig { ); } + if self.reservation_sweep_ttl_secs == Some(0) { + return Err("reservation_sweep_ttl_secs must be > 0 when set; use without_reservation_sweep() to disable the sweep".to_string()); + } + + if self.broadcast_acceptance_threshold == 0 { + return Err("broadcast_acceptance_threshold must be > 0".to_string()); + } + if self.broadcast_acceptance_timeout.is_zero() { + return Err("broadcast_acceptance_timeout must be > 0".to_string()); + } + match (self.network == Network::Devnet, &self.devnet) { (true, Some(devnet)) => devnet.validate()?, (true, None) => { diff --git a/dash-spv/src/client/config_test.rs b/dash-spv/src/client/config_test.rs index dd1a763d5..e954bb8e1 100644 --- a/dash-spv/src/client/config_test.rs +++ b/dash-spv/src/client/config_test.rs @@ -23,7 +23,7 @@ mod tests { assert_eq!(config.validation_mode, ValidationMode::Full); assert!(config.enable_filters); assert!(config.enable_masternodes); - assert_eq!(config.max_peers, 8); + assert_eq!(config.max_peers, 3); // Mempool defaults assert!(config.enable_mempool_tracking); @@ -197,6 +197,18 @@ mod tests { assert!(result.unwrap_err().contains("max_mempool_transactions must be > 0")); } + #[test] + fn test_validation_rejects_zero_reservation_ttl() { + let config = ClientConfig { + reservation_sweep_ttl_secs: Some(0), + ..Default::default() + }; + + let result = config.validate(); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("reservation_sweep_ttl_secs must be > 0")); + } + #[test] fn test_apply_global_overrides_no_devnet_is_noop() { let tmp = TempDir::new().unwrap(); diff --git a/dash-spv/src/client/event_handler.rs b/dash-spv/src/client/event_handler.rs index a6ef1aace..4ea1c53b8 100644 --- a/dash-spv/src/client/event_handler.rs +++ b/dash-spv/src/client/event_handler.rs @@ -6,6 +6,7 @@ use std::fmt::Display; use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::{broadcast, mpsc, watch, RwLock}; use tokio::task::JoinHandle; @@ -150,7 +151,8 @@ pub(crate) fn spawn_progress_monitor( /// [`SyncEvent::SyncComplete`], serializing all `apply_chain_lock` /// calls on a single task. During the initial sync cycle (cycle 0) /// chainlock arrivals are buffered as the latest-seen height rather -/// than applied. At `SyncComplete { cycle: 0 }` the buffered height +/// than applied; only that height is passed on +/// (`note_chain_lock_height`). At `SyncComplete { cycle: 0 }` the buffered height /// is applied once, then every subsequent validated chainlock is /// applied directly. Only validated chainlocks advance the wallet. /// @@ -184,6 +186,7 @@ where .as_ref() .is_none_or(|buffered| chain_lock.block_height > buffered.block_height) { + wallet.write().await.note_chain_lock_height(chain_lock.block_height); deferred_chain_lock = Some(chain_lock); } } @@ -217,6 +220,48 @@ where }) } +/// How often the reservation-sweep task wakes to reclaim expired reservations. +const RESERVATION_SWEEP_INTERVAL: Duration = Duration::from_secs(60); + +/// Periodically reclaim receive-address reservations that were handed out but +/// never funded and never explicitly released, which would otherwise pin +/// gap-limit headroom forever. +/// +/// The wallet keeps no clock, so this task supplies `now` from the system clock +/// and the caller-supplied `ttl` (seconds). A system-clock failure yields a +/// `now` of `0`, which the wallet treats as "no clock" and reclaims nothing. +/// Runs until `shutdown` is cancelled. +pub(crate) fn spawn_reservation_sweep( + wallet: Arc>, + ttl_secs: u64, + shutdown: CancellationToken, +) -> JoinHandle<()> +where + W: WalletInterface + 'static, +{ + const NAME: &str = "Reservation sweep"; + tokio::spawn(async move { + tracing::debug!("{} task started", NAME); + let mut interval = tokio::time::interval(RESERVATION_SWEEP_INTERVAL); + loop { + tokio::select! { + _ = interval.tick() => { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let reclaimed = wallet.write().await.sweep_expired_reservations(now, ttl_secs); + if reclaimed > 0 { + tracing::debug!("{}: reclaimed {} expired reservation(s)", NAME, reclaimed); + } + } + _ = shutdown.cancelled() => break, + } + } + tracing::debug!("{} task exiting", NAME); + }) +} + #[cfg(test)] mod tests { use std::net::SocketAddr; diff --git a/dash-spv/src/client/lifecycle.rs b/dash-spv/src/client/lifecycle.rs index 8c7d4f29f..46e26f71c 100644 --- a/dash-spv/src/client/lifecycle.rs +++ b/dash-spv/src/client/lifecycle.rs @@ -9,7 +9,7 @@ //! - Wallet data loading use super::{ClientConfig, DashSpvClient, EventHandler}; -use crate::chain::checkpoints::{mainnet_checkpoints, testnet_checkpoints, CheckpointManager}; +use crate::chain::checkpoints::CheckpointManager; use crate::error::{Result, SpvError}; use crate::network::NetworkManager; use crate::storage::{ @@ -20,8 +20,12 @@ use crate::sync::{ BlockHeadersManager, BlocksManager, ChainLockManager, FilterHeadersManager, FiltersManager, InstantSendManager, Managers, MasternodesManager, MempoolManager, SyncCoordinator, }; +use crate::types::HashedBlockHeader; +use dashcore::block::{Header as BlockHeader, Version}; use dashcore::network::constants::NetworkExt; +use dashcore::pow::CompactTarget; use dashcore::sml::masternode_list_engine::MasternodeListEngine; +use dashcore::TxMerkleNode; use dashcore_hashes::Hash; use key_wallet_manager::WalletInterface; use std::sync::Arc; @@ -42,9 +46,24 @@ impl DashSpvClient Some(height), + None => { + let birth_height = wallet.read().await.earliest_required_height().await; + let start = birth_height.max(config.network.hd_wallet_sync_floor()); + (start > 0).then_some(start) + } + }; + // Initialize genesis block or checkpoint before creating managers, // so they can read the tip from storage during construction. - Self::initialize_genesis_block(&config, &mut storage).await?; + Self::initialize_genesis_block(&config, start_from_height, &mut storage).await?; let masternode_engine = { if config.enable_masternodes { @@ -65,12 +84,7 @@ impl DashSpvClient = Managers::default(); - let checkpoints = match config.network { - dashcore::Network::Mainnet => mainnet_checkpoints(), - dashcore::Network::Testnet => testnet_checkpoints(), - _ => Vec::new(), - }; - let checkpoint_manager = Arc::new(CheckpointManager::new(checkpoints)); + let checkpoint_manager = Arc::new(CheckpointManager::for_network(config.network)); managers.block_headers = Some( BlockHeadersManager::new( storage.block_headers(), @@ -131,6 +145,7 @@ impl DashSpvClient DashSpvClient Result<()> { + async fn initialize_genesis_block( + config: &ClientConfig, + start_from_height: Option, + storage: &mut S, + ) -> Result<()> { // Check if we already have any headers in storage let current_tip = storage.get_tip_height().await; @@ -239,16 +258,8 @@ impl DashSpvClient crate::chain::checkpoints::mainnet_checkpoints(), - dashcore::Network::Testnet => crate::chain::checkpoints::testnet_checkpoints(), - _ => vec![], - }; - - // Create checkpoint manager - let checkpoint_manager = crate::chain::checkpoints::CheckpointManager::new(checkpoints); + if let Some(start_height) = start_from_height { + let checkpoint_manager = CheckpointManager::for_network(config.network); // Find the best checkpoint at or before the requested height if let Some(checkpoint) = checkpoint_manager.last_checkpoint_before_height(start_height) @@ -260,48 +271,37 @@ impl DashSpvClient DashSpvClient, + ) -> (u32, Option) { + let mut wallet_manager = WalletManager::::new(Network::Mainnet); + wallet_manager + .create_wallet_from_mnemonic( + TEST_MNEMONIC, + birth_height, + WalletAccountCreationOptions::Default, + ) + .expect("wallet creation must succeed"); + let wallet = Arc::new(RwLock::new(wallet_manager)); + + let temp_dir = TempDir::new().unwrap(); + let mut config = ClientConfig::mainnet() + .without_filters() + .without_masternodes() + .with_storage_path(temp_dir.path()); + config.start_from_height = start_from_height; + + let storage = DiskStorageManager::new(&config).await.expect("Failed to create storage"); + let client = DashSpvClient::new( + config, + MockNetworkManager::new(), + storage, + wallet, + vec![Arc::new(())], + ) + .await + .expect("client construction must succeed"); + (client.tip_height().await, client.tip_hash().await) + } + + #[tokio::test] + async fn birth_height_anchors_chain_to_nearest_checkpoint() { + // A wallet birth height below mainnet's HD/BIP39 activation floor is clamped up to + // the floor, which anchors at the nearest checkpoint at or before it: the 200_000 + // checkpoint. The stored tip carries the trusted checkpoint hash (what the next + // header's `prev_blockhash` is validated against), not a hash recomputed from a header. + // 120_000 is below mainnet's HD/BIP39 sync floor, so it is clamped up. + let (height, hash) = anchored_tip(120_000, None).await; + assert_eq!(height, 200_000); + let expected: dashcore::BlockHash = + "000000000004d0615ff622ec78457ca211dc63fc9c62cca9d9d9af7206be721b".parse().unwrap(); + assert_eq!(hash, Some(expected)); + + // Birth height 0 no longer drags mainnet sync to genesis; it floors at HD activation. + assert_eq!(anchored_tip(0, None).await.0, 200_000); + + // A birth height above the floor is unaffected and anchors at the nearest + // checkpoint at or below it. + assert_eq!(anchored_tip(560_000, None).await.0, 550_000); + + // Explicit `start_from_height` wins over the wallet birth height and is honored + // even below the floor. + assert_eq!(anchored_tip(120_000, Some(60_000)).await.0, 50_000); + } + #[tokio::test] async fn client_exposes_shared_wallet_manager() { let config = ClientConfig::mainnet() @@ -70,4 +143,59 @@ mod tests { let wallet_ref = client.wallet(); let _wallet_guard = wallet_ref.read().await; } + + /// The client reports the coinbase payload the masternode list engine + /// kept, and nothing while masternode sync is off. + #[tokio::test] + async fn latest_coinbase_reports_what_the_masternode_list_engine_kept() { + let make_client = |config: ClientConfig| async move { + let storage = + DiskStorageManager::with_temp_dir().await.expect("Failed to create tmp storage"); + let wallet = + Arc::new(RwLock::new(WalletManager::::new(config.network))); + DashSpvClient::new( + config, + MockNetworkManager::new(), + storage, + wallet, + vec![Arc::new(())], + ) + .await + .expect("client construction must succeed") + }; + let storage_dir = TempDir::new().unwrap(); + + let without_masternodes = make_client( + ClientConfig::mainnet() + .without_filters() + .without_masternodes() + .with_storage_path(storage_dir.path()), + ) + .await; + assert_eq!(without_masternodes.latest_coinbase().await, None); + + let with_masternodes = make_client( + ClientConfig::mainnet().without_filters().with_storage_path(storage_dir.path()), + ) + .await; + assert_eq!(with_masternodes.latest_coinbase().await, None); + + let coinbase = CoinbasePayload { + version: 4, + height: 2_400_000, + merkle_root_masternode_list: MerkleRootMasternodeList::all_zeros(), + merkle_root_quorums: MerkleRootQuorums::all_zeros(), + best_cl_height: Some(0), + best_cl_signature: Some(BLSSignature::from([0; 96])), + asset_locked_amount: Some(0), + merkle_root_asset_unlocks: Some(MerkleRootAssetUnlocks::all_zeros()), + }; + with_masternodes + .masternode_list_engine() + .expect("masternode sync is on") + .write() + .await + .latest_coinbase = Some(coinbase.clone()); + assert_eq!(with_masternodes.latest_coinbase().await, Some(coinbase)); + } } diff --git a/dash-spv/src/client/queries.rs b/dash-spv/src/client/queries.rs index 7e215cc86..ad6357261 100644 --- a/dash-spv/src/client/queries.rs +++ b/dash-spv/src/client/queries.rs @@ -12,6 +12,7 @@ use crate::storage::StorageManager; use dashcore::sml::llmq_type::LLMQType; use dashcore::sml::masternode_list_engine::MasternodeListEngine; use dashcore::sml::quorum_entry::qualified_quorum_entry::QualifiedQuorumEntry; +use dashcore::transaction::special_transaction::coinbase::CoinbasePayload; use dashcore::QuorumHash; use key_wallet_manager::WalletInterface; use std::sync::Arc; @@ -43,6 +44,20 @@ impl DashSpvClient Option { + let engine = self.masternode_engine.as_ref()?; + engine.read().await.latest_coinbase().cloned() + } + /// Get a quorum entry by type and hash at a specific block height. /// Returns `SpvError::QuorumLookupError` if the quorum is not found. pub async fn get_quorum_at_height( @@ -53,56 +68,31 @@ impl DashSpvClient Result { let masternode_engine = self.masternode_list_engine()?; let masternode_engine_guard = masternode_engine.read().await; - let (before, _after) = masternode_engine_guard.masternode_lists_around_height(height); - if let Some(ml) = before { - let list_height = ml.known_height; - match ml.quorums.get(&quorum_type) { - Some(quorums) => match quorums.get(&quorum_hash) { - Some(quorum) => { - tracing::debug!( - "Found quorum type {} at list height {} (requested {}) with hash {}", - quorum_type, - list_height, - height, - hex::encode(quorum_hash) - ); - return Ok(quorum.clone()); - } - None => { - let message = format!( - "Quorum not found: type {} at list height {} (requested {}) with hash {} (masternode list exists with {} quorums of this type)", - quorum_type, - list_height, - height, - hex::encode(quorum_hash), - quorums.len() - ); - tracing::warn!(message); - return Err(SpvError::QuorumLookupError(message)); - } - }, - None => { - tracing::warn!( - "No quorums of type {} found at list height {} (requested {}) (masternode list exists)", - quorum_type, - list_height, - height - ); - return Err(SpvError::QuorumLookupError(format!( - "No quorums of type {} found at list height {} (requested {})", - quorum_type, list_height, height - ))); - } + match masternode_engine_guard.quorum_entry_for_hash_at_or_before_height( + quorum_type, + quorum_hash, + height, + ) { + Some((list_height, quorum)) => { + tracing::debug!( + "Found quorum type {} at list height {} (requested {}) with hash {}", + quorum_type, + list_height, + height, + hex::encode(quorum_hash) + ); + Ok(quorum.clone()) + } + None => { + let message = format!( + "Quorum not found: type {} at or before height {} with hash {}", + quorum_type, + height, + hex::encode(quorum_hash) + ); + tracing::warn!("{}", message); + Err(SpvError::QuorumLookupError(message)) } } - - tracing::warn!( - "No masternode list found at or before height {} - cannot retrieve quorum", - height - ); - Err(SpvError::QuorumLookupError(format!( - "No masternode list found at or before height {}", - height - ))) } } diff --git a/dash-spv/src/client/sync_coordinator.rs b/dash-spv/src/client/sync_coordinator.rs index e428bcb80..2f0efaa93 100644 --- a/dash-spv/src/client/sync_coordinator.rs +++ b/dash-spv/src/client/sync_coordinator.rs @@ -7,6 +7,7 @@ use tokio_util::sync::CancellationToken; use super::event_handler::{ spawn_broadcast_monitor, spawn_chainlock_wallet_dispatch, spawn_progress_monitor, + spawn_reservation_sweep, }; use super::DashSpvClient; use crate::error::Result; @@ -103,6 +104,14 @@ impl DashSpvClient DashSpvClient DashSpvClient { - /// Broadcast a transaction to all connected peers. + /// Broadcast a transaction to the network. + /// + /// With mempool tracking enabled (the default), the transaction is sent to + /// a subset of connected peers while being withheld from the rest; an + /// `inv` announcement from a withheld peer proves the transaction relayed + /// through the network and fires + /// [`SyncEvent::TransactionBroadcastResult`]. Use + /// [`broadcast_transaction_and_wait`](Self::broadcast_transaction_and_wait) + /// to await that outcome directly. /// - /// The transaction is also injected into the local message pipeline so that - /// the mempool manager processes it immediately. + /// With mempool tracking disabled there is no acceptance tracking and the + /// transaction is simply sent to all connected peers. pub async fn broadcast_transaction(&self, tx: &dashcore::Transaction) -> Result<()> { let network_guard = self.network.lock().await; @@ -20,12 +36,79 @@ impl DashSpvClient, + ) -> Result { + let config_timeout = { + let config = self.config.read().await; + if !config.enable_mempool_tracking { + return Err(SpvError::Config( + "broadcast_transaction_and_wait requires mempool tracking".to_string(), + )); + } + config.broadcast_acceptance_timeout + }; + let wait = timeout.unwrap_or(config_timeout + AWAIT_GRACE); + let txid = tx.txid(); + + // Subscribe before dispatching so the outcome event cannot be missed. + let mut events = self.subscribe_sync_events().await; + self.broadcast_transaction(tx).await?; + + let deadline = tokio::time::Instant::now() + wait; + loop { + match tokio::time::timeout_at(deadline, events.recv()).await { + // An interim `Uncertain` (the manager's acceptance timeout + // firing) is not returned early: a late echo can still + // upgrade the outcome, so keep waiting until the deadline. + Ok(Ok(SyncEvent::TransactionBroadcastResult { + txid: event_txid, + result, + })) if event_txid == txid && !matches!(result, BroadcastResult::Uncertain) => { + return Ok(result) + } + Ok(Ok(_)) => continue, + Ok(Err(RecvError::Lagged(skipped))) => { + tracing::warn!( + "Sync event stream lagged by {} while awaiting broadcast result", + skipped + ); + continue; + } + // Bus closed or deadline elapsed: no definitive outcome observed. + Ok(Err(RecvError::Closed)) | Err(_) => return Ok(BroadcastResult::Uncertain), + } + } + } } diff --git a/dash-spv/src/lib.rs b/dash-spv/src/lib.rs index ecf8f432a..b0263f08e 100644 --- a/dash-spv/src/lib.rs +++ b/dash-spv/src/lib.rs @@ -79,6 +79,7 @@ pub use error::{ LoggingError, LoggingResult, NetworkError, SpvError, StorageError, SyncError, ValidationError, }; pub use logging::{init_console_logging, init_logging, LogFileConfig, LoggingConfig, LoggingGuard}; +pub use sync::{BroadcastHoldout, BroadcastResult}; pub use tracing::level_filters::LevelFilter; pub use types::{FilterMatch, ValidationMode}; diff --git a/dash-spv/src/network/constants.rs b/dash-spv/src/network/constants.rs index e1278f0f6..30928f70e 100644 --- a/dash-spv/src/network/constants.rs +++ b/dash-spv/src/network/constants.rs @@ -2,9 +2,6 @@ use std::time::Duration; -// Connection limits -pub const TARGET_PEERS: usize = 3; - // Timeouts pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(30); pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); diff --git a/dash-spv/src/network/discovery.rs b/dash-spv/src/network/discovery.rs index 035bc7f28..74cb7bee7 100644 --- a/dash-spv/src/network/discovery.rs +++ b/dash-spv/src/network/discovery.rs @@ -12,6 +12,7 @@ //! Results from both sources are merged and deduplicated. use dashcore::Network; +use rand::seq::SliceRandom; use std::net::SocketAddr; /// DNS discovery for finding initial peers. @@ -60,6 +61,10 @@ impl DnsDiscovery { addresses.sort(); addresses.dedup(); + // Dedup needs the sort above, but a sorted list makes every client pick + // the same lowest-address peers via `take`/`truncate`, herding testers + // onto a handful of nodes. Shuffle so clients fan out across the set. + addresses.shuffle(&mut rand::thread_rng()); tracing::info!( "Discovered {} unique peer addresses for {:?} ({} from embedded seeds + DNS)", diff --git a/dash-spv/src/network/latency.rs b/dash-spv/src/network/latency.rs new file mode 100644 index 000000000..c9431f76e --- /dev/null +++ b/dash-spv/src/network/latency.rs @@ -0,0 +1,210 @@ +//! Per-peer response-latency tracking used to route sync requests. +//! +//! Routing quality is deliberately separate from the misbehavior score in +//! [`super::reputation`]. That score is an accusation: it only ever rises, and a +//! peer carrying no accusations is indistinguishable from one that has never been +//! tried. Routing needs the opposite property, evidence that expires back to +//! "unknown", so a peer that lost an early race is retried instead of starved. + +use std::collections::{HashMap, HashSet}; +use std::net::SocketAddr; +use std::time::{Duration, Instant}; + +/// Weight of the newest observation when folding it into a peer's mean, in +/// percent. Low enough that one slow response does not drop a peer out of +/// rotation, high enough that a peer going bad is demoted within a few requests. +const NEW_SAMPLE_WEIGHT: u32 = 30; + +/// How long an observation stays authoritative. Past this the peer counts as +/// unmeasured and is routed to again whatever its last mean, which is what stops +/// a peer from being frozen out permanently. This must exceed the stall timeout +/// plus a maintenance tick: a peer that is actually stalling then keeps refreshing +/// a bad mean and stays out of rotation, rather than aging back into it. +const SAMPLE_TTL: Duration = Duration::from_secs(30); + +/// A peer keeps receiving requests while its mean stays within this multiple of +/// the best measured peer's. +/// +/// Deliberately generous. Requests pipeline across peers, so a merely slower peer +/// still adds throughput and is worth keeping in rotation, and dropping it would +/// concentrate load on fewer peers for no gain. The job here is to shed peers that +/// have effectively stopped answering, which stand out by an order of magnitude +/// (a stall is recorded at ten seconds against sub-second healthy responses), not +/// to rank healthy peers against each other. +const SLOW_MULTIPLIER: u32 = 4; + +/// Absolute slack allowed alongside `SLOW_MULTIPLIER`, so that when peers are all +/// fast in absolute terms the multiple does not split hairs over normal jitter. +const SLOW_MARGIN: Duration = Duration::from_millis(500); + +#[derive(Debug, Clone, Copy)] +struct Observation { + mean: Duration, + at: Instant, +} + +/// Rolling response-time means for connected peers. +#[derive(Debug, Default)] +pub(super) struct PeerLatency { + observations: HashMap, +} + +impl PeerLatency { + /// Fold an observed response time into `addr`'s mean. + /// + /// A stalling request is recorded with the time it has been outstanding, which + /// is by definition at least the stall timeout, so a stalling peer's mean + /// climbs clear of any healthy peer's without needing a sentinel value. + pub(super) fn record(&mut self, addr: SocketAddr, elapsed: Duration) { + let now = Instant::now(); + match self.observations.get_mut(&addr) { + Some(observation) => { + observation.mean = blend(observation.mean, elapsed); + observation.at = now; + } + None => { + self.observations.insert( + addr, + Observation { + mean: elapsed, + at: now, + }, + ); + } + } + } + + /// Drop observations for peers that are no longer connected. + pub(super) fn retain_connected(&mut self, connected: &[SocketAddr]) { + self.observations.retain(|addr, _| connected.contains(addr)); + } + + /// The subset of `addrs` that should receive requests. + /// + /// Peers with no fresh observation are always eligible. We have no evidence + /// about them, and since only a request can produce evidence, excluding them + /// would make that lack of evidence permanent. The measured peers stay + /// eligible while they remain within reach of the best of them. + pub(super) fn eligible(&self, addrs: &[SocketAddr]) -> HashSet { + let now = Instant::now(); + let fresh = |addr: &SocketAddr| { + self.observations + .get(addr) + .filter(|observation| now.saturating_duration_since(observation.at) < SAMPLE_TTL) + }; + + let Some(best) = addrs.iter().filter_map(fresh).map(|observation| observation.mean).min() + else { + return addrs.iter().copied().collect(); + }; + let threshold = best.saturating_mul(SLOW_MULTIPLIER).max(best.saturating_add(SLOW_MARGIN)); + + addrs + .iter() + .copied() + .filter(|addr| fresh(addr).is_none_or(|observation| observation.mean <= threshold)) + .collect() + } +} + +/// Fold `sample` into `mean`, weighting the sample by `NEW_SAMPLE_WEIGHT`. +fn blend(mean: Duration, sample: Duration) -> Duration { + let old = mean.as_micros().saturating_mul(u128::from(100 - NEW_SAMPLE_WEIGHT)); + let new = sample.as_micros().saturating_mul(u128::from(NEW_SAMPLE_WEIGHT)); + let micros = (old.saturating_add(new) / 100).min(u128::from(u64::MAX)); + Duration::from_micros(micros as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn addr(last_octet: u8) -> SocketAddr { + SocketAddr::from(([127, 0, 0, last_octet], 9999)) + } + + #[test] + fn unmeasured_peers_are_always_eligible() { + let latency = PeerLatency::default(); + let peers = [addr(1), addr(2), addr(3)]; + + assert_eq!(latency.eligible(&peers).len(), 3); + } + + #[test] + fn slow_peer_drops_out_but_a_starved_peer_returns_when_its_sample_expires() { + let mut latency = PeerLatency::default(); + let (fast, slow) = (addr(1), addr(2)); + latency.record(fast, Duration::from_millis(50)); + latency.record(slow, Duration::from_secs(5)); + + let eligible = latency.eligible(&[fast, slow]); + assert!(eligible.contains(&fast)); + assert!(!eligible.contains(&slow), "a clearly slower peer stays out of rotation"); + + // Starvation is the failure this guards: once out of rotation the peer + // receives nothing, so only expiry can ever make it measurable again. + latency.observations.get_mut(&slow).expect("recorded above").at = + Instant::now() - SAMPLE_TTL - Duration::from_secs(1); + + assert!( + latency.eligible(&[fast, slow]).contains(&slow), + "an expired observation must put the peer back in rotation" + ); + } + + #[test] + fn comparable_peers_share_rotation() { + let mut latency = PeerLatency::default(); + let (a, b) = (addr(1), addr(2)); + latency.record(a, Duration::from_millis(20)); + latency.record(b, Duration::from_millis(90)); + + assert_eq!( + latency.eligible(&[a, b]).len(), + 2, + "peers that are all fast in absolute terms stay in rotation together" + ); + } + + #[test] + fn a_single_slow_response_does_not_evict_a_proven_peer() { + let mut latency = PeerLatency::default(); + let (fast, blip) = (addr(1), addr(2)); + latency.record(fast, Duration::from_millis(50)); + for _ in 0..10 { + latency.record(blip, Duration::from_millis(50)); + } + latency.record(blip, Duration::from_secs(1)); + + assert!( + latency.eligible(&[fast, blip]).contains(&blip), + "one blip should not undo a peer's track record" + ); + } + + #[test] + fn repeated_stalls_demote_a_peer() { + let mut latency = PeerLatency::default(); + let (fast, stalling) = (addr(1), addr(2)); + latency.record(fast, Duration::from_millis(50)); + latency.record(stalling, Duration::from_millis(50)); + for _ in 0..3 { + latency.record(stalling, Duration::from_secs(10)); + } + + assert!(!latency.eligible(&[fast, stalling]).contains(&stalling)); + } + + #[test] + fn disconnected_peers_are_forgotten() { + let mut latency = PeerLatency::default(); + latency.record(addr(1), Duration::from_millis(50)); + latency.record(addr(2), Duration::from_millis(50)); + + latency.retain_connected(&[addr(1)]); + + assert_eq!(latency.observations.len(), 1); + assert!(latency.observations.contains_key(&addr(1))); + } +} diff --git a/dash-spv/src/network/manager.rs b/dash-spv/src/network/manager.rs index e1984c0af..83404697e 100644 --- a/dash-spv/src/network/manager.rs +++ b/dash-spv/src/network/manager.rs @@ -1,12 +1,12 @@ //! Peer network manager for SPV client -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -use tokio::sync::{broadcast, Mutex, RwLock}; +use tokio::sync::{broadcast, Mutex, OwnedSemaphorePermit, RwLock, Semaphore}; use tokio::task::JoinSet; use tokio::time; @@ -15,10 +15,9 @@ use crate::error::{NetworkError, NetworkResult, SpvError as Error}; use crate::network::addrv2::AddrV2Handler; use crate::network::constants::*; use crate::network::discovery::DnsDiscovery; +use crate::network::latency::PeerLatency; use crate::network::pool::PeerPool; -use crate::network::reputation::{ - misbehavior_scores, positive_scores, PeerReputationManager, ReputationAware, -}; +use crate::network::reputation::{ChangeReason, PeerReputationManager, ReputationAware}; use crate::network::{ HandshakeManager, Message, MessageDispatcher, MessageType, NetworkEvent, NetworkManager, NetworkRequest, Peer, RequestSender, @@ -28,18 +27,138 @@ use async_trait::async_trait; use dashcore::network::address::{AddrV2, AddrV2Message}; use dashcore::network::constants::ServiceFlags; use dashcore::network::message::NetworkMessage; +use dashcore::network::message_blockdata::Inventory; use dashcore::network::message_headers2::CompressionState; -use dashcore::Network; +use dashcore::{BlockHash, Header, Network}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use tokio::time::Instant; use tokio_util::sync::CancellationToken; const DEFAULT_NETWORK_EVENT_CAPACITY: usize = 10000; +const MAX_CONCURRENT_HEADERS2_DECOMPRESSIONS: usize = 4; + +fn headers2_decompression_parallelism() -> usize { + std::thread::available_parallelism() + .map(|parallelism| parallelism.get().min(MAX_CONCURRENT_HEADERS2_DECOMPRESSIONS)) + .unwrap_or(1) +} + +enum HeaderDispatchOutcome { + Compressed(Vec<(Header, BlockHash)>), + Regular(Vec
), + Invalid(String), + TaskFailed(String), +} + +/// Releases header messages in wire order because an announcement can depend on the preceding +/// message from the same peer. +struct OrderedHeaderResults { + next_sequence: u64, + completed: BTreeMap, +} + +impl Default for OrderedHeaderResults { + fn default() -> Self { + Self { + next_sequence: 0, + completed: BTreeMap::new(), + } + } +} + +impl OrderedHeaderResults { + fn complete(&mut self, sequence: u64, result: T) -> Vec { + let replaced = self.completed.insert(sequence, result); + debug_assert!(replaced.is_none(), "header sequence completed twice"); + + let mut ready = Vec::new(); + while let Some(result) = self.completed.remove(&self.next_sequence) { + ready.push(result); + self.next_sequence += 1; + } + ready + } +} + +#[derive(Clone)] +struct HeaderDispatchContext { + addr: SocketAddr, + session_token: CancellationToken, + ordered_results: + Arc>>, + message_dispatcher: Arc>, + headers2_disabled: Arc>>, + reputation_manager: Arc, +} + +impl HeaderDispatchContext { + async fn complete( + &self, + sequence: u64, + outcome: HeaderDispatchOutcome, + permit: OwnedSemaphorePermit, + ) { + if self.session_token.is_cancelled() { + return; + } + + let mut ordered_results = self.ordered_results.lock().await; + // Retain each permit while its result waits for earlier messages, keeping the reorder + // buffer within the worker limit. Keep the ordering lock through dispatch so another + // completed task cannot overtake this batch after its sequence has been released. + let ready = ordered_results.complete(sequence, (outcome, permit)); + + for (outcome, _permit) in ready { + if self.session_token.is_cancelled() { + return; + } + + match outcome { + HeaderDispatchOutcome::Compressed(headers_with_hashes) => { + tracing::info!( + "Decompressed {} headers from {} - forwarding as regular Headers", + headers_with_hashes.len(), + self.addr + ); + let message = Message::new_headers(self.addr, headers_with_hashes); + let mut message_dispatcher = self.message_dispatcher.lock().await; + if self.session_token.is_cancelled() { + return; + } + message_dispatcher.dispatch(&message); + } + HeaderDispatchOutcome::Regular(headers) => { + let message = Message::new(self.addr, NetworkMessage::Headers(headers)); + let mut message_dispatcher = self.message_dispatcher.lock().await; + if self.session_token.is_cancelled() { + return; + } + message_dispatcher.dispatch(&message); + } + HeaderDispatchOutcome::Invalid(e) => { + tracing::error!( + "Headers2 decompression failed from {}: {} - disabling headers2", + self.addr, + e + ); + self.headers2_disabled.lock().await.insert(self.addr); + self.reputation_manager + .update_reputation(self.addr, ChangeReason::Headers2DecompressionFailed) + .await; + } + HeaderDispatchOutcome::TaskFailed(e) => { + tracing::error!("Headers2 decompression task failed for {}: {}", self.addr, e); + } + } + } + } +} /// Peer network manager pub struct PeerNetworkManager { /// Peer pool pool: Arc, + max_peers: usize, /// DNS discovery discovery: Arc, /// AddrV2 handler @@ -71,6 +190,8 @@ pub struct PeerNetworkManager { connected_peer_count: Arc, /// Disable headers2 after decompression failure headers2_disabled: Arc>>, + /// Global bound for CPU-heavy headers2 decompression work. + headers2_decompression_semaphore: Arc, /// Dispatcher for unbounded and message-type filtered message distribution. message_dispatcher: Arc>, /// Request queue sender, cloneable handle for sending requests to the network manager. @@ -81,10 +202,69 @@ pub struct PeerNetworkManager { round_robin_counter: Arc, /// Network event bus for notifying about network/peer related changes. network_event_sender: broadcast::Sender, + /// Instant each peer's oldest unanswered request of a given kind was sent. + /// The answering response clears the entry, a stale one marks a stalling peer. + outstanding_requests: Arc>>, + /// Rolling response times per peer, used to route requests to the peers that + /// are actually answering fastest. + latency: Arc>, + /// Instant of the previous stall sweep, used to detect a suspend/resume gap so + /// a burst of stale requests on resume does not penalize every peer at once. + last_maintenance_at: Arc>, + /// Instant of the last reputation-driven eviction, enforcing a cooldown so a + /// replacement can prove itself before another peer is dropped. + last_eviction_at: Arc>>, } const CAPABILITY_REJECTED_TTL: Duration = Duration::from_secs(30 * 60); +/// A connected peer whose oldest sync request stays unanswered this long is +/// treated as stalling and penalized. Any response resets its timer, so a peer +/// streaming a large block is not punished for the download taking a while. +/// +/// Only kinds whose response time is a fair measure of the peer are judged here +/// (see `RequestKind::response_time_is_fair`), so this is generous: a filter, +/// filter-header or header response is at most a few tens of KB and arrives in +/// well under a second on any usable link. Large payloads are excluded, since +/// their transfer can legitimately exceed this on a slow mobile link. +pub(super) const REQUEST_STALL_TIMEOUT: Duration = Duration::from_secs(10); + +/// A peer whose request of some kind has gone unanswered this long stops being +/// picked for further requests of that same kind, until it answers. +/// +/// This is routing only, never a penalty, so it can be far stricter than +/// `REQUEST_STALL_TIMEOUT` without risking an honest peer: the cost of skipping +/// one wrongly is that another peer serves the request instead. It must stay +/// below the sync layer's own retry timeouts, so that a peer which dropped a +/// request is already out of the rotation by the time that request is reissued. +pub(super) const REQUEST_OWED_TIMEOUT: Duration = Duration::from_secs(5); + +/// If a maintenance sweep runs at least this long after the previous one, the +/// process was likely suspended (e.g. an iOS app backgrounded). Every +/// outstanding request would look stale at once, so the sweep refreshes their +/// timers and skips penalties for that round instead of blaming every peer. +const SUSPEND_GAP: Duration = Duration::from_secs(90); + +/// A connected peer scoring at least this becomes an eviction candidate: two +/// consecutive stalls, or equivalent misbehavior. Well below the +100 ban line, +/// so a peer is dropped and replaced long before it would be banned outright, and +/// eviction stays a soft demotion (the peer keeps its address-book entry and +/// decays back). +/// +/// Reachable only because a stalling peer's timer is re-armed rather than +/// dropped: one stall already records a mean response time that freezes the peer +/// out of routing, so it receives no new request to re-arm from and could +/// otherwise never earn a second strike. +const STUCK_PEER_EVICTION_SCORE: i32 = 20; + +/// Never evict a peer for reputation while at or below this many connections, so +/// a small pool is never churned down toward zero usable peers. +const MIN_CONNECTED_FLOOR: usize = 2; + +/// Minimum spacing between reputation-driven evictions, so a fresh replacement +/// has time to connect and prove itself before another peer is dropped. +const EVICTION_COOLDOWN: Duration = Duration::from_secs(30); + fn required_services_from_config(config: &ClientConfig, exclusive_mode: bool) -> ServiceFlags { if exclusive_mode { return ServiceFlags::NONE; @@ -117,8 +297,11 @@ impl PeerNetworkManager { // Create request queue for outgoing messages let (request_tx, request_rx) = unbounded_channel(); + let max_peers = config.max_peers.max(1) as usize; + Ok(Self { - pool: Arc::new(PeerPool::new()), + pool: Arc::new(PeerPool::new(max_peers)), + max_peers, discovery: Arc::new(discovery), addrv2_handler: Arc::new(AddrV2Handler::new()), peer_store: Arc::new(peer_store), @@ -134,11 +317,18 @@ impl PeerNetworkManager { capability_rejected: Arc::new(RwLock::new(HashMap::new())), connected_peer_count: Arc::new(AtomicUsize::new(0)), headers2_disabled: Arc::new(Mutex::new(HashSet::new())), + headers2_decompression_semaphore: Arc::new(Semaphore::new( + headers2_decompression_parallelism(), + )), message_dispatcher: Arc::new(Mutex::new(MessageDispatcher::default())), request_tx, request_rx: Arc::new(Mutex::new(Some(request_rx))), round_robin_counter: Arc::new(AtomicUsize::new(0)), network_event_sender: broadcast::Sender::new(DEFAULT_NETWORK_EVENT_CAPACITY), + outstanding_requests: Arc::new(Mutex::new(HashMap::new())), + latency: Arc::new(Mutex::new(PeerLatency::default())), + last_maintenance_at: Arc::new(Mutex::new(Instant::now())), + last_eviction_at: Arc::new(Mutex::new(None)), }) } @@ -194,7 +384,7 @@ impl PeerNetworkManager { peer_addresses.extend( dns_peers .into_iter() - .take(TARGET_PEERS) + .take(self.max_peers) .map(|addr| AddrV2Message::new(addr, ServiceFlags::NETWORK)), ); tracing::info!( @@ -247,11 +437,14 @@ impl PeerNetworkManager { let addrv2_handler = self.addrv2_handler.clone(); let shutdown_token = self.shutdown_token.clone(); let reputation_manager = self.reputation_manager.clone(); + let outstanding_requests = self.outstanding_requests.clone(); + let latency = self.latency.clone(); let user_agent = self.user_agent.clone(); let required_services = self.required_services; let capability_rejected = self.capability_rejected.clone(); let connected_peer_count = self.connected_peer_count.clone(); let headers2_disabled = self.headers2_disabled.clone(); + let headers2_decompression_semaphore = self.headers2_decompression_semaphore.clone(); let message_dispatcher = self.message_dispatcher.clone(); let network_event_sender = self.network_event_sender.clone(); @@ -311,9 +504,14 @@ impl PeerNetworkManager { // Record successful connection reputation_manager.record_successful_connection(addr).await; - // Add to pool + // Both ways this can fail, a full pool and an address + // already connected, are ordinary outcomes of dialling + // several candidates for the same slot: one wins and the + // rest arrive to find it taken. Topping up after an + // eviction does exactly that, so logging it as an error + // reports routine contention as a fault. if let Err(e) = pool.add_peer(addr, peer).await { - tracing::error!("Failed to add peer to pool: {}", e); + tracing::debug!("Not adding peer {} to pool: {}", addr, e); return; } @@ -343,8 +541,11 @@ impl PeerNetworkManager { addrv2_handler, shutdown_token, reputation_manager.clone(), + outstanding_requests, + latency, connected_peer_count.clone(), headers2_disabled.clone(), + headers2_decompression_semaphore.clone(), message_dispatcher, network_event_sender.clone(), ) @@ -356,11 +557,7 @@ impl PeerNetworkManager { pool.remove_peer(&addr).await; // Update reputation for handshake failure reputation_manager - .update_reputation( - addr, - misbehavior_scores::INVALID_MESSAGE, - "Handshake failed", - ) + .update_reputation(addr, ChangeReason::HandshakeFailed) .await; // For handshake failures, try again later tokio::time::sleep(RECONNECT_DELAY).await; @@ -373,11 +570,7 @@ impl PeerNetworkManager { pool.remove_peer(&addr).await; // Minor reputation penalty for connection failure reputation_manager - .update_reputation( - addr, - misbehavior_scores::TIMEOUT / 2, - "Connection failed", - ) + .update_reputation(addr, ChangeReason::ConnectionFailed) .await; } } @@ -431,21 +624,32 @@ impl PeerNetworkManager { addrv2_handler: Arc, shutdown_token: CancellationToken, reputation_manager: Arc, + outstanding_requests: Arc>>, + latency: Arc>, connected_peer_count: Arc, headers2_disabled: Arc>>, + headers2_decompression_semaphore: Arc, message_dispatcher: Arc>, network_event_sender: broadcast::Sender, ) { tokio::spawn(async move { tracing::debug!("Starting peer reader loop for {}", addr); let mut loop_iteration = 0; - let mut headers2_state = CompressionState::default(); - + let mut header_sequence = 0_u64; + let session_token = shutdown_token.child_token(); + let header_dispatch = HeaderDispatchContext { + addr, + session_token: session_token.clone(), + ordered_results: Arc::new(Mutex::new(OrderedHeaderResults::default())), + message_dispatcher: message_dispatcher.clone(), + headers2_disabled: headers2_disabled.clone(), + reputation_manager: reputation_manager.clone(), + }; loop { loop_iteration += 1; // Check shutdown signal first with detailed logging - if shutdown_token.is_cancelled() { + if session_token.is_cancelled() { tracing::info!("Breaking peer reader loop for {} - shutdown signal received (iteration {})", addr, loop_iteration); break; } @@ -479,7 +683,7 @@ impl PeerNetworkManager { _ = tokio::time::sleep(MESSAGE_POLL_INTERVAL) => { Ok(None) }, - _ = shutdown_token.cancelled() => { + _ = session_token.cancelled() => { tracing::info!("Breaking peer reader loop for {} - shutdown signal received while reading (iteration {})", addr, loop_iteration); break; } @@ -489,7 +693,94 @@ impl PeerNetworkManager { match msg_result { Ok(Some(msg)) => { // Log all received messages at debug level to help troubleshoot - tracing::debug!("Received {:?} from {}", msg.cmd(), addr); + tracing::trace!("Received {:?} from {}", msg.cmd(), addr); + + // A substantive response clearing a tracked request is the one + // point where a peer's speed is observable, so time it here. + // The elapsed time runs from the oldest request still + // unanswered, so it counts queueing behind that peer's own + // backlog as well as its service time. That is deliberate: + // both delay us equally, and it lets a peer we are + // over-feeding report itself as congested. + if let Some(kind) = timed_response_kind(msg.inner()) { + let sent = outstanding_requests.lock().await.remove(&(addr, kind)); + if let Some(sent) = sent { + if kind.response_time_is_fair() { + latency.lock().await.record(addr, sent.elapsed()); + } + } + } + + let inner = msg.into_inner(); + if let NetworkMessage::Headers2(headers2) = inner { + tracing::info!( + "Received Headers2 from {} with {} compressed headers - decompressing", + addr, + headers2.headers.len() + ); + + let permit = tokio::select! { + permit = headers2_decompression_semaphore.clone().acquire_owned() => { + match permit { + Ok(permit) => permit, + Err(_) => break, + } + } + _ = session_token.cancelled() => break, + }; + let header_dispatch = header_dispatch.clone(); + let sequence = header_sequence; + header_sequence += 1; + tokio::spawn(async move { + let result = tokio::task::spawn_blocking(move || { + let mut state = CompressionState::default(); + state.process_headers_with_hashes(&headers2.headers) + }) + .await; + if header_dispatch.session_token.is_cancelled() { + return; + } + + let outcome = match result { + Ok(Ok(headers_with_hashes)) => { + HeaderDispatchOutcome::Compressed(headers_with_hashes) + } + Ok(Err(e)) => HeaderDispatchOutcome::Invalid(e.to_string()), + Err(e) => HeaderDispatchOutcome::TaskFailed(e.to_string()), + }; + header_dispatch.complete(sequence, outcome, permit).await; + }); + continue; + } + if let NetworkMessage::Headers(headers) = inner { + tracing::info!( + "📨 Received Headers message from {} with {} headers! (regular uncompressed)", + addr, + headers.len() + ); + let peer_guard = peer.read().await; + if peer_guard.supports_headers2() { + tracing::warn!("⚠️ Peer {} supports headers2 but sent regular headers - possible protocol issue", addr); + } + drop(peer_guard); + + let permit = tokio::select! { + permit = headers2_decompression_semaphore.clone().acquire_owned() => { + match permit { + Ok(permit) => permit, + Err(_) => break, + } + } + _ = session_token.cancelled() => break, + }; + let sequence = header_sequence; + header_sequence += 1; + header_dispatch + .complete(sequence, HeaderDispatchOutcome::Regular(headers), permit) + .await; + continue; + } + let msg = Message::new(addr, inner); // Handle some messages directly match &msg.inner() { @@ -587,60 +878,8 @@ impl PeerNetworkManager { } continue; } - NetworkMessage::Headers(headers) => { - // Log headers messages specifically - tracing::info!( - "📨 Received Headers message from {} with {} headers! (regular uncompressed)", - addr, - headers.len() - ); - // Check if peer supports headers2 - let peer_guard = peer.read().await; - if peer_guard.supports_headers2() { - tracing::warn!("⚠️ Peer {} supports headers2 but sent regular headers - possible protocol issue", addr); - } - drop(peer_guard); - // Forward to client - } - NetworkMessage::Headers2(headers2) => { - // Decompress headers in network layer and forward as regular Headers - tracing::info!( - "Received Headers2 from {} with {} compressed headers - decompressing", - addr, - headers2.headers.len() - ); - - match headers2_state.process_headers(&headers2.headers) { - Ok(headers) => { - tracing::info!( - "Decompressed {} headers from {} - forwarding as regular Headers", - headers.len(), - addr - ); - // Forward as regular Headers message - let headers_msg = NetworkMessage::Headers(headers); - let message = Message::new(msg.peer_address(), headers_msg); - message_dispatcher.lock().await.dispatch(&message); - continue; // Already sent, don't forward the original Headers2 - } - Err(e) => { - tracing::error!( - "Headers2 decompression failed from {}: {} - disabling headers2", - addr, - e - ); - headers2_disabled.lock().await.insert(addr); - // Apply reputation penalty - reputation_manager - .update_reputation( - addr, - misbehavior_scores::INVALID_MESSAGE, - "Headers2 decompression failed", - ) - .await; - continue; // Don't forward corrupted message - } - } + NetworkMessage::Headers(_) | NetworkMessage::Headers2(_) => { + unreachable!() } NetworkMessage::GetHeaders(_) => { // SPV clients don't serve headers to peers @@ -691,15 +930,9 @@ impl PeerNetworkManager { break; } NetworkError::Timeout => { + // Idle socket reads time out constantly on healthy + // connections, so this is not a quality signal. tracing::debug!("Timeout reading from {}, continuing...", addr); - // Minor reputation penalty for timeout - reputation_manager - .update_reputation( - addr, - misbehavior_scores::TIMEOUT, - "Read timeout", - ) - .await; continue; } _ => { @@ -718,8 +951,7 @@ impl PeerNetworkManager { reputation_manager .update_reputation( addr, - misbehavior_scores::INVALID_TRANSACTION, - "Invalid transaction type in block", + ChangeReason::InvalidTransactionInBlock, ) .await; } else if error_msg @@ -765,6 +997,7 @@ impl PeerNetworkManager { } // Remove from pool and notify consumers + session_token.cancel(); tracing::warn!("Disconnecting from {} (peer reader loop ended)", addr); Self::remove_peer_and_notify( &pool, @@ -775,15 +1008,6 @@ impl PeerNetworkManager { .await; headers2_disabled.lock().await.remove(&addr); - - // Give small positive reputation if peer maintained long connection - let conn_duration = Duration::from_secs(60 * loop_iteration); // Rough estimate - if conn_duration > Duration::from_secs(3600) { - // 1 hour - reputation_manager - .update_reputation(addr, positive_scores::LONG_UPTIME, "Long connection uptime") - .await; - } }); } @@ -811,7 +1035,7 @@ impl PeerNetworkManager { request = request_rx.recv() => { match request { Some(NetworkRequest::SendMessage(msg)) => { - tracing::debug!("Request processor: sending {}", msg.cmd()); + tracing::trace!("Request processor: sending {}", msg.cmd()); // Spawn each send concurrently to allow parallel requests across peers. let this = this.clone(); tokio::spawn(async move { @@ -836,7 +1060,7 @@ impl PeerNetworkManager { }); } Some(NetworkRequest::SendMessageToPeer(msg, peer_address)) => { - tracing::debug!("Request processor: sending {} to peer {}", msg.cmd(), peer_address); + tracing::trace!("Request processor: sending {} to peer {}", msg.cmd(), peer_address); let this = this.clone(); tokio::spawn(async move { let fallback_msg = msg.clone(); @@ -941,6 +1165,159 @@ impl PeerNetworkManager { } } + /// Penalize connected peers that have left a sync request unanswered past the + /// stall timeout, so they fall out of routing and become eviction candidates. + /// A long gap since the previous sweep is treated as a process suspend and the + /// timers are refreshed instead, so a backgrounded client does not blame every + /// peer at once on resume. + /// Returns true if this tick was treated as a suspend/resume grace tick, so + /// the caller can skip eviction rather than acting on stale pre-suspend state. + async fn sweep_stalled_peers(&self) -> bool { + let now = Instant::now(); + let gap = { + let mut last = self.last_maintenance_at.lock().await; + let gap = now.saturating_duration_since(*last); + *last = now; + gap + }; + + // Snapshot connected peers before taking the outstanding lock so this path + // never holds the outstanding lock while touching the pool lock. + let connected = self.pool.get_connected_addresses().await; + self.latency.lock().await.retain_connected(&connected); + + let stalled = { + let mut outstanding = self.outstanding_requests.lock().await; + outstanding.retain(|(addr, _), _| connected.contains(addr)); + + if gap > SUSPEND_GAP { + for sent in outstanding.values_mut() { + *sent = now; + } + return true; + } + + let aged: Vec<((SocketAddr, RequestKind), Duration)> = outstanding + .iter() + .filter(|((_, kind), _)| kind.response_time_is_fair()) + .filter_map(|(key, sent)| { + let waited = now.saturating_duration_since(*sent); + (waited > REQUEST_STALL_TIMEOUT).then_some((*key, waited)) + }) + .collect(); + // Re-arm rather than drop: one stall already freezes the peer out of + // routing, so it gets no new request to re-arm the timer, and a still + // unanswered request must keep counting for it to ever be evicted. + for (key, _) in &aged { + outstanding.insert(*key, now); + } + + // Collapse to one entry per peer, keeping its worst wait. A peer + // stalling on several kinds at once is one failing peer, not several, + // and must not take several strikes from a single sweep. + let mut stalled: HashMap = HashMap::new(); + for ((addr, _), waited) in aged { + let worst = stalled.entry(addr).or_insert(waited); + *worst = (*worst).max(waited); + } + stalled + }; + + // Recording how long the request has gone unanswered keeps the stalling + // peer's measured mean fresh as well as bad, so it stays out of rotation + // instead of ageing back into it while it is still failing to answer. + { + let mut latency = self.latency.lock().await; + for (addr, waited) in &stalled { + latency.record(*addr, *waited); + } + } + + // A peer already at the eviction threshold is condemned, so further strikes + // add nothing and would only march it toward an outright ban. + let scores = self.reputation_manager.scores_for(stalled.keys().copied()).await; + for (addr, _) in stalled { + if scores.get(&addr).copied().unwrap_or(0) >= STUCK_PEER_EVICTION_SCORE { + continue; + } + tracing::debug!("Peer {} stalled on a sync request, penalizing", addr); + self.reputation_manager.update_reputation(addr, ChangeReason::RequestTimeout).await; + } + false + } + + /// Evict the single worst-scoring connected peer when it is clearly bad and a + /// healthy replacement is available, so the client stops staying stuck on a + /// peer that stalls. Heavily guarded so a small pool is never churned toward + /// zero: it keeps a connection floor, only acts on a full pool with a + /// replacement ready, skips when every peer is equally bad (a network or + /// device problem, not a peer problem), never drops the sole peer providing a + /// required service, and evicts at most one peer per cooldown. + async fn evict_worst_stuck_peer(&self) { + if let Some(t) = *self.last_eviction_at.lock().await { + if Instant::now().saturating_duration_since(t) < EVICTION_COOLDOWN { + return; + } + } + + let connected = self.pool.get_connected_addresses().await; + let floor = self.max_peers.min(MIN_CONNECTED_FLOOR); + if connected.len() < self.max_peers || connected.len() <= floor { + return; + } + + // A replacement must be ready, or eviction is pure loss. + let mut has_replacement = false; + for known in self.addrv2_handler.get_known_addresses().await { + let Ok(sa) = known.socket_addr() else { + continue; + }; + if !connected.contains(&sa) && !self.is_capability_rejected(&sa).await { + has_replacement = true; + break; + } + } + if !has_replacement { + return; + } + + let scores = self.reputation_manager.scores_for(connected.iter().copied()).await; + let Some(min_score) = scores.values().copied().min() else { + return; + }; + let Some((worst_addr, worst_score)) = scores.into_iter().max_by_key(|(_, s)| *s) else { + return; + }; + + // The worst must be clearly bad, and at least one peer clearly better, + // otherwise the whole pool is bad and the problem is not this peer. + if worst_score < STUCK_PEER_EVICTION_SCORE || min_score >= STUCK_PEER_EVICTION_SCORE { + return; + } + + if self.is_sole_service_provider(&worst_addr).await { + return; + } + + tracing::info!( + "Evicting stalled peer {} (score {}) so a fresh peer can replace it", + worst_addr, + worst_score + ); + let _ = self.disconnect_peer(&worst_addr, "poor reputation (stalled requests)").await; + *self.last_eviction_at.lock().await = Some(Instant::now()); + } + + /// Whether `addr` is the only connected peer advertising the required service, + /// so evicting it would strand a sync phase with no capable peer. + async fn is_sole_service_provider(&self, addr: &SocketAddr) -> bool { + if self.required_services == ServiceFlags::NONE { + return false; + } + let providers = self.pool.peers_with_service(self.required_services).await; + providers.len() == 1 && providers[0].0 == *addr + } + async fn maintenance_tick(&self) { // Remove peers that the reader loop failed to clean up. // This should not trigger under normal operation. @@ -969,15 +1346,23 @@ impl PeerNetworkManager { } } } else { + // Penalize peers stalling on sync requests so they drop out of routing + // and become eviction candidates before the top-up below. + let grace_tick = self.sweep_stalled_peers().await; // Evict peers that lack required services before top-up so replacements // can be pulled in during the same tick. self.evict_mismatched_peers().await; + // Drop one clearly-bad stalling peer, but never on a resume grace tick + // where scores reflect stale pre-suspend state. + if !grace_tick { + self.evict_worst_stuck_peer().await; + } // Re-read count after potential churn so top-up sees the current pool size. let count = self.pool.peer_count().await; - if count < TARGET_PEERS { + if count < self.max_peers { // Try known addresses first, sorted by reputation let known = self.addrv2_handler.get_known_addresses().await; - let needed = TARGET_PEERS.saturating_sub(count); + let needed = self.max_peers.saturating_sub(count); // Select best peers based on reputation let best_peers = self.reputation_manager.select_best_peers(known, needed * 2).await; let mut attempted = 0; @@ -1009,9 +1394,7 @@ impl PeerNetworkManager { if let Err(e) = peer_guard.send_ping().await { tracing::error!("Failed to ping {}: {}", addr, e); // Update reputation for ping failure - self.reputation_manager - .update_reputation(addr, misbehavior_scores::TIMEOUT, "Ping failed") - .await; + self.reputation_manager.update_reputation(addr, ChangeReason::PingFailed).await; } } let has_expired = peer_guard.remove_expired_pings(); @@ -1039,7 +1422,7 @@ impl PeerNetworkManager { async fn dns_fallback_tick(&self) { let count = self.pool.peer_count().await; - if count >= TARGET_PEERS { + if count >= self.max_peers { return; } let dns_peers = tokio::select! { @@ -1049,7 +1432,7 @@ impl PeerNetworkManager { return } }; - let needed = TARGET_PEERS.saturating_sub(count); + let needed = self.max_peers.saturating_sub(count); tracing::debug!("DNS fallback tick found {} addresses. Needed {}", dns_peers.len(), needed); let mut dns_attempted = 0; for addr in dns_peers.iter() { @@ -1080,7 +1463,7 @@ impl PeerNetworkManager { while !this.shutdown_token.is_cancelled() { tokio::select! { _ = maintenance_interval.tick() => { - tracing::debug!("Maintenance interval elapsed"); + tracing::trace!("Maintenance interval elapsed"); this.maintenance_tick().await; } _ = dns_interval.tick(), if !this.exclusive_mode => { @@ -1144,10 +1527,10 @@ impl PeerNetworkManager { tracing::warn!("No peers support {}, cannot send {}", flags, message.cmd()); return Err(NetworkError::ProtocolError(format!("No peers support {}", flags))); } - None => self.next_peer(&peers), + None => self.next_peer(&peers).await, } } else { - self.next_peer(&peers) + self.next_peer(&peers).await }; self.send_message_to_peer(&addr, &peer, message).await @@ -1200,20 +1583,92 @@ impl PeerNetworkManager { }; } - let (addr, peer) = self.next_peer(&selected_peers); + let selected_peers = match tracked_request_kind(&message) { + Some(kind) => self.without_peers_owing(kind, selected_peers).await, + None => selected_peers, + }; + + let (addr, peer) = self.next_peer(&selected_peers).await; - tracing::debug!("Distributing {} request to peer {}", message.cmd(), addr); + tracing::trace!("Distributing {} request to peer {}", message.cmd(), addr); self.send_message_to_peer(&addr, &peer, message).await } - /// Pick the next peer from `peers` using round-robin rotation. - fn next_peer( + /// Drop peers that already owe us a response of `kind` from the candidates. + /// + /// A peer that silently discards a request is invisible to latency-based + /// routing: latency only ever records a response, so a peer that answers + /// nothing is never measured and keeps its turn in the rotation. It then wins + /// the retry of the very request it just dropped, which is how one such peer + /// turned a 30s block timeout into 60s and 120s stalls, with filter sync + /// halted behind it the whole time. + /// + /// The bar is what the peer owes rather than what it did wrong, so this needs + /// no judgement about how slow is too slow and never accuses an honest peer on + /// a slow link. It is also self-clearing: the peer becomes a candidate again + /// the moment it answers, or the moment the request is satisfied elsewhere. + /// + /// Skipping is only ever a preference. If every candidate owes us something, + /// the full set is kept, because refusing to send is worse than sending to a + /// busy peer. + async fn without_peers_owing( + &self, + kind: RequestKind, + peers: Vec<(SocketAddr, Arc>)>, + ) -> Vec<(SocketAddr, Arc>)> { + let now = Instant::now(); + let owing: HashSet = { + let outstanding = self.outstanding_requests.lock().await; + outstanding + .iter() + .filter(|((_, k), sent)| { + *k == kind && now.saturating_duration_since(**sent) > REQUEST_OWED_TIMEOUT + }) + .map(|((addr, _), _)| *addr) + .collect() + }; + + if owing.is_empty() { + return peers; + } + let free: Vec<(SocketAddr, Arc>)> = + peers.iter().filter(|(addr, _)| !owing.contains(addr)).cloned().collect(); + if free.is_empty() { + return peers; + } + free + } + + /// Pick a peer from `peers`, rotating over those answering fastest. + /// + /// Selection reads measured response times rather than the misbehavior score. + /// A score only ever accuses a peer, so a peer that is merely untried scores + /// the same as a good one, and rewarding responses to break that tie makes the + /// first peer to answer pull away and take the whole rotation: it earns more + /// traffic, which earns it more reward, while the peers it starves get no + /// traffic and so can never earn their way back. Latency has no such ratchet. + /// It expires, so a starved peer becomes unmeasured and is retried, and it is + /// bounded by what the peer actually did rather than by how often we picked it. + /// + /// The pool guard is already released by the caller (`get_all_peers` returns + /// owned `Arc`s), so taking the latency lock here keeps a single lock order. + async fn next_peer( &self, peers: &[(SocketAddr, Arc>)], ) -> (SocketAddr, Arc>) { - let idx = self.round_robin_counter.fetch_add(1, Ordering::Relaxed) % peers.len(); - (peers[idx].0, peers[idx].1.clone()) + let addrs: Vec = peers.iter().map(|(addr, _)| *addr).collect(); + let eligible_addrs = self.latency.lock().await.eligible(&addrs); + let eligible: Vec<&(SocketAddr, Arc>)> = + peers.iter().filter(|(addr, _)| eligible_addrs.contains(addr)).collect(); + + // `eligible` only ever narrows `peers`, and never to nothing: a peer with + // no fresh measurement always qualifies, so an empty result would mean + // every peer was measured and none was within reach of the fastest, which + // the fastest itself contradicts. + let idx = self.round_robin_counter.fetch_add(1, Ordering::Relaxed) % eligible.len(); + let (addr, peer) = eligible[idx]; + (*addr, peer.clone()) } /// Send a message to the given peer. @@ -1237,11 +1692,27 @@ impl PeerNetworkManager { other => other, }; - let mut peer_guard = peer.write().await; - peer_guard - .send_message(message) - .await - .map_err(|e| NetworkError::ProtocolError(format!("Failed to send to {}: {}", addr, e))) + let request_kind = tracked_request_kind(&message); + let send_result = { + let mut peer_guard = peer.write().await; + peer_guard.send_message(message).await + }; + let result = send_result + .map_err(|e| NetworkError::ProtocolError(format!("Failed to send to {}: {}", addr, e))); + + // Arm the stall timer for request-type messages, keeping the oldest + // unanswered timestamp so a peer that never replies is caught. The peer + // lock is already released, so this only ever holds the outstanding lock. + if result.is_ok() { + if let Some(kind) = request_kind { + self.outstanding_requests + .lock() + .await + .entry((*addr, kind)) + .or_insert_with(Instant::now); + } + } + result } /// Broadcast a message to all connected peers @@ -1312,13 +1783,7 @@ impl PeerNetworkManager { self.disconnect_peer(addr, reason).await?; // Update reputation to trigger ban - self.reputation_manager - .update_reputation( - *addr, - misbehavior_scores::INVALID_HEADER * 2, // Severe penalty - reason, - ) - .await; + self.reputation_manager.update_reputation(*addr, ChangeReason::ManuallyBanned).await; Ok(()) } @@ -1399,6 +1864,7 @@ impl Clone for PeerNetworkManager { fn clone(&self) -> Self { Self { pool: self.pool.clone(), + max_peers: self.max_peers, discovery: self.discovery.clone(), addrv2_handler: self.addrv2_handler.clone(), peer_store: self.peer_store.clone(), @@ -1414,11 +1880,16 @@ impl Clone for PeerNetworkManager { capability_rejected: self.capability_rejected.clone(), connected_peer_count: self.connected_peer_count.clone(), headers2_disabled: self.headers2_disabled.clone(), + headers2_decompression_semaphore: self.headers2_decompression_semaphore.clone(), message_dispatcher: self.message_dispatcher.clone(), request_tx: self.request_tx.clone(), request_rx: self.request_rx.clone(), round_robin_counter: self.round_robin_counter.clone(), network_event_sender: self.network_event_sender.clone(), + outstanding_requests: self.outstanding_requests.clone(), + latency: self.latency.clone(), + last_maintenance_at: self.last_maintenance_at.clone(), + last_eviction_at: self.last_eviction_at.clone(), } } } @@ -1509,6 +1980,81 @@ impl NetworkManager for PeerNetworkManager { } } +/// The sync request a timer belongs to. +/// +/// Timers are kept per kind rather than per peer, so a response can only clear a +/// request it actually answers. Sharing one timer across kinds lets a peer that is +/// fast at one kind hide being slow at another: its quick responses clear the +/// timer the slow request armed, the slow response then finds nothing to clear and +/// is never measured, and the stall sweep never sees an aged entry to penalize. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum RequestKind { + Headers, + FilterHeaders, + Filters, + Blocks, +} + +impl RequestKind { + /// Whether how long this kind took is a fair measure of the peer, and so may + /// be scored against it: recorded as its response time, and penalized when it + /// exceeds `REQUEST_STALL_TIMEOUT`. + /// + /// Only kinds with a small, fixed-size response qualify. A header, filter or + /// filter-header reply is at most tens of KB and arrives in well under a + /// second on any usable link, so a slow one really is a slow peer. A block + /// body is megabytes and the peer sends nothing until the transfer finishes, + /// so its elapsed time measures the payload rather than the peer: scoring it + /// would both punish an honest peer on a slow link and, because response + /// times feed one shared routing metric, push that peer out of serving + /// filters and headers it was answering perfectly well. + /// + /// Blocks are still tracked, because an unanswered block request is exactly + /// what should stop us handing that peer the next one. The judgement just + /// stays at routing: the peer is skipped while it owes us, never scored. + fn response_time_is_fair(self) -> bool { + !matches!(self, RequestKind::Blocks) + } +} + +/// The kind of timer an outbound message arms, or `None` if it is not timed. +/// +/// A `getdata` only arms the block timer when it actually asks for blocks. The +/// mempool sends `getdata` for transactions, which a `block` response would never +/// clear, so timing those would leave a timer armed forever against a peer that +/// answered everything it was asked. +/// +/// Masternode-diff and quorum-info requests stay untimed: they are issued as a +/// single burst against one peer rather than as a rotation, so knowing the peer +/// owes us one changes nothing about where the next one goes. +fn tracked_request_kind(msg: &NetworkMessage) -> Option { + match msg { + NetworkMessage::GetHeaders(_) | NetworkMessage::GetHeaders2(_) => { + Some(RequestKind::Headers) + } + NetworkMessage::GetCFHeaders(_) => Some(RequestKind::FilterHeaders), + NetworkMessage::GetCFilters(_) => Some(RequestKind::Filters), + NetworkMessage::GetData(inv) => inv + .iter() + .any(|item| matches!(item, Inventory::Block(_))) + .then_some(RequestKind::Blocks), + _ => None, + } +} + +/// The kind of timer an inbound message clears, or `None` if it answers nothing we +/// timed. Unsolicited gossip (inv, tx, addr, ping) is excluded: it arrives +/// unprompted and would clear a timer the peer has not actually answered. +fn timed_response_kind(msg: &NetworkMessage) -> Option { + match msg { + NetworkMessage::Headers(_) | NetworkMessage::Headers2(_) => Some(RequestKind::Headers), + NetworkMessage::CFHeaders(_) => Some(RequestKind::FilterHeaders), + NetworkMessage::CFilter(_) => Some(RequestKind::Filters), + NetworkMessage::Block(_) => Some(RequestKind::Blocks), + _ => None, + } +} + #[cfg(test)] impl PeerNetworkManager { pub(crate) async fn new_for_test(required_services: ServiceFlags) -> Self { @@ -1518,7 +2064,8 @@ impl PeerNetworkManager { let discovery = DnsDiscovery::new(); let (request_tx, request_rx) = unbounded_channel(); Self { - pool: Arc::new(PeerPool::new()), + pool: Arc::new(PeerPool::new(8)), + max_peers: 8, discovery: Arc::new(discovery), addrv2_handler: Arc::new(AddrV2Handler::new()), peer_store: Arc::new(peer_store), @@ -1534,11 +2081,18 @@ impl PeerNetworkManager { capability_rejected: Arc::new(RwLock::new(HashMap::new())), connected_peer_count: Arc::new(AtomicUsize::new(0)), headers2_disabled: Arc::new(Mutex::new(HashSet::new())), + headers2_decompression_semaphore: Arc::new(Semaphore::new( + headers2_decompression_parallelism(), + )), message_dispatcher: Arc::new(Mutex::new(MessageDispatcher::default())), request_tx, request_rx: Arc::new(Mutex::new(Some(request_rx))), round_robin_counter: Arc::new(AtomicUsize::new(0)), network_event_sender: broadcast::Sender::new(DEFAULT_NETWORK_EVENT_CAPACITY), + outstanding_requests: Arc::new(Mutex::new(HashMap::new())), + latency: Arc::new(Mutex::new(PeerLatency::default())), + last_maintenance_at: Arc::new(Mutex::new(Instant::now())), + last_eviction_at: Arc::new(Mutex::new(None)), } } @@ -1575,4 +2129,186 @@ impl PeerNetworkManager { pub(crate) async fn test_should_reject_after_handshake(&self, peer: &Peer) -> bool { Self::should_reject_after_handshake(&self.pool, peer, self.required_services).await } + + pub(crate) async fn test_update_reputation(&self, addr: SocketAddr, reason: ChangeReason) { + self.reputation_manager.update_reputation(addr, reason).await; + } + + pub(crate) async fn test_add_known_address(&self, addr: SocketAddr) { + self.addrv2_handler.add_known_address(addr, ServiceFlags::NETWORK).await; + } + + pub(crate) async fn test_next_peer(&self) -> SocketAddr { + let peers = self.pool.get_all_peers().await; + self.next_peer(&peers).await.0 + } + + /// Pick the peer `send_distributed` would route `msg` to, including the skip + /// of peers that still owe a response of that kind. + pub(crate) async fn test_route(&self, msg: &NetworkMessage) -> SocketAddr { + let peers = self.pool.get_all_peers().await; + let peers = match tracked_request_kind(msg) { + Some(kind) => self.without_peers_owing(kind, peers).await, + None => peers, + }; + self.next_peer(&peers).await.0 + } + + pub(crate) async fn test_record_latency(&self, addr: SocketAddr, elapsed: Duration) { + self.latency.lock().await.record(addr, elapsed); + } + + /// Arm a stall timer exactly as a successful send does. + pub(crate) async fn test_arm_request(&self, addr: SocketAddr, msg: &NetworkMessage) { + if let Some(kind) = tracked_request_kind(msg) { + self.outstanding_requests.lock().await.entry((addr, kind)).or_insert_with(Instant::now); + } + } + + /// Clear a stall timer exactly as the peer reader loop does on a response. + pub(crate) async fn test_deliver_response(&self, addr: SocketAddr, msg: &NetworkMessage) { + if let Some(kind) = timed_response_kind(msg) { + let sent = self.outstanding_requests.lock().await.remove(&(addr, kind)); + if let Some(sent) = sent { + self.latency.lock().await.record(addr, sent.elapsed()); + } + } + } + + pub(crate) async fn test_sweep_stalled_peers(&self) -> bool { + self.sweep_stalled_peers().await + } + + pub(crate) async fn test_score(&self, addr: SocketAddr) -> i32 { + self.reputation_manager.scores_for([addr]).await.get(&addr).copied().unwrap_or(0) + } + + pub(crate) async fn test_evict_worst_stuck_peer(&self) { + self.evict_worst_stuck_peer().await; + } +} + +#[cfg(test)] +mod header_ordering_tests { + use super::*; + use crate::test_utils::test_socket_address; + + fn context( + session_token: CancellationToken, + ) -> (HeaderDispatchContext, UnboundedReceiver) { + let mut dispatcher = MessageDispatcher::default(); + let receiver = dispatcher.message_receiver(&[MessageType::Headers]); + ( + HeaderDispatchContext { + addr: test_socket_address(1), + session_token, + ordered_results: Arc::new(Mutex::new(OrderedHeaderResults::default())), + message_dispatcher: Arc::new(Mutex::new(dispatcher)), + headers2_disabled: Arc::new(Mutex::new(HashSet::new())), + reputation_manager: Arc::new(PeerReputationManager::new()), + }, + receiver, + ) + } + + #[test] + fn completed_decompressions_are_released_in_receive_order() { + let mut results = OrderedHeaderResults::default(); + + assert!(results.complete(2, "third").is_empty()); + assert_eq!(results.complete(0, "first"), ["first"]); + assert_eq!(results.complete(1, "second"), ["second", "third"]); + } + + #[tokio::test] + async fn compressed_and_regular_headers_are_dispatched_in_wire_order() { + let (context, mut receiver) = context(CancellationToken::new()); + let semaphore = Arc::new(Semaphore::new(2)); + let first = Header::dummy(1); + let first_hash = first.block_hash(); + let second = Header::dummy(2); + + context + .complete( + 1, + HeaderDispatchOutcome::Regular(vec![second]), + semaphore.clone().acquire_owned().await.unwrap(), + ) + .await; + assert!(receiver.try_recv().is_err()); + + context + .complete( + 0, + HeaderDispatchOutcome::Compressed(vec![(first, first_hash)]), + semaphore.clone().acquire_owned().await.unwrap(), + ) + .await; + + let first_message = receiver.recv().await.unwrap(); + assert_eq!(first_message.header_hashes(), Some([first_hash].as_slice())); + assert_eq!(first_message.inner(), &NetworkMessage::Headers(vec![first])); + let second_message = receiver.recv().await.unwrap(); + assert_eq!(second_message.header_hashes(), None); + assert_eq!(second_message.inner(), &NetworkMessage::Headers(vec![second])); + assert_eq!(semaphore.available_permits(), 2); + } + + #[tokio::test] + async fn ended_peer_session_discards_headers_and_releases_permit() { + let session_token = CancellationToken::new(); + let (context, mut receiver) = context(session_token.clone()); + let semaphore = Arc::new(Semaphore::new(1)); + let permit = semaphore.clone().acquire_owned().await.unwrap(); + session_token.cancel(); + + context.complete(0, HeaderDispatchOutcome::Regular(vec![Header::dummy(1)]), permit).await; + + assert!(receiver.try_recv().is_err()); + assert_eq!(semaphore.available_permits(), 1); + } + + #[tokio::test] + async fn failed_decompression_does_not_block_later_headers() { + let (context, mut receiver) = context(CancellationToken::new()); + let semaphore = Arc::new(Semaphore::new(2)); + let header = Header::dummy(1); + + context + .complete( + 1, + HeaderDispatchOutcome::Regular(vec![header]), + semaphore.clone().acquire_owned().await.unwrap(), + ) + .await; + context + .complete( + 0, + HeaderDispatchOutcome::TaskFailed("cancelled".to_owned()), + semaphore.clone().acquire_owned().await.unwrap(), + ) + .await; + + assert_eq!(receiver.recv().await.unwrap().inner(), &NetworkMessage::Headers(vec![header])); + assert_eq!(semaphore.available_permits(), 2); + } + + #[tokio::test] + async fn invalid_decompression_disables_headers2_and_penalizes_peer() { + let (context, mut receiver) = context(CancellationToken::new()); + let semaphore = Arc::new(Semaphore::new(1)); + + context + .complete( + 0, + HeaderDispatchOutcome::Invalid("invalid compressed header".to_owned()), + semaphore.clone().acquire_owned().await.unwrap(), + ) + .await; + + assert!(receiver.try_recv().is_err()); + assert!(context.headers2_disabled.lock().await.contains(&context.addr)); + assert!(context.reputation_manager.scores_for([context.addr]).await[&context.addr] > 0); + assert_eq!(semaphore.available_permits(), 1); + } } diff --git a/dash-spv/src/network/message_dispatcher.rs b/dash-spv/src/network/message_dispatcher.rs index d88540d7d..10c3de4f6 100644 --- a/dash-spv/src/network/message_dispatcher.rs +++ b/dash-spv/src/network/message_dispatcher.rs @@ -10,6 +10,7 @@ use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; use dashcore::network::message::NetworkMessage; +use dashcore::BlockHash; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use crate::network::MessageType; @@ -19,6 +20,7 @@ use crate::network::MessageType; pub struct Message { peer_address: SocketAddr, inner: NetworkMessage, + header_hashes: Option>, } impl Message { @@ -27,6 +29,20 @@ impl Message { Self { peer_address, inner, + header_hashes: None, + } + } + + /// Creates a headers message carrying hashes already computed during decompression. + pub(crate) fn new_headers( + peer_address: SocketAddr, + headers_with_hashes: Vec<(dashcore::Header, BlockHash)>, + ) -> Self { + let (headers, hashes) = headers_with_hashes.into_iter().unzip(); + Self { + peer_address, + inner: NetworkMessage::Headers(headers), + header_hashes: Some(hashes), } } @@ -44,6 +60,16 @@ impl Message { pub fn inner(&self) -> &NetworkMessage { &self.inner } + + /// Consumes the wrapper and returns its network message. + pub(crate) fn into_inner(self) -> NetworkMessage { + self.inner + } + + /// Returns cached hashes for a decompressed headers2 message. + pub(crate) fn header_hashes(&self) -> Option<&[BlockHash]> { + self.header_hashes.as_deref() + } } /// Routes incoming network messages to subscribers based on message type. @@ -96,6 +122,17 @@ mod tests { assert_eq!(*msg.inner(), inner); } + #[test] + fn test_headers_message_carries_precomputed_hashes() { + let peer_address = test_socket_address(1); + let header = dashcore::Header::dummy(1); + let hash = header.block_hash(); + let msg = Message::new_headers(peer_address, vec![(header, hash)]); + + assert_eq!(msg.header_hashes(), Some([hash].as_slice())); + assert_eq!(msg.inner(), &NetworkMessage::Headers(vec![header])); + } + #[tokio::test] async fn test_dispatch_to_interested_receiver() { let mut message_dispatcher = MessageDispatcher::default(); diff --git a/dash-spv/src/network/mod.rs b/dash-spv/src/network/mod.rs index a428ddde1..b02adb4cd 100644 --- a/dash-spv/src/network/mod.rs +++ b/dash-spv/src/network/mod.rs @@ -5,11 +5,12 @@ pub mod constants; pub mod discovery; mod event; pub mod handshake; +mod latency; pub mod manager; mod message_dispatcher; pub mod peer; pub mod pool; -pub mod reputation; +mod reputation; mod message_type; #[cfg(test)] @@ -35,6 +36,7 @@ pub use manager::PeerNetworkManager; pub use message_dispatcher::{Message, MessageDispatcher}; pub use message_type::MessageType; pub use peer::Peer; +pub(crate) use reputation::PeerReputation; use std::net::SocketAddr; use tokio::sync::mpsc::UnboundedReceiver; @@ -90,6 +92,15 @@ impl RequestSender { .map_err(|e| NetworkError::ProtocolError(e.to_string())) } + /// Send a transaction to a specific peer. + pub(crate) fn send_transaction( + &self, + tx: dashcore::Transaction, + peer_address: SocketAddr, + ) -> NetworkResult<()> { + self.send_message_to_peer(NetworkMessage::Tx(tx), peer_address) + } + /// Request inventory from a specific peer. pub fn request_inventory( &self, diff --git a/dash-spv/src/network/peer.rs b/dash-spv/src/network/peer.rs index 2966ac52d..48e2f631c 100644 --- a/dash-spv/src/network/peer.rs +++ b/dash-spv/src/network/peer.rs @@ -351,7 +351,7 @@ impl Peer { tracing::warn!("Failed to flush socket {}: {}", self.address, e); } self.bytes_sent += serialized.len() as u64; - tracing::debug!("Sent message to {}: {:?}", self.address, raw_message.payload); + tracing::trace!("Sent message to {}: {:?}", self.address, raw_message.payload); Ok(()) } Err(e) => { @@ -681,7 +681,7 @@ impl Peer { let pong_message = NetworkMessage::Pong(nonce); self.send_message(pong_message).await?; - tracing::debug!("Responded to ping from {} with pong nonce {}", self.address, nonce); + tracing::trace!("Responded to ping from {} with pong nonce {}", self.address, nonce); Ok(()) } diff --git a/dash-spv/src/network/pool.rs b/dash-spv/src/network/pool.rs index e1982bf46..1a2e68b21 100644 --- a/dash-spv/src/network/pool.rs +++ b/dash-spv/src/network/pool.rs @@ -1,7 +1,6 @@ //! Peer pool for managing multiple peer connections use crate::error::{NetworkError, SpvError as Error}; -use crate::network::constants::TARGET_PEERS; use crate::network::peer::Peer; use dashcore::network::constants::ServiceFlags; use dashcore::prelude::CoreBlockHeight; @@ -16,14 +15,21 @@ pub struct PeerPool { peers: Arc>>>>, /// Addresses currently being connected to connecting: Arc>>, + /// Maximum number of simultaneous peer connections (from `ClientConfig::max_peers`). + max_peers: usize, } impl PeerPool { - /// Create a new peer pool - pub fn new() -> Self { + /// Create a new peer pool with a connection cap. + pub fn new(max_peers: usize) -> Self { + // Assert peers are greater than 0. We may change this + // so 0 means 'connect to as many peers as you can' + debug_assert!(max_peers > 0, "max_peers must be greater than 0 for the spv client to sync"); + Self { peers: Arc::new(RwLock::new(HashMap::new())), connecting: Arc::new(RwLock::new(HashSet::new())), + max_peers, } } @@ -42,10 +48,10 @@ impl PeerPool { connecting.remove(&addr); // Check if we're at capacity - if peers.len() >= TARGET_PEERS { + if peers.len() >= self.max_peers { return Err(Error::Network(NetworkError::ConnectionFailed(format!( "Maximum peers ({}) reached", - TARGET_PEERS + self.max_peers )))); } @@ -192,12 +198,12 @@ impl PeerPool { /// Check if we need more peers pub async fn needs_more_peers(&self) -> bool { - self.peer_count().await < TARGET_PEERS + self.peer_count().await < self.max_peers } /// Check if we can accept more peers pub async fn can_accept_peers(&self) -> bool { - self.peer_count().await < TARGET_PEERS + self.peer_count().await < self.max_peers } /// Remove unhealthy peers and return their addresses so the caller can @@ -230,7 +236,7 @@ impl PeerPool { impl Default for PeerPool { fn default() -> Self { - Self::new() + Self::new(8) } } @@ -249,7 +255,7 @@ mod tests { #[tokio::test] async fn test_peer_pool_basic() { - let pool = PeerPool::new(); + let pool = PeerPool::new(8); // Initial state assert_eq!(pool.peer_count().await, 0); @@ -265,7 +271,7 @@ mod tests { #[tokio::test] async fn test_service_lookup() { - let pool = PeerPool::new(); + let pool = PeerPool::new(8); let compact_filters = ServiceFlags::COMPACT_FILTERS; let combined = compact_filters | ServiceFlags::NODE_HEADERS_COMPRESSED; diff --git a/dash-spv/src/network/reputation.rs b/dash-spv/src/network/reputation.rs index 5c65ce32b..801f5f409 100644 --- a/dash-spv/src/network/reputation.rs +++ b/dash-spv/src/network/reputation.rs @@ -7,65 +7,57 @@ use crate::storage::PeerStorage; use dashcore::network::address::AddrV2Message; +use rand::seq::SliceRandom; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime}; use tokio::sync::RwLock; -/// Misbehavior score thresholds for different violations -pub mod misbehavior_scores { - /// Invalid message format or protocol violation - pub const INVALID_MESSAGE: i32 = 10; - - /// Invalid block header - pub const INVALID_HEADER: i32 = 50; - - /// Invalid compact filter - pub const INVALID_FILTER: i32 = 25; - - /// Timeout or slow response - pub const TIMEOUT: i32 = 5; - - /// Sending unsolicited data - pub const UNSOLICITED_DATA: i32 = 15; - - /// Invalid transaction - pub const INVALID_TRANSACTION: i32 = 20; - - /// Invalid masternode list diff - pub const INVALID_MASTERNODE_DIFF: i32 = 30; - - /// Invalid ChainLock - pub const INVALID_CHAINLOCK: i32 = 40; - - /// Invalid InstantLock - pub const INVALID_INSTANTLOCK: i32 = 35; - - /// Duplicate message - pub const DUPLICATE_MESSAGE: i32 = 5; - - /// Connection flood attempt - pub const CONNECTION_FLOOD: i32 = 20; +/// Reason a peer's misbehavior score rose. Each reason owns its penalty and a +/// human-readable label. There is deliberately no reason that lowers the score: +/// only decay does that, so the score cannot be earned down by a peer that is +/// merely busy. Useful behavior is measured by [`super::latency`] instead. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChangeReason { + HandshakeFailed, + ConnectionFailed, + Headers2DecompressionFailed, + RequestTimeout, + PingFailed, + InvalidTransactionInBlock, + ManuallyBanned, } -/// Positive behavior scores -pub mod positive_scores { - /// Successfully provided valid headers - pub const VALID_HEADERS: i32 = -5; - - /// Successfully provided valid filters - pub const VALID_FILTERS: i32 = -3; - - /// Successfully provided valid block - pub const VALID_BLOCK: i32 = -10; - - /// Fast response time - pub const FAST_RESPONSE: i32 = -2; +impl ChangeReason { + /// Penalty this reason adds to a peer's misbehavior score. + pub fn score(&self) -> i32 { + match self { + ChangeReason::HandshakeFailed => 10, + ChangeReason::ConnectionFailed => 2, + ChangeReason::Headers2DecompressionFailed => 10, + ChangeReason::RequestTimeout => 10, + ChangeReason::PingFailed => 5, + ChangeReason::InvalidTransactionInBlock => 20, + ChangeReason::ManuallyBanned => 100, + } + } +} - /// Long uptime connection - pub const LONG_UPTIME: i32 = -5; +impl std::fmt::Display for ChangeReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let label = match self { + ChangeReason::HandshakeFailed => "Handshake failed", + ChangeReason::ConnectionFailed => "Connection failed", + ChangeReason::Headers2DecompressionFailed => "Headers2 decompression failed", + ChangeReason::RequestTimeout => "Request timed out", + ChangeReason::PingFailed => "Ping failed", + ChangeReason::InvalidTransactionInBlock => "Invalid transaction type in block", + ChangeReason::ManuallyBanned => "Manually banned", + }; + f.write_str(label) + } } /// Ban duration for misbehaving peers @@ -80,13 +72,30 @@ const DECAY_AMOUNT: i32 = 5; /// Maximum misbehavior score before a peer is banned const MAX_MISBEHAVIOR_SCORE: i32 = 100; -/// Minimum score (most positive reputation) -const MIN_MISBEHAVIOR_SCORE: i32 = -50; +/// Minimum score. Zero, because this score only measures misbehavior and "none +/// observed" is as clean as a peer can be. Letting it run negative would bank +/// credit that a peer could later spend on misbehaving: decay alone would carry a +/// long-lived peer far below zero, and it would then need several extra strikes to +/// reach eviction, making the peers most likely to be quietly failing the hardest +/// ones to remove. How useful a peer actually is, as opposed to how badly it is +/// behaving, is measured by [`super::latency`] instead. +const MIN_MISBEHAVIOR_SCORE: i32 = 0; + +/// Highest score a peer may load with after a restart. A persisted near-ban +/// score can then never ban a peer on its first post-restart penalty, which +/// would otherwise let stale suspicion lock the client out of known-good peers. +const RESTART_SCORE_CEILING: i32 = 40; const MAX_BAN_COUNT: u32 = 1000; const MAX_ACTION_COUNT: u64 = 1_000_000; +/// Lowest score any earlier version could persist, back when answering a request +/// earned credit below zero. A stored score in that range is stale rather than +/// corrupt, so it is clamped quietly: warning would fire once per persisted peer +/// on the first run after an upgrade. +const LEGACY_MIN_SCORE: i32 = -50; + fn clamp_peer_score<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, @@ -94,7 +103,9 @@ where let mut v = i32::deserialize(deserializer)?; if v < MIN_MISBEHAVIOR_SCORE { - tracing::warn!("Peer has invalid score {v}, clamping to min {MIN_MISBEHAVIOR_SCORE}"); + if v < LEGACY_MIN_SCORE { + tracing::warn!("Peer has invalid score {v}, clamping to min {MIN_MISBEHAVIOR_SCORE}"); + } v = MIN_MISBEHAVIOR_SCORE } else if v > MAX_MISBEHAVIOR_SCORE { tracing::warn!("Peer has invalid score {v}, clamping to max {MAX_MISBEHAVIOR_SCORE}"); @@ -164,6 +175,12 @@ pub struct PeerReputation { /// Last connection time #[serde(skip)] pub last_connection: Option, + + /// Wall-clock time this reputation was last persisted. Used to credit decay + /// for time the client spent offline, since the monotonic `last_update` + /// resets on load and cannot measure downtime. + #[serde(default = "SystemTime::now")] + pub last_seen: SystemTime, } impl Default for PeerReputation { @@ -178,6 +195,7 @@ impl Default for PeerReputation { connection_attempts: 0, successful_connections: 0, last_connection: None, + last_seen: SystemTime::now(), } } } @@ -223,25 +241,10 @@ impl PeerReputation { } } -/// Reputation change event -#[derive(Debug, Clone)] -pub struct ReputationEvent { - pub peer: SocketAddr, - pub change: i32, - pub reason: String, - pub timestamp: Instant, -} - /// Peer reputation manager pub struct PeerReputationManager { /// Reputation data for each peer reputations: Arc>>, - - /// Recent reputation events for monitoring - recent_events: Arc>>, - - /// Maximum number of events to keep - max_events: usize, } impl Default for PeerReputationManager { @@ -255,18 +258,13 @@ impl PeerReputationManager { pub fn new() -> Self { Self { reputations: Arc::new(RwLock::new(HashMap::new())), - recent_events: Arc::new(RwLock::new(Vec::new())), - max_events: 1000, } } - /// Update peer reputation - pub async fn update_reputation( - &self, - peer: SocketAddr, - score_change: i32, - reason: &str, - ) -> bool { + /// Update peer reputation by the score delta of `reason`. + pub async fn update_reputation(&self, peer: SocketAddr, reason: ChangeReason) -> bool { + let score_change = reason.score(); + let mut reputations = self.reputations.write().await; let reputation = reputations.entry(peer).or_default(); @@ -311,32 +309,9 @@ impl PeerReputationManager { ); } - // Record event - let event = ReputationEvent { - peer, - change: score_change, - reason: reason.to_string(), - timestamp: Instant::now(), - }; - - drop(reputations); // Release lock before recording event - self.record_event(event).await; - should_ban } - /// Record a reputation event - async fn record_event(&self, event: ReputationEvent) { - let mut events = self.recent_events.write().await; - events.push(event); - - // Keep only recent events - if events.len() > self.max_events { - let drain_count = events.len() - self.max_events; - events.drain(0..drain_count); - } - } - /// Check if a peer is banned pub async fn is_banned(&self, peer: &SocketAddr) -> bool { let mut reputations = self.reputations.write().await; @@ -348,35 +323,6 @@ impl PeerReputationManager { } } - /// Get peer reputation score - pub async fn get_score(&self, peer: &SocketAddr) -> i32 { - let mut reputations = self.reputations.write().await; - if let Some(reputation) = reputations.get_mut(peer) { - reputation.apply_decay(); - reputation.score - } else { - 0 - } - } - - /// Temporarily ban a peer for a specified duration, regardless of score. - /// This can be used for critical protocol violations (e.g., invalid ChainLocks). - pub async fn temporary_ban_peer(&self, peer: SocketAddr, duration: Duration, reason: &str) { - let mut reputations = self.reputations.write().await; - let reputation = reputations.entry(peer).or_default(); - - reputation.banned_until = Some(Instant::now() + duration); - reputation.ban_count += 1; - - tracing::warn!( - "Peer {} temporarily banned for {:?} (ban #{}, reason: {})", - peer, - duration, - reputation.ban_count, - reason - ); - } - /// Record a connection attempt pub async fn record_connection_attempt(&self, peer: SocketAddr) { let mut reputations = self.reputations.write().await; @@ -404,9 +350,20 @@ impl PeerReputationManager { reputations.clone() } - /// Get recent reputation events - pub async fn get_recent_events(&self) -> Vec { - self.recent_events.read().await.clone() + /// Return the current score for each requested peer. Peers with no record + /// default to 0. Takes a read lock and does not apply decay, so a caller sees + /// the score as of the last `update_reputation` or maintenance pass. Acquires + /// only the reputation lock, so callers must not hold a pool guard across this + /// to keep a single lock order. + pub async fn scores_for( + &self, + addrs: impl IntoIterator, + ) -> HashMap { + let reputations = self.reputations.read().await; + addrs + .into_iter() + .map(|addr| (addr, reputations.get(&addr).map_or(0, |rep| rep.score))) + .collect() } /// Clear banned status for a peer (admin function) @@ -419,36 +376,13 @@ impl PeerReputationManager { } } - /// Reset reputation for a peer - pub async fn reset_reputation(&self, peer: &SocketAddr) { - let mut reputations = self.reputations.write().await; - reputations.remove(peer); - tracing::info!("Reset reputation for peer {}", peer); - } - - /// Get peers sorted by reputation (best first) - pub async fn get_peers_by_reputation(&self) -> Vec<(SocketAddr, i32)> { - let mut reputations = self.reputations.write().await; - - // Apply decay and collect scores - let mut peer_scores: Vec<(SocketAddr, i32)> = reputations - .iter_mut() - .map(|(addr, rep)| { - rep.apply_decay(); - (*addr, rep.score) - }) - .filter(|(_, score)| *score < MAX_MISBEHAVIOR_SCORE) // Exclude banned peers - .collect(); - - // Sort by score (lower is better) - peer_scores.sort_by_key(|(_, score)| *score); - - peer_scores - } - /// Save reputation data to persistent storage pub async fn save_to_storage(&self, storage: &impl PeerStorage) -> std::io::Result<()> { - let reputations = self.reputations.read().await; + let now = SystemTime::now(); + let mut reputations = self.reputations.write().await; + for reputation in reputations.values_mut() { + reputation.last_seen = now; + } storage.save_peers_reputation(&reputations).await.map_err(std::io::Error::other) } @@ -475,11 +409,21 @@ impl PeerReputationManager { continue; } - // Apply initial decay based on ban count - if reputation.ban_count > 0 { - reputation.score = reputation.score.max(50); // Start with higher score for previously banned peers + // Credit decay for the wall-clock time the client spent offline. The + // monotonic `last_update` reset to now on load, so without this a peer + // never recovers while the client is closed. + if let Ok(offline) = SystemTime::now().duration_since(reputation.last_seen) { + let intervals = offline.as_secs() / DECAY_INTERVAL.as_secs(); + if intervals > 0 { + let intervals_i32 = intervals.min(i32::MAX as u64) as i32; + let decay = intervals_i32.saturating_mul(DECAY_AMOUNT); + reputation.score = (reputation.score - decay).max(MIN_MISBEHAVIOR_SCORE); + } } + // Never let a peer return one strike short of a ban after a restart. + reputation.score = reputation.score.min(RESTART_SCORE_CEILING); + reputations.insert(addr, reputation); loaded_count += 1; } @@ -532,6 +476,10 @@ impl ReputationAware for PeerReputationManager { } } + // Shuffle before the stable sort so equal-scored candidates (e.g. every + // peer at score 0 on a fresh install) are returned in random order + // rather than a fixed address order that herds clients onto the same nodes. + peer_scores.shuffle(&mut rand::thread_rng()); // Sort by score (lower is better) peer_scores.sort_by_key(|(_, score)| *score); diff --git a/dash-spv/src/network/reputation_tests.rs b/dash-spv/src/network/reputation_tests.rs index b8b140057..71546cab1 100644 --- a/dash-spv/src/network/reputation_tests.rs +++ b/dash-spv/src/network/reputation_tests.rs @@ -2,28 +2,29 @@ #[cfg(test)] mod tests { - use crate::storage::{PersistentPeerStorage, PersistentStorage}; + use crate::storage::{PeerStorage, PersistentPeerStorage, PersistentStorage}; use super::super::*; + use std::collections::HashMap; use std::net::SocketAddr; + use std::time::{Duration, SystemTime}; + + async fn score(manager: &PeerReputationManager, peer: &SocketAddr) -> i32 { + manager.get_all_reputations().await.get(peer).map_or(0, |rep| rep.score) + } #[tokio::test] async fn test_basic_reputation_operations() { let manager = PeerReputationManager::new(); let peer: SocketAddr = "127.0.0.1:8333".parse().unwrap(); - // Initial score should be 0 - assert_eq!(manager.get_score(&peer).await, 0); + assert_eq!(score(&manager, &peer).await, 0); - // Test misbehavior - manager - .update_reputation(peer, misbehavior_scores::INVALID_MESSAGE, "Test invalid message") - .await; - assert_eq!(manager.get_score(&peer).await, 10); + manager.update_reputation(peer, ChangeReason::HandshakeFailed).await; + assert_eq!(score(&manager, &peer).await, 10); - // Test positive behavior - manager.update_reputation(peer, positive_scores::VALID_HEADERS, "Test valid headers").await; - assert_eq!(manager.get_score(&peer).await, 5); + manager.update_reputation(peer, ChangeReason::PingFailed).await; + assert_eq!(score(&manager, &peer).await, 15); } #[tokio::test] @@ -31,17 +32,9 @@ mod tests { let manager = PeerReputationManager::new(); let peer: SocketAddr = "192.168.1.1:8333".parse().unwrap(); - // Accumulate misbehavior + // Banned on the 10th violation (10 * 10 = 100). for i in 0..10 { - let banned = manager - .update_reputation( - peer, - misbehavior_scores::INVALID_MESSAGE, - &format!("Violation {}", i), - ) - .await; - - // Should be banned on the 10th violation (total score = 100) + let banned = manager.update_reputation(peer, ChangeReason::HandshakeFailed).await; if i == 9 { assert!(banned); } else { @@ -58,11 +51,9 @@ mod tests { let peer1: SocketAddr = "10.0.0.1:8333".parse().unwrap(); let peer2: SocketAddr = "10.0.0.2:8333".parse().unwrap(); - // Set reputations - manager.update_reputation(peer1, -10, "Good peer").await; - manager.update_reputation(peer2, 50, "Bad peer").await; + manager.update_reputation(peer1, ChangeReason::PingFailed).await; + manager.update_reputation(peer2, ChangeReason::InvalidTransactionInBlock).await; - // Save and load let temp_dir = tempfile::TempDir::new().unwrap(); let peer_storage = PersistentPeerStorage::open(temp_dir.path()) .await @@ -72,31 +63,34 @@ mod tests { let new_manager = PeerReputationManager::new(); new_manager.load_from_storage(&peer_storage).await.unwrap(); - // Verify scores were preserved - assert_eq!(new_manager.get_score(&peer1).await, -10); - assert_eq!(new_manager.get_score(&peer2).await, 50); + assert_eq!(score(&new_manager, &peer1).await, 5); + assert_eq!(score(&new_manager, &peer2).await, 20); } #[tokio::test] async fn test_peer_selection() { let manager = PeerReputationManager::new(); - let good_peer = AddrV2Message::dummy(0, "1.1.1.1".parse().unwrap(), 8333); - let neutral_peer = AddrV2Message::dummy(0, "2.2.2.2".parse().unwrap(), 8333); + let clean_peer = AddrV2Message::dummy(0, "1.1.1.1".parse().unwrap(), 8333); + let other_clean_peer = AddrV2Message::dummy(0, "2.2.2.2".parse().unwrap(), 8333); let bad_peer = AddrV2Message::dummy(0, "3.3.3.3".parse().unwrap(), 8333); - // Set different reputations - manager.update_reputation(good_peer.socket_addr().unwrap(), -20, "Very good").await; - manager.update_reputation(bad_peer.socket_addr().unwrap(), 80, "Very bad").await; - // neutral_peer has default score of 0 + manager + .update_reputation( + bad_peer.socket_addr().unwrap(), + ChangeReason::InvalidTransactionInBlock, + ) + .await; - let all_peers = vec![good_peer.clone(), neutral_peer.clone(), bad_peer.clone()]; + let all_peers = vec![clean_peer.clone(), other_clean_peer.clone(), bad_peer.clone()]; let selected = manager.select_best_peers(all_peers, 2).await; - // Should select good_peer first, then neutral_peer + // The two clean peers are indistinguishable, and deliberately so: the score + // only records misbehavior, and equal scores are returned shuffled so + // clients do not all herd onto the same node. Only their exclusion of the + // misbehaving peer is guaranteed, not their order. assert_eq!(selected.len(), 2); - assert_eq!(selected[0], good_peer.socket_addr().unwrap()); - assert_eq!(selected[1], neutral_peer.socket_addr().unwrap()); + assert!(!selected.contains(&bad_peer.socket_addr().unwrap())); } #[tokio::test] @@ -115,4 +109,72 @@ mod tests { assert_eq!(rep.connection_attempts, 2); assert_eq!(rep.successful_connections, 1); } + + #[tokio::test] + async fn test_request_timeout_penalty_and_ban() { + let manager = PeerReputationManager::new(); + let peer: SocketAddr = "127.0.0.1:7777".parse().unwrap(); + + // Ten stalls (10 * 10) reach the ban line; only the tenth bans. + for i in 0..10 { + let banned = manager.update_reputation(peer, ChangeReason::RequestTimeout).await; + assert_eq!(banned, i == 9); + } + assert_eq!(score(&manager, &peer).await, 100); + assert!(manager.is_banned(&peer).await); + } + + #[tokio::test] + async fn test_scores_for_returns_scores_and_defaults() { + let manager = PeerReputationManager::new(); + let slight: SocketAddr = "127.0.0.1:1".parse().unwrap(); + let bad: SocketAddr = "127.0.0.1:2".parse().unwrap(); + let unknown: SocketAddr = "127.0.0.1:3".parse().unwrap(); + + manager.update_reputation(slight, ChangeReason::PingFailed).await; + manager.update_reputation(bad, ChangeReason::InvalidTransactionInBlock).await; + + let scores = manager.scores_for([slight, bad, unknown]).await; + assert_eq!(scores.get(&slight), Some(&5)); + assert_eq!(scores.get(&bad), Some(&20)); + assert_eq!(scores.get(&unknown), Some(&0)); + } + + #[tokio::test] + async fn test_load_clamps_score_and_credits_offline_decay() { + let temp_dir = tempfile::TempDir::new().unwrap(); + let storage = PersistentPeerStorage::open(temp_dir.path()).await.unwrap(); + + // Persisted near the ban line but seen recently: clamps to the restart ceiling. + let near_ban: SocketAddr = "10.0.0.1:8333".parse().unwrap(); + // Persisted mid-score but offline ~2 hours: decays by two intervals on load. + let offline: SocketAddr = "10.0.0.2:8333".parse().unwrap(); + + let mut map: HashMap = HashMap::new(); + map.insert( + near_ban, + PeerReputation { + score: 95, + last_seen: SystemTime::now(), + ..Default::default() + }, + ); + map.insert( + offline, + PeerReputation { + score: 30, + last_seen: SystemTime::now() - Duration::from_secs(2 * 60 * 60 + 60), + ..Default::default() + }, + ); + storage.save_peers_reputation(&map).await.unwrap(); + + let manager = PeerReputationManager::new(); + manager.load_from_storage(&storage).await.unwrap(); + + // Clamped to RESTART_SCORE_CEILING, so one post-restart penalty cannot ban it. + assert_eq!(score(&manager, &near_ban).await, 40); + // 30 - 2 intervals * 5 decay. + assert_eq!(score(&manager, &offline).await, 20); + } } diff --git a/dash-spv/src/network/tests.rs b/dash-spv/src/network/tests.rs index 0aa9f5390..f6153bd75 100644 --- a/dash-spv/src/network/tests.rs +++ b/dash-spv/src/network/tests.rs @@ -29,12 +29,12 @@ mod pool_tests { #[tokio::test] async fn test_pool_limits() { - let pool = PeerPool::new(); + let pool = PeerPool::new(8); // Test needs_more_peers logic assert!(pool.needs_more_peers().await); - // Can accept up to TARGET_PEERS + // Can accept up to 8 peers assert!(pool.can_accept_peers().await); // Test peer count @@ -117,3 +117,362 @@ mod pool_tests { assert_eq!(manager.test_capability_rejected_count().await, 1); } } + +#[cfg(test)] +mod selection_tests { + use crate::network::manager::{ + PeerNetworkManager, REQUEST_OWED_TIMEOUT, REQUEST_STALL_TIMEOUT, + }; + use crate::network::reputation::ChangeReason; + use crate::test_utils::test_socket_address; + use dashcore::blockdata::block::{Block, Header, Version}; + use dashcore::network::constants::ServiceFlags; + use dashcore::network::message::NetworkMessage; + use dashcore::network::message_blockdata::Inventory; + use dashcore::network::message_filter::{CFilter, GetCFHeaders, GetCFilters}; + use dashcore::{BlockHash, CompactTarget, TxMerkleNode, Txid}; + use dashcore_hashes::Hash; + use std::collections::HashMap; + use std::net::SocketAddr; + use std::time::Duration; + + fn get_filter_headers() -> NetworkMessage { + NetworkMessage::GetCFHeaders(GetCFHeaders { + filter_type: 0, + start_height: 0, + stop_hash: BlockHash::all_zeros(), + }) + } + + fn get_filters() -> NetworkMessage { + NetworkMessage::GetCFilters(GetCFilters { + filter_type: 0, + start_height: 0, + stop_hash: BlockHash::all_zeros(), + }) + } + + fn filter_response() -> NetworkMessage { + NetworkMessage::CFilter(CFilter { + filter_type: 0, + block_hash: BlockHash::all_zeros(), + filter: vec![], + }) + } + + fn get_blocks() -> NetworkMessage { + NetworkMessage::GetData(vec![Inventory::Block(BlockHash::all_zeros())]) + } + + fn get_transactions() -> NetworkMessage { + NetworkMessage::GetData(vec![Inventory::Transaction(Txid::all_zeros())]) + } + + fn block_response() -> NetworkMessage { + NetworkMessage::Block(Block { + header: Header { + version: Version::ONE, + prev_blockhash: BlockHash::all_zeros(), + merkle_root: TxMerkleNode::all_zeros(), + time: 0, + bits: CompactTarget::from_consensus(0), + nonce: 0, + }, + txdata: vec![], + }) + } + + async fn full_pool_with_bad_peer(bad: u8) -> (PeerNetworkManager, std::net::SocketAddr) { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + for i in 1u8..=8 { + manager.insert_test_peer(test_socket_address(i), cf).await; + } + let bad_addr = test_socket_address(bad); + // Two consecutive stalls reach the eviction threshold. + manager.test_update_reputation(bad_addr, ChangeReason::RequestTimeout).await; + manager.test_update_reputation(bad_addr, ChangeReason::RequestTimeout).await; + (manager, bad_addr) + } + + /// A peer that is fast at one request kind must not be able to hide being slow + /// at another. + /// + /// Seen on mainnet: a peer averaging 11.9s on filter headers, with a 32s p90, + /// measured as 52ms and took 40% of the traffic with zero stall penalties. Its + /// quick `cfilter` responses kept clearing the timer its slow `getcfheaders` + /// had armed, so the slow response found nothing to clear and was never + /// recorded, and the sweep never saw an aged entry. Only 255 of 1050 filter + /// header timers were cleared by an actual `cfheaders` response. + #[tokio::test(start_paused = true)] + async fn test_fast_response_does_not_clear_another_kinds_timer() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + let peer = test_socket_address(1); + manager.insert_test_peer(peer, cf).await; + + // The peer owes us both a filter-header and a filter response. + manager.test_arm_request(peer, &get_filter_headers()).await; + manager.test_arm_request(peer, &get_filters()).await; + + // It answers the filter promptly. That must not count as answering the + // filter headers, which it is still sitting on. + tokio::time::advance(Duration::from_millis(50)).await; + manager.test_deliver_response(peer, &filter_response()).await; + + // Past the stall timeout with the filter headers still unanswered. + tokio::time::advance(REQUEST_STALL_TIMEOUT + Duration::from_secs(1)).await; + manager.test_sweep_stalled_peers().await; + + assert_eq!( + manager.test_score(peer).await, + ChangeReason::RequestTimeout.score(), + "the unanswered filter-header request must still be caught as a stall" + ); + } + + /// A peer stalling on several request kinds at once is one failing peer, not + /// several, so a single sweep must not stack strikes and evict it outright. + #[tokio::test(start_paused = true)] + async fn test_stalling_on_two_kinds_is_one_strike_per_sweep() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + let peer = test_socket_address(1); + manager.insert_test_peer(peer, cf).await; + + manager.test_arm_request(peer, &get_filter_headers()).await; + manager.test_arm_request(peer, &get_filters()).await; + + tokio::time::advance(REQUEST_STALL_TIMEOUT + Duration::from_secs(1)).await; + manager.test_sweep_stalled_peers().await; + + assert_eq!( + manager.test_score(peer).await, + ChangeReason::RequestTimeout.score(), + "two stalled kinds are still one failing peer, so one strike" + ); + } + + #[tokio::test] + async fn test_next_peer_excludes_slow_peer() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + let good1 = test_socket_address(1); + let good2 = test_socket_address(2); + let slow = test_socket_address(3); + for addr in [good1, good2, slow] { + manager.insert_test_peer(addr, cf).await; + } + + manager.test_record_latency(good1, Duration::from_millis(40)).await; + manager.test_record_latency(good2, Duration::from_millis(60)).await; + manager.test_record_latency(slow, Duration::from_secs(10)).await; + + let mut seen_good1 = false; + let mut seen_good2 = false; + for _ in 0..20 { + let picked = manager.test_next_peer().await; + assert_ne!(picked, slow, "a peer that has stopped answering must not be routed to"); + seen_good1 |= picked == good1; + seen_good2 |= picked == good2; + } + assert!(seen_good1 && seen_good2, "load should spread across the responsive peers"); + } + + /// The failure this guards against was seen on mainnet: one peer answered a + /// few requests before the others had answered any, that head start was enough + /// to rank it alone at the top, and from then on it took every request while + /// the peers it starved got none and so could never earn their way back. Here + /// the same head start must not stop the pool sharing the work. + #[tokio::test] + async fn test_next_peer_keeps_spreading_after_one_peer_gets_a_head_start() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + let peers = [test_socket_address(1), test_socket_address(2), test_socket_address(3)]; + for addr in peers { + manager.insert_test_peer(addr, cf).await; + } + + // The head start: the first peer is measured, and fast, before the rest + // have answered anything at all. + for _ in 0..3 { + manager.test_record_latency(peers[0], Duration::from_millis(20)).await; + } + + let mut picks: HashMap = HashMap::new(); + for _ in 0..300 { + let picked = manager.test_next_peer().await; + *picks.entry(picked).or_default() += 1; + manager.test_record_latency(picked, Duration::from_millis(50)).await; + } + + for addr in peers { + assert_eq!( + picks.get(&addr).copied().unwrap_or(0), + 100, + "every peer should keep an equal share, got {picks:?}" + ); + } + } + + /// A peer sitting on a block request must not be handed the retry of that + /// same request. + /// + /// Seen on mainnet: one of three peers silently dropped every `getdata` it + /// received. Latency only ever records a response, so a peer answering + /// nothing was never measured and kept its turn in the rotation, which it + /// then used to win the retry of the request it had just dropped. Filter sync + /// commits behind the missing block, so the client spent 66% of a 16 minute + /// run frozen, in stalls of exactly one, two and four block timeouts. + #[tokio::test(start_paused = true)] + async fn test_peer_owing_a_block_is_skipped_until_it_answers() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + let silent = test_socket_address(1); + let good = test_socket_address(2); + for addr in [silent, good] { + manager.insert_test_peer(addr, cf).await; + } + + manager.test_arm_request(silent, &get_blocks()).await; + tokio::time::advance(REQUEST_OWED_TIMEOUT + Duration::from_secs(1)).await; + + for _ in 0..10 { + assert_eq!( + manager.test_route(&get_blocks()).await, + good, + "a peer already sitting on a block request must not be sent another" + ); + } + + // Answering clears the debt, and the peer is a candidate again. + manager.test_deliver_response(silent, &block_response()).await; + let mut picks: HashMap = HashMap::new(); + for _ in 0..10 { + *picks.entry(manager.test_route(&get_blocks()).await).or_default() += 1; + } + assert!(picks.contains_key(&silent), "a peer that answered must be routed to again"); + } + + /// The skip is a preference, not a rule: with every peer owing a response, + /// refusing to send is worse than sending to a busy peer. + #[tokio::test(start_paused = true)] + async fn test_routing_falls_back_when_every_peer_owes_a_block() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + let peers = [test_socket_address(1), test_socket_address(2)]; + for addr in peers { + manager.insert_test_peer(addr, cf).await; + manager.test_arm_request(addr, &get_blocks()).await; + } + tokio::time::advance(REQUEST_OWED_TIMEOUT + Duration::from_secs(1)).await; + + assert!(peers.contains(&manager.test_route(&get_blocks()).await)); + } + + /// Blocks are routed away from, never penalized for. A block body can be + /// megabytes and a peer sends nothing until the transfer completes, so a slow + /// link is indistinguishable from a peer ignoring us and must not cost + /// reputation or trigger eviction. + #[tokio::test(start_paused = true)] + async fn test_unanswered_block_request_does_not_penalize_the_peer() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + let peer = test_socket_address(1); + manager.insert_test_peer(peer, cf).await; + + manager.test_arm_request(peer, &get_blocks()).await; + tokio::time::advance(REQUEST_STALL_TIMEOUT + Duration::from_secs(1)).await; + manager.test_sweep_stalled_peers().await; + + assert_eq!( + manager.test_score(peer).await, + 0, + "a slow block transfer must not be scored as a stall" + ); + } + + /// The mempool asks for transactions with the same `getdata` message blocks + /// use, and a `block` response would never arrive to clear that timer. Timing + /// it would leave a peer permanently owing a response it was never asked for. + #[tokio::test(start_paused = true)] + async fn test_transaction_getdata_does_not_arm_the_block_timer() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + // Only one peer is sent the tx request, so a wrongly armed block timer + // shows up as that peer being skipped rather than being hidden by the + // all-peers-owing fallback. + let mempool_peer = test_socket_address(1); + let idle_peer = test_socket_address(2); + for addr in [mempool_peer, idle_peer] { + manager.insert_test_peer(addr, cf).await; + } + manager.test_arm_request(mempool_peer, &get_transactions()).await; + tokio::time::advance(REQUEST_OWED_TIMEOUT + Duration::from_secs(1)).await; + + let mut picks: HashMap = HashMap::new(); + for _ in 0..20 { + *picks.entry(manager.test_route(&get_blocks()).await).or_default() += 1; + } + assert!( + picks.contains_key(&mempool_peer), + "a pending tx request must not make a peer look like it owes a block, got {picks:?}" + ); + } + + #[tokio::test] + async fn test_evict_worst_stuck_peer_removes_the_stalling_peer() { + let (manager, bad) = full_pool_with_bad_peer(3).await; + // A replacement must be available or eviction is a no-op. + manager.test_add_known_address(test_socket_address(99)).await; + + manager.test_evict_worst_stuck_peer().await; + + assert!(!manager.test_is_connected(&bad).await, "stalling peer should be evicted"); + assert_eq!(manager.test_peer_count().await, 7); + } + + #[tokio::test] + async fn test_evict_worst_stuck_peer_skips_without_replacement() { + let (manager, _bad) = full_pool_with_bad_peer(3).await; + // No known addresses -> no replacement candidate -> no eviction. + manager.test_evict_worst_stuck_peer().await; + assert_eq!(manager.test_peer_count().await, 8); + } + + #[tokio::test] + async fn test_evict_worst_stuck_peer_skips_when_all_peers_bad() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + for i in 1u8..=8 { + let addr = test_socket_address(i); + manager.insert_test_peer(addr, cf).await; + manager.test_update_reputation(addr, ChangeReason::RequestTimeout).await; + manager.test_update_reputation(addr, ChangeReason::RequestTimeout).await; + } + manager.test_add_known_address(test_socket_address(99)).await; + + manager.test_evict_worst_stuck_peer().await; + + // Every peer is equally bad, so the problem is systemic: evict none. + assert_eq!(manager.test_peer_count().await, 8); + } + + #[tokio::test] + async fn test_evict_worst_stuck_peer_skips_when_pool_not_full() { + let cf = ServiceFlags::COMPACT_FILTERS; + let manager = PeerNetworkManager::new_for_test(cf).await; + for i in 1u8..=3 { + manager.insert_test_peer(test_socket_address(i), cf).await; + } + let bad = test_socket_address(2); + manager.test_update_reputation(bad, ChangeReason::RequestTimeout).await; + manager.test_update_reputation(bad, ChangeReason::RequestTimeout).await; + manager.test_add_known_address(test_socket_address(99)).await; + + manager.test_evict_worst_stuck_peer().await; + + // Pool is below max_peers, so eviction would be pure loss: skip. + assert_eq!(manager.test_peer_count().await, 3); + } +} diff --git a/dash-spv/src/storage/block_headers.rs b/dash-spv/src/storage/block_headers.rs index e7c07b36d..df3ac26c4 100644 --- a/dash-spv/src/storage/block_headers.rs +++ b/dash-spv/src/storage/block_headers.rs @@ -42,20 +42,9 @@ impl BlockHeaderTip { #[async_trait] pub trait BlockHeaderStorage: Send + Sync + 'static { - async fn store_headers(&mut self, headers: &[BlockHeader]) -> StorageResult<()>; + async fn store_headers(&mut self, headers: &[HashedBlockHeader]) -> StorageResult<()>; async fn store_headers_at_height( - &mut self, - headers: &[BlockHeader], - height: u32, - ) -> StorageResult<()>; - - //TODO - change API of the BlockHeaderStorage trait to accept (store) and return (load) - // HashedBlockHeaders instead of BlockHeaders to avoid unnecessary hashing and remove - // the two store_hashed_headers methods below. - async fn store_hashed_headers(&mut self, headers: &[HashedBlockHeader]) -> StorageResult<()>; - - async fn store_hashed_headers_at_height( &mut self, headers: &[HashedBlockHeader], height: u32, @@ -66,9 +55,9 @@ pub trait BlockHeaderStorage: Send + Sync + 'static { /// Returns `StorageError::InvalidArgument` when the range extends into a /// segment queued for deletion by a prior `truncate_above` (before the next /// `persist`). Callers must clamp the range to at most `get_tip_height`. - async fn load_headers(&self, range: Range) -> StorageResult>; + async fn load_headers(&self, range: Range) -> StorageResult>; - async fn get_header(&self, height: u32) -> StorageResult> { + async fn get_header(&self, height: u32) -> StorageResult> { if let Some(tip_height) = self.get_tip_height().await { if height > tip_height { return Ok(None); @@ -85,7 +74,7 @@ pub trait BlockHeaderStorage: Send + Sync + 'static { return Ok(None); } - Ok(self.load_headers(height..height + 1).await?.first().copied()) + Ok(self.load_headers(height..height + 1).await?.into_iter().next()) } async fn get_tip_height(&self) -> Option; @@ -161,27 +150,12 @@ impl PersistentStorage for PersistentBlockHeaderStorage { #[async_trait] impl BlockHeaderStorage for PersistentBlockHeaderStorage { - async fn store_headers(&mut self, headers: &[BlockHeader]) -> StorageResult<()> { + async fn store_headers(&mut self, headers: &[HashedBlockHeader]) -> StorageResult<()> { let height = self.block_headers.read().await.next_height(); self.store_headers_at_height(headers, height).await } async fn store_headers_at_height( - &mut self, - headers: &[BlockHeader], - height: u32, - ) -> StorageResult<()> { - let headers = - headers.iter().map(HashedBlockHeader::from).collect::>(); - self.store_hashed_headers_at_height(&headers, height).await - } - - async fn store_hashed_headers(&mut self, headers: &[HashedBlockHeader]) -> StorageResult<()> { - let height = self.block_headers.read().await.next_height(); - self.store_hashed_headers_at_height(headers, height).await - } - - async fn store_hashed_headers_at_height( &mut self, headers: &[HashedBlockHeader], height: u32, @@ -198,16 +172,9 @@ impl BlockHeaderStorage for PersistentBlockHeaderStorage { Ok(()) } - async fn load_headers(&self, range: Range) -> StorageResult> { - Ok(self - .block_headers - .write() - .await - .get_items(range) - .await? - .into_iter() - .map(|cached| *cached.header()) - .collect()) + // Returns the cached `HashedBlockHeader`s straight from the segment cache — no recomputation. + async fn load_headers(&self, range: Range) -> StorageResult> { + self.block_headers.write().await.get_items(range).await } async fn get_tip_height(&self) -> Option { @@ -268,9 +235,13 @@ mod tests { use super::*; use tempfile::TempDir; + fn hashed_batch(r: std::ops::Range) -> Vec { + BlockHeader::dummy_batch(r).into_iter().map(HashedBlockHeader::from).collect() + } + #[tokio::test] async fn test_get_tip() { - let headers = BlockHeader::dummy_batch(0..5); + let headers = hashed_batch(0..5); let tmp_dir = TempDir::new().unwrap(); let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); // Tip should be none before storing headers @@ -278,13 +249,13 @@ mod tests { // Add one header and validate tip storage.store_headers(&headers[0..1]).await.unwrap(); let tip = storage.get_tip().await.unwrap(); - let expected_tip = BlockHeaderTip::new(0, HashedBlockHeader::from(headers[0])); + let expected_tip = BlockHeaderTip::new(0, headers[0].clone()); assert_eq!(tip, expected_tip); assert_eq!(storage.get_tip_height().await, Some(0)); // Add multiple headers and validate tip storage.store_headers(&headers[1..]).await.unwrap(); let tip = storage.get_tip().await.unwrap(); - let expected_tip = BlockHeaderTip::new(4, HashedBlockHeader::from(headers[4])); + let expected_tip = BlockHeaderTip::new(4, headers[4].clone()); assert_eq!(tip, expected_tip); assert_eq!(storage.get_tip_height().await, Some(4)); } @@ -294,35 +265,32 @@ mod tests { let tmp_dir = TempDir::new().unwrap(); let mut storage = PersistentBlockHeaderStorage::open(tmp_dir.path()).await.unwrap(); - let headers = BlockHeader::dummy_batch(0..10); + let headers = hashed_batch(0..10); storage.store_headers(&headers).await.unwrap(); - let orphaned_hash = headers[7].block_hash(); + let orphaned_hash = *headers[7].hash(); assert_eq!(storage.get_header_height_by_hash(&orphaned_hash).await.unwrap(), Some(7)); storage.truncate_above(100).await.unwrap(); assert_eq!(storage.get_tip_height().await, Some(9)); - assert_eq!( - storage.get_header_height_by_hash(&headers[4].block_hash()).await.unwrap(), - Some(4) - ); + assert_eq!(storage.get_header_height_by_hash(headers[4].hash()).await.unwrap(), Some(4)); storage.truncate_above(5).await.unwrap(); assert_eq!(storage.get_tip_height().await, Some(5)); assert_eq!(storage.get_header_height_by_hash(&orphaned_hash).await.unwrap(), None); - let kept_hash = headers[3].block_hash(); + let kept_hash = *headers[3].hash(); assert_eq!(storage.get_header_height_by_hash(&kept_hash).await.unwrap(), Some(3)); - let replacement = BlockHeader::dummy_batch(100..105); + let replacement = hashed_batch(100..105); storage.store_headers_at_height(&replacement, 6).await.unwrap(); assert_eq!(storage.get_tip_height().await, Some(10)); let reloaded = storage.load_headers(6..11).await.unwrap(); assert_eq!(reloaded, replacement); - let new_hash = replacement[0].block_hash(); + let new_hash = *replacement[0].hash(); assert_eq!(storage.get_header_height_by_hash(&new_hash).await.unwrap(), Some(6)); // Exercise the durability contract: persist, drop, reopen, and verify diff --git a/dash-spv/src/storage/blocks.rs b/dash-spv/src/storage/blocks.rs index 27324d5c9..34b90a76f 100644 --- a/dash-spv/src/storage/blocks.rs +++ b/dash-spv/src/storage/blocks.rs @@ -33,6 +33,17 @@ pub trait BlockStorage: Send + Sync + 'static { /// A crash between `truncate_above` and `persist` may leave orphaned segment /// files on disk and cause the storage to reopen at the pre-truncation tip. async fn truncate_above(&mut self, target_height: CoreBlockHeight) -> StorageResult<()>; + + /// Declare the highest block height that has been applied to every + /// interested wallet, allowing the storage to stop holding those block + /// bodies in memory. + /// + /// Blocks at or below this height stay readable: `load_block` reloads them + /// from disk on demand. The watermark may move backwards when a rescan + /// re-processes lower heights. + /// + /// Defaults to a no-op for implementations that hold no in-memory cache. + async fn set_committed_height(&mut self, _height: CoreBlockHeight) {} } /// Persistent storage for full blocks using segmented files. @@ -81,6 +92,10 @@ impl BlockStorage for PersistentBlockStorage { async fn truncate_above(&mut self, target_height: u32) -> StorageResult<()> { self.blocks.write().await.truncate_above(target_height).await } + + async fn set_committed_height(&mut self, height: u32) { + self.blocks.write().await.set_committed_height(height); + } } #[cfg(test)] @@ -143,6 +158,49 @@ mod tests { assert_eq!(storage.load_block(3).await.unwrap(), None); } + /// Block bodies are the dominant memory consumer during a long backfill. + /// Once applied, they must leave memory while staying loadable from disk — + /// `handle_sync_event` re-reads stored blocks by height on resume/rescan. + #[tokio::test] + async fn test_committed_blocks_are_released_but_still_loadable() { + let temp_dir = TempDir::new().unwrap(); + let mut storage = PersistentBlockStorage::open(temp_dir.path()).await.unwrap(); + + // One block in each of segments 0, 1 and 2 (50_000 heights per segment). + // Carry real transaction payloads so the reload proves the block bodies + // round-trip, not merely that a slot is occupied. + let txs = vec![dashcore::Transaction::dummy_empty()]; + let low = HashedBlock::dummy(10, txs.clone()); + let mid = HashedBlock::dummy(50_010, txs.clone()); + let tip = HashedBlock::dummy(100_010, txs); + + storage.store_block(10, low.clone()).await.unwrap(); + storage.store_block(50_010, mid.clone()).await.unwrap(); + storage.store_block(100_010, tip.clone()).await.unwrap(); + storage.persist(temp_dir.path()).await.unwrap(); + + assert_eq!(storage.blocks.read().await.resident_segment_ids(), vec![0, 1, 2]); + + // Everything below segment 2 has been applied to the wallets. + storage.set_committed_height(99_999).await; + storage.persist(temp_dir.path()).await.unwrap(); + + // Segments 0 and 1 are gone from memory; the frontier segment stays. + assert_eq!( + storage.blocks.read().await.resident_segment_ids(), + vec![2], + "applied block segments must be released" + ); + + // All three blocks still load, byte-identically, via the disk fallback. + assert_eq!(storage.load_block(10).await.unwrap(), Some(low)); + assert_eq!(storage.load_block(50_010).await.unwrap(), Some(mid)); + assert_eq!(storage.load_block(100_010).await.unwrap(), Some(tip)); + + // Gaps inside a released segment still report absent, not sentinel data. + assert_eq!(storage.load_block(11).await.unwrap(), None); + } + #[tokio::test] async fn test_returns_none_for_gaps() { let temp_dir = TempDir::new().unwrap(); diff --git a/dash-spv/src/storage/filters.rs b/dash-spv/src/storage/filters.rs index 37a0cd827..249b966b9 100644 --- a/dash-spv/src/storage/filters.rs +++ b/dash-spv/src/storage/filters.rs @@ -21,6 +21,26 @@ pub trait FilterStorage: Send + Sync + 'static { async fn filter_tip_height(&self) -> StorageResult; + /// Lowest filter height present in storage, or `None` when no filters are + /// stored. + /// + /// `filter_tip_height` is a tip watermark only: storage is dense within + /// `[filter_start_height, filter_tip_height]`, but heights below the start + /// were never stored. Callers must not treat the tip as proof that storage + /// is contiguous from an arbitrary lower height. + async fn filter_start_height(&self) -> Option; + + /// Drop every stored filter, leaving the storage empty. + /// + /// Used when the stored range cannot serve the current scan (e.g. filters + /// were previously stored from a checkpoint above a wallet's rescan + /// start): keeping an unreachable island of tip-region filters would leave + /// a permanent hole below it that the contiguous store path cannot fill. + /// + /// Like `truncate_above`, the deletion is not durable until the next + /// successful `persist` call. + async fn clear_filters(&mut self) -> StorageResult<()>; + /// Drop all filters with `height > target_height`. /// /// Truncating above the current tip is a no-op, truncating below @@ -31,6 +51,17 @@ pub trait FilterStorage: Send + Sync + 'static { /// A crash between `truncate_above` and `persist` may leave orphaned segment /// files on disk and cause the storage to reopen at the pre-truncation tip. async fn truncate_above(&mut self, target_height: u32) -> StorageResult<()>; + + /// Declare the highest filter height that has been scanned and committed + /// for every wallet, allowing the storage to stop holding those filters in + /// memory. + /// + /// Filters at or below this height stay readable: `load_filters` reloads + /// them from disk on demand. The watermark may move backwards when a + /// rescan rolls the scan position back. + /// + /// Defaults to a no-op for implementations that hold no in-memory cache. + async fn set_committed_height(&mut self, _height: u32) {} } pub struct PersistentFilterStorage { @@ -79,9 +110,21 @@ impl FilterStorage for PersistentFilterStorage { Ok(self.filters.read().await.tip_height().unwrap_or(0)) } + async fn filter_start_height(&self) -> Option { + self.filters.read().await.start_height() + } + + async fn clear_filters(&mut self) -> StorageResult<()> { + self.filters.write().await.clear() + } + async fn truncate_above(&mut self, target_height: u32) -> StorageResult<()> { self.filters.write().await.truncate_above(target_height).await } + + async fn set_committed_height(&mut self, height: u32) { + self.filters.write().await.set_committed_height(height); + } } #[cfg(test)] @@ -107,4 +150,57 @@ mod tests { assert_eq!(storage.filter_tip_height().await.unwrap(), 2); assert!(storage.load_filters(3..4).await.is_err()); } + + /// Filters stored only in a high region (as when headers previously + /// synced from a checkpoint): the start accessor reports the true lowest + /// stored height, reads below it fail loudly instead of returning + /// sentinels, and `clear_filters` durably empties the storage. + #[tokio::test] + async fn test_filter_start_height_and_clear_filters() { + let tmp_dir = TempDir::new().unwrap(); + let mut storage = PersistentFilterStorage::open(tmp_dir.path()).await.unwrap(); + + assert_eq!(storage.filter_start_height().await, None); + + for height in 900..=1000 { + storage.store_filter(height, &filter_bytes(height as u8)).await.unwrap(); + } + assert_eq!(storage.filter_start_height().await, Some(900)); + assert_eq!(storage.filter_tip_height().await.unwrap(), 1000); + + // Reads that begin below the stored range must error, not hand back + // sentinel data zipped against real headers. + assert!(storage.load_filters(100..200).await.is_err()); + assert!(storage.load_filters(850..950).await.is_err()); + assert_eq!(storage.load_filters(900..1001).await.unwrap().len(), 101); + + storage.persist(tmp_dir.path()).await.unwrap(); + let segment_file = + tmp_dir.path().join(PersistentFilterStorage::FOLDER_NAME).join("segment_0000.dat"); + assert!(segment_file.exists()); + + // The start height survives a reload. + let mut storage = PersistentFilterStorage::open(tmp_dir.path()).await.unwrap(); + assert_eq!(storage.filter_start_height().await, Some(900)); + + storage.clear_filters().await.unwrap(); + assert_eq!(storage.filter_start_height().await, None); + assert_eq!(storage.filter_tip_height().await.unwrap(), 0); + assert!(storage.load_filters(900..901).await.is_err()); + + storage.persist(tmp_dir.path()).await.unwrap(); + assert!(!segment_file.exists()); + + // Reopen: the clear is durable, and the storage accepts a fresh + // contiguous range from a lower start. + let mut storage = PersistentFilterStorage::open(tmp_dir.path()).await.unwrap(); + assert_eq!(storage.filter_start_height().await, None); + assert_eq!(storage.filter_tip_height().await.unwrap(), 0); + + for height in 100..=110 { + storage.store_filter(height, &filter_bytes(height as u8)).await.unwrap(); + } + assert_eq!(storage.filter_start_height().await, Some(100)); + assert_eq!(storage.load_filters(100..111).await.unwrap().len(), 11); + } } diff --git a/dash-spv/src/storage/lockfile.rs b/dash-spv/src/storage/lockfile.rs index 03d3a3141..6a85b1cc3 100644 --- a/dash-spv/src/storage/lockfile.rs +++ b/dash-spv/src/storage/lockfile.rs @@ -22,18 +22,10 @@ impl LockFile { .open(&path) .map_err(|e| StorageError::WriteFailed(format!("Failed to create lock file: {}", e)))?; - file.try_lock().map_err(|e| match e { - std::fs::TryLockError::WouldBlock => StorageError::DirectoryLocked(format!( - "Data directory '{}' is already in use by another process", - path.parent().map(|p| p.display().to_string()).unwrap_or_default() - )), - std::fs::TryLockError::Error(io_err) => { - StorageError::WriteFailed(format!("Failed to acquire lock: {}", io_err)) + if classify_lock_result(file.try_lock(), &path)? { + if let Err(e) = writeln!(file, "{}", std::process::id()) { + tracing::warn!("Failed to write PID to lock file: {}", e); } - })?; - - if let Err(e) = writeln!(file, "{}", std::process::id()) { - tracing::warn!("Failed to write PID to lock file: {}", e); } Ok(Self { @@ -43,6 +35,42 @@ impl LockFile { } } +/// Interprets the result of [`File::try_lock`] for an advisory data-directory lock. +/// +/// Returns `Ok(true)` when the lock was acquired, `Ok(false)` when advisory +/// locking is unsupported on the platform (so the caller should proceed without +/// it), and `Err(..)` when the directory is already locked or a genuine I/O +/// error occurred. +fn classify_lock_result( + result: Result<(), std::fs::TryLockError>, + path: &std::path::Path, +) -> StorageResult { + match result { + Ok(()) => Ok(true), + Err(std::fs::TryLockError::WouldBlock) => Err(StorageError::DirectoryLocked(format!( + "Data directory '{}' is already in use by another process", + path.parent().map(|p| p.display().to_string()).unwrap_or_default() + ))), + // Some platforms (e.g. Android) do not implement advisory file locking + // and return `ErrorKind::Unsupported` at the std layer. The lock is only + // a best-effort guard against multiple processes sharing one data + // directory; on such single-process platforms its absence is safe, so + // degrade gracefully instead of failing init. + Err(std::fs::TryLockError::Error(io_err)) + if io_err.kind() == std::io::ErrorKind::Unsupported => + { + tracing::warn!( + "advisory file lock not supported on this platform ({io_err}); \ + continuing without multi-process protection" + ); + Ok(false) + } + Err(std::fs::TryLockError::Error(io_err)) => { + Err(StorageError::WriteFailed(format!("Failed to acquire lock: {io_err}"))) + } + } +} + impl Drop for LockFile { fn drop(&mut self) { if let Err(e) = std::fs::remove_file(&self.path) { @@ -123,6 +151,48 @@ mod tests { assert!(lock2.is_ok(), "Should acquire lock after previous one dropped"); } + #[test] + fn test_classify_lock_result_acquired() { + let path = PathBuf::from("/tmp/data/.lock"); + assert!(classify_lock_result(Ok(()), &path).expect("acquired lock should be Ok")); + } + + #[test] + fn test_classify_lock_result_would_block() { + let path = PathBuf::from("/tmp/data/.lock"); + let result = classify_lock_result(Err(std::fs::TryLockError::WouldBlock), &path); + match result { + Err(StorageError::DirectoryLocked(msg)) => { + assert!(msg.contains("already in use"), "unexpected message: {msg}"); + } + other => panic!("Expected DirectoryLocked error, got: {:?}", other), + } + } + + #[test] + fn test_classify_lock_result_unsupported_is_non_fatal() { + // Platforms such as Android return `ErrorKind::Unsupported` from + // `try_lock`; this must degrade gracefully rather than fail init. + let path = PathBuf::from("/tmp/data/.lock"); + let io_err = + std::io::Error::new(std::io::ErrorKind::Unsupported, "try_lock() not supported"); + let result = classify_lock_result(Err(std::fs::TryLockError::Error(io_err)), &path); + assert!(!result.expect("unsupported lock should be Ok(false)")); + } + + #[test] + fn test_classify_lock_result_genuine_io_error_is_fatal() { + let path = PathBuf::from("/tmp/data/.lock"); + let io_err = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "denied"); + let result = classify_lock_result(Err(std::fs::TryLockError::Error(io_err)), &path); + match result { + Err(StorageError::WriteFailed(msg)) => { + assert!(msg.contains("Failed to acquire lock"), "unexpected message: {msg}"); + } + other => panic!("Expected WriteFailed error, got: {:?}", other), + } + } + #[test] fn test_lock_file_removed_on_drop() { let temp_dir = TempDir::new().expect("Failed to create temp directory"); diff --git a/dash-spv/src/storage/mod.rs b/dash-spv/src/storage/mod.rs index 57e4fb736..70a851acb 100644 --- a/dash-spv/src/storage/mod.rs +++ b/dash-spv/src/storage/mod.rs @@ -19,9 +19,8 @@ use crate::ClientConfig; use async_trait::async_trait; use dashcore::hash_types::FilterHeader; use dashcore::prelude::CoreBlockHeight; -use dashcore::Header as BlockHeader; use std::ops::Range; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use tokio::sync::RwLock; @@ -100,16 +99,34 @@ pub struct DiskStorageManager { _lock_file: LockFile, } +/// Derives the lockfile path for a storage directory by appending `.lock` to +/// the directory's full name. `Path::set_extension` is unsuitable here: it +/// replaces everything after the last dot, so directory names containing dots +/// (e.g. `95.183.53.19-9999` and `95.183.53.20-9999`) would collide on the +/// same lockfile. +fn lock_file_path(storage_path: &Path) -> PathBuf { + match storage_path.file_name() { + Some(name) => { + let mut lock_name = name.to_os_string(); + lock_name.push(".lock"); + storage_path.with_file_name(lock_name) + } + // Paths without a final component (e.g. `/` or `..`); keep the + // previous `set_extension` behavior. + None => { + let mut lock_file = storage_path.to_path_buf(); + lock_file.set_extension("lock"); + lock_file + } + } +} + impl DiskStorageManager { pub async fn new(config: &ClientConfig) -> StorageResult { use std::fs; let storage_path = config.storage_path.clone(); - let lock_file = { - let mut lock_file = storage_path.clone(); - lock_file.set_extension("lock"); - lock_file - }; + let lock_file = lock_file_path(&storage_path); fs::create_dir_all(&storage_path)?; @@ -269,31 +286,19 @@ impl StorageManager for DiskStorageManager { #[async_trait] impl BlockHeaderStorage for DiskStorageManager { - async fn store_headers(&mut self, headers: &[BlockHeader]) -> StorageResult<()> { + async fn store_headers(&mut self, headers: &[HashedBlockHeader]) -> StorageResult<()> { self.block_headers.write().await.store_headers(headers).await } async fn store_headers_at_height( - &mut self, - headers: &[BlockHeader], - height: u32, - ) -> StorageResult<()> { - self.block_headers.write().await.store_headers_at_height(headers, height).await - } - - async fn store_hashed_headers(&mut self, headers: &[HashedBlockHeader]) -> StorageResult<()> { - self.block_headers.write().await.store_hashed_headers(headers).await - } - - async fn store_hashed_headers_at_height( &mut self, headers: &[HashedBlockHeader], height: u32, ) -> StorageResult<()> { - self.block_headers.write().await.store_hashed_headers_at_height(headers, height).await + self.block_headers.write().await.store_headers_at_height(headers, height).await } - async fn load_headers(&self, range: Range) -> StorageResult> { + async fn load_headers(&self, range: Range) -> StorageResult> { self.block_headers.read().await.load_headers(range).await } @@ -370,9 +375,21 @@ impl filters::FilterStorage for DiskStorageManager { self.filters.read().await.filter_tip_height().await } + async fn filter_start_height(&self) -> Option { + self.filters.read().await.filter_start_height().await + } + + async fn clear_filters(&mut self) -> StorageResult<()> { + self.filters.write().await.clear_filters().await + } + async fn truncate_above(&mut self, target_height: u32) -> StorageResult<()> { self.filters.write().await.truncate_above(target_height).await } + + async fn set_committed_height(&mut self, height: u32) { + self.filters.write().await.set_committed_height(height).await; + } } #[async_trait] @@ -388,6 +405,10 @@ impl BlockStorage for DiskStorageManager { async fn truncate_above(&mut self, target_height: u32) -> StorageResult<()> { self.blocks.write().await.truncate_above(target_height).await } + + async fn set_committed_height(&mut self, height: u32) { + self.blocks.write().await.set_committed_height(height).await; + } } #[async_trait] @@ -426,6 +447,45 @@ mod tests { use dashcore::Header as BlockHeader; use tempfile::TempDir; + fn hashed_batch(r: std::ops::Range) -> Vec { + BlockHeader::dummy_batch(r).into_iter().map(HashedBlockHeader::from).collect() + } + + #[test] + fn test_lock_file_path_appends_to_full_name() { + // Dot-free directory names keep the historical behavior. + assert_eq!( + lock_file_path(Path::new("data/discovery")), + PathBuf::from("data/discovery.lock") + ); + + // Dotted directory names must not have their "extension" replaced: + // distinct sibling directories need distinct lockfiles. + let a = lock_file_path(Path::new("probes/95.183.53.19-9999")); + let b = lock_file_path(Path::new("probes/95.183.53.20-9999")); + assert_eq!(a, PathBuf::from("probes/95.183.53.19-9999.lock")); + assert_eq!(b, PathBuf::from("probes/95.183.53.20-9999.lock")); + assert_ne!(a, b); + } + + #[tokio::test] + async fn test_dotted_sibling_dirs_do_not_share_lock() -> Result<(), Box> + { + let temp_dir = TempDir::new()?; + let path_a = temp_dir.path().join("95.183.53.19-9999"); + let path_b = temp_dir.path().join("95.183.53.20-9999"); + + let config_a = ClientConfig::testnet().with_storage_path(&path_a); + let config_b = ClientConfig::testnet().with_storage_path(&path_b); + + let _storage_a = DiskStorageManager::new(&config_a).await?; + // Before the fix this failed with "Data directory ... is already in + // use by another process" because both paths derived the same lockfile. + let _storage_b = DiskStorageManager::new(&config_b).await?; + + Ok(()) + } + #[tokio::test] async fn test_store_load_headers() -> Result<(), Box> { // Create a temporary directory for the test @@ -433,7 +493,7 @@ mod tests { let config = ClientConfig::testnet().with_storage_path(temp_dir.path()); let mut storage = DiskStorageManager::new(&config).await.expect("Unable to create storage"); - let headers = BlockHeader::dummy_batch(0..60_000); + let headers = hashed_batch(0..60_000); storage.store_headers(&headers[0..0]).await.expect("Should handle empty header batch"); assert_eq!(storage.get_tip_height().await, None); @@ -478,8 +538,8 @@ mod tests { // Create test headers starting from checkpoint height const CHECKPOINT_HEIGHT: u32 = 1_100_000; - let headers: Vec = - BlockHeader::dummy_batch(CHECKPOINT_HEIGHT..CHECKPOINT_HEIGHT + 100); + let headers: Vec = + hashed_batch(CHECKPOINT_HEIGHT..CHECKPOINT_HEIGHT + 100); storage.store_headers_at_height(&headers, CHECKPOINT_HEIGHT).await?; @@ -496,18 +556,18 @@ mod tests { async fn check_storage( storage: &DiskStorageManager, - headers: &[BlockHeader], + headers: &[HashedBlockHeader], ) -> StorageResult<()> { assert_eq!(storage.get_stored_headers_len().await, headers.len() as u32); let header_at_base = storage.get_header(CHECKPOINT_HEIGHT).await?; - assert_eq!(header_at_base, Some(headers[0])); + assert_eq!(header_at_base, Some(headers[0].clone())); let header_at_ending = storage.get_header(CHECKPOINT_HEIGHT + 99).await?; - assert_eq!(header_at_ending, Some(headers[99])); + assert_eq!(header_at_ending, Some(headers[99].clone())); // Test the reverse index (hash -> blockchain height) - let hash_0 = headers[0].block_hash(); + let hash_0 = *headers[0].hash(); let height_0 = storage.get_header_height_by_hash(&hash_0).await?; assert_eq!( height_0, @@ -515,7 +575,7 @@ mod tests { "Hash should map to blockchain height 1,100,000" ); - let hash_99 = headers[99].block_hash(); + let hash_99 = *headers[99].hash(); let height_99 = storage.get_header_height_by_hash(&hash_99).await?; assert_eq!( height_99, @@ -536,13 +596,13 @@ mod tests { let mut storage = DiskStorageManager::new(&config).await.unwrap(); // Create and store headers - let headers = BlockHeader::dummy_batch(0..10); + let headers = hashed_batch(0..10); storage.store_headers(&headers).await.unwrap(); // Test reverse lookups for (i, header) in headers.iter().enumerate() { - let hash = header.block_hash(); + let hash = *header.hash(); let height = storage.get_header_height_by_hash(&hash).await.unwrap(); assert_eq!(height, Some(i as u32), "Height mismatch for header {}", i); } @@ -558,7 +618,7 @@ mod tests { // We need to get the actual headers that were stored to test properly for i in 0..10 { let stored_header = storage.get_header(i).await.unwrap().unwrap(); - let hash = stored_header.block_hash(); + let hash = *stored_header.hash(); let height = storage.get_header_height_by_hash(&hash).await.unwrap(); assert_eq!(height, Some(i), "Height mismatch after reload for header {}", i); } @@ -571,10 +631,10 @@ mod tests { DiskStorageManager::with_temp_dir().await.expect("Failed to create tmp storage"); // Store some headers - let header = BlockHeader::dummy_batch(0..1); + let header = hashed_batch(0..1); storage.store_headers(&header).await.unwrap(); - let hash = header[0].block_hash(); + let hash = *header[0].hash(); assert!(storage.get_header_height_by_hash(&hash).await.unwrap().is_some()); // Clear storage @@ -590,10 +650,10 @@ mod tests { let config = ClientConfig::regtest().with_storage_path(temp_dir.path()); let mut mgr = DiskStorageManager::new(&config).await.unwrap(); - let headers = BlockHeader::dummy_batch(0..10); + let headers = hashed_batch(0..10); mgr.store_headers(&headers).await.unwrap(); - let orphaned_hash = headers[7].block_hash(); + let orphaned_hash = *headers[7].hash(); assert_eq!(mgr.get_header_height_by_hash(&orphaned_hash).await.unwrap(), Some(7)); ::truncate_above(&mut mgr, 5).await.unwrap(); @@ -601,7 +661,7 @@ mod tests { assert_eq!(mgr.get_tip_height().await, Some(5)); assert_eq!(mgr.get_header_height_by_hash(&orphaned_hash).await.unwrap(), None); - let kept_hash = headers[3].block_hash(); + let kept_hash = *headers[3].hash(); assert_eq!(mgr.get_header_height_by_hash(&kept_hash).await.unwrap(), Some(3)); } diff --git a/dash-spv/src/storage/peers.rs b/dash-spv/src/storage/peers.rs index 69e075ab9..360e83650 100644 --- a/dash-spv/src/storage/peers.rs +++ b/dash-spv/src/storage/peers.rs @@ -10,7 +10,7 @@ use dashcore::{ use crate::{ error::StorageResult, - network::reputation::PeerReputation, + network::PeerReputation, storage::{io::atomic_write, PersistentStorage}, StorageError, }; @@ -97,12 +97,12 @@ impl PeerStorage for PersistentPeerStorage { return Ok(Vec::new()); }; - let mut peers = Vec::new(); - let peers = tokio::task::spawn_blocking(move || { let file = File::open(&peers_file)?; let mut reader = BufReader::new(file); + let mut peers = Vec::new(); + loop { match AddrV2Message::consensus_decode(&mut reader) { Ok(peer) => peers.push(peer), @@ -116,7 +116,6 @@ impl PeerStorage for PersistentPeerStorage { } } } - Ok(peers) }) .await diff --git a/dash-spv/src/storage/segments.rs b/dash-spv/src/storage/segments.rs index 49f291add..ec900899c 100644 --- a/dash-spv/src/storage/segments.rs +++ b/dash-spv/src/storage/segments.rs @@ -83,6 +83,14 @@ pub struct SegmentCache { /// Segment ids whose backing files must be removed on the next `persist`. /// Populated by `truncate_above` for segments that are dropped entirely. to_delete: HashSet, + /// Highest height whose items are durably persisted *and* fully consumed + /// by every reader, as declared by the owning sync manager through + /// [`SegmentCache::set_committed_height`]. + /// + /// `None` — the default — disables committed-height release entirely, so a + /// cache whose owner never declares a watermark keeps exactly the residency + /// behavior it had before this field existed. + committed_height: Option, } impl SegmentCache { @@ -98,6 +106,7 @@ impl SegmentCache { start_height: None, segments_dir: segments_dir.clone(), to_delete: HashSet::new(), + committed_height: None, }; // Building the metadata @@ -107,16 +116,9 @@ impl SegmentCache { for entry in entries.flatten() { if let Some(name) = entry.file_name().to_str() { - if name.starts_with(I::SEGMENT_PREFIX) - && name.ends_with(&format!(".{}", I::DATA_FILE_EXTENSION)) - { - let segment_id_start = I::SEGMENT_PREFIX.len() + 1; - let segment_id_end = segment_id_start + 4; - - if let Ok(id) = name[segment_id_start..segment_id_end].parse::() { - max_seg_id = Some(max_seg_id.map_or(id, |max: u32| max.max(id))); - min_seg_id = Some(min_seg_id.map_or(id, |min: u32| min.min(id))); - } + if let Some(id) = Self::parse_segment_id(name) { + max_seg_id = Some(max_seg_id.map_or(id, |max: u32| max.max(id))); + min_seg_id = Some(min_seg_id.map_or(id, |min: u32| min.min(id))); } } } @@ -141,6 +143,21 @@ impl SegmentCache { Ok(cache) } + /// Parse the segment id out of a segment file name of the form + /// `{SEGMENT_PREFIX}_{id:04}.{DATA_FILE_EXTENSION}` (see + /// [`Persistable::segment_file_name`]). + /// + /// The entire remaining component between the `{prefix}_` and + /// `.{extension}` fixtures must parse as a `u32`, so trailing junk + /// (`segment_0000junk.dat`) is rejected and ids longer than the + /// zero-padding width (`segment_100000.dat`) are accepted, not truncated. + fn parse_segment_id(file_name: &str) -> Option { + let separator = format!("{}_", I::SEGMENT_PREFIX); + let suffix = format!(".{}", I::DATA_FILE_EXTENSION); + + file_name.strip_prefix(&separator)?.strip_suffix(&suffix)?.parse::().ok() + } + #[inline] fn height_to_segment_id(height: u32) -> u32 { height / Segment::::ITEMS_PER_SEGMENT @@ -226,6 +243,17 @@ impl SegmentCache { ))); } + // Reject ranges that begin below the lowest stored height: those slots + // were never populated and hold only sentinels. Without this guard the + // read would trip the `first_valid_offset` debug assertion below in + // debug builds, and silently hand back sentinel data in release builds. + if self.start_height.is_none_or(|lowest| start < lowest) { + return Err(StorageError::InvalidArgument(format!( + "get_items range {height_range:?} begins below the lowest stored height {:?}", + self.start_height + ))); + } + let mut items = Vec::with_capacity((end - start) as usize); let start_segment = Self::height_to_segment_id(start); @@ -340,7 +368,7 @@ impl SegmentCache { let mut height = start_height; - tracing::debug!( + tracing::trace!( "SegmentsCache: storing {} items starting at height {}", items.len(), height, @@ -426,6 +454,66 @@ impl SegmentCache { self.tip_height = Some(target_height); + // The watermark must never claim heights that no longer exist: a + // rescan re-reads this range, and a stale high watermark would keep + // releasing segments it is actively refilling. + if let Some(committed) = self.committed_height { + if committed > target_height { + self.committed_height = Some(target_height); + } + } + + Ok(()) + } + + /// Drop every stored item, leaving the cache empty. + /// + /// All backing segment files — including segments not currently resident + /// in memory — are queued for deletion on the next `persist`. Like + /// `truncate_above`, the deletion is not durable until that `persist` + /// succeeds; a crash in between leaves the old files on disk and the + /// cache reopens with the pre-clear contents. + /// + /// The directory scan runs to completion *before* any cache state is + /// mutated: a failed scan must not leave the cache reporting empty while + /// persisted segment files survive on disk, since those would resurrect + /// after restart. A missing segments directory is treated as an empty + /// (already-cleared) cache; any other I/O error is propagated. + pub fn clear(&mut self) -> StorageResult<()> { + // Queue every on-disk segment file, discovered by directory scan since + // only up to MAX_ACTIVE_SEGMENTS of them are resident in memory. + let mut to_delete = HashSet::new(); + match fs::read_dir(&self.segments_dir) { + Ok(entries) => { + for entry in entries { + let entry = entry?; + if let Some(name) = entry.file_name().to_str() { + if let Some(id) = Self::parse_segment_id(name) { + to_delete.insert(id); + } + } + } + } + // No directory yet means nothing was ever persisted here. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(StorageError::Io(e)), + } + + // Scan succeeded — now it is safe to mutate cache state. Resident and + // evicted segments may be dirty and not persisted yet, so the scan + // above cannot see them. Queue their ids too; `persist` ignores + // missing files. + to_delete.extend(self.segments.keys().copied()); + to_delete.extend(self.evicted.keys().copied()); + + self.to_delete.extend(to_delete); + self.segments.clear(); + self.evicted.clear(); + self.tip_height = None; + self.start_height = None; + // Nothing is stored, so nothing is committed. + self.committed_height = None; + Ok(()) } @@ -459,6 +547,123 @@ impl SegmentCache { tracing::error!("Failed to persist segment with id {id}: {e}"); } } + + // After the writes above, so segments cleaned by *this* pass are + // eligible immediately. A segment whose persist failed is still Dirty + // and is therefore skipped. + let released = self.release_committed_segments(); + if released > 0 { + tracing::debug!( + "SegmentCache: released {} committed segment(s) below height {:?}; {} resident", + released, + self.committed_height, + self.segments.len(), + ); + } + } + + /// Declare the highest height whose items are durably persisted and fully + /// consumed, so the cache may stop holding them in memory. + /// + /// Segments lying *entirely* at or below `height` are released from memory + /// by the next [`SegmentCache::persist`], once they are `Clean`. The + /// on-disk file remains the source of truth and a later read of a released + /// height transparently reloads the segment through the same lazy path used + /// for any never-resident segment — the data a reader observes is + /// unchanged, only the resident-set size is. + /// + /// The watermark may move **backwards**: a wallet rescan rolls it back + /// before re-reading lower heights. A regression only ever narrows the + /// release window, so it is always safe. Callers therefore assign rather + /// than accumulate, and no monotonicity is enforced here. + pub fn set_committed_height(&mut self, height: u32) { + self.committed_height = Some(height); + } + + /// The committed-height watermark, or `None` when the owner has never + /// declared one (in which case no committed-height release occurs). + /// + /// Test-only for now — nothing reads the watermark back in production. + /// Promote it to an unconditional accessor when a caller needs it. + #[cfg(test)] + #[inline] + pub fn committed_height(&self) -> Option { + self.committed_height + } + + /// Sorted ids of the segments currently held in memory. Test-only hook for + /// asserting residency from the storage wrappers. + #[cfg(test)] + pub(crate) fn resident_segment_ids(&self) -> Vec { + let mut ids: Vec = self.segments.keys().copied().collect(); + ids.sort_unstable(); + ids + } + + /// Release the in-memory items of every fully-committed clean segment, + /// keeping the on-disk file as the source of truth. + /// + /// This is what stops a long backfill from pinning every segment it ever + /// touched. `MAX_ACTIVE_SEGMENTS` alone cannot: a scan spanning fewer than + /// ten segments never trips the LRU, so every downloaded item stays + /// resident until the process dies. + /// + /// Releasing a `Clean` segment is invisible to readers. `Clean` is reachable + /// only via [`Segment::load`] from an existing file or a successful + /// [`Segment::persist`], so a clean segment's items are byte-identical to + /// its backing file, and `get_segment_mut` reloads exactly those bytes on + /// the next access. + /// + /// Three classes are deliberately kept: + /// - `Dirty` segments — their contents exist *only* in memory, so dropping + /// them would lose data outright. + /// - The segment holding the sync frontier (`tip_height`) — it is still + /// being written, and releasing it would force a reload on the very next + /// store. + /// - Any segment not entirely at or below the watermark, so a partially + /// committed segment is never released out from under an active reader. + /// + /// Segments queued in `to_delete` cannot appear here: both `truncate_above` + /// and `clear` remove a segment from `self.segments` in the same step that + /// queues it, so the two sets are disjoint by construction. + /// + /// Returns the number of segments released. + fn release_committed_segments(&mut self) -> usize { + let Some(committed) = self.committed_height else { + return 0; + }; + + // Still-written frontier; never release it. + let frontier_segment = self.tip_height.map(Self::height_to_segment_id); + + let items_per_segment = Segment::::ITEMS_PER_SEGMENT as u64; + let committed = committed as u64; + + // Widened to u64 so a segment id near u32::MAX cannot overflow the + // range arithmetic; such a segment simply fails the comparison. + let releasable: Vec = self + .segments + .iter() + .filter(|(id, segment)| { + if segment.state != SegmentState::Clean { + return false; + } + + if Some(**id) == frontier_segment { + return false; + } + + let last_height = (**id as u64) * items_per_segment + items_per_segment - 1; + last_height <= committed + }) + .map(|(id, _)| *id) + .collect(); + + for id in &releasable { + self.segments.remove(id); + } + + releasable.len() } #[inline] @@ -1092,6 +1297,353 @@ mod tests { ); } + /// Ranges that begin below the lowest stored height read never-populated + /// sentinel slots and must fail with a typed error rather than panicking + /// on the `first_valid_offset` debug assertion (or silently returning + /// sentinels in release builds). + #[tokio::test] + async fn test_get_items_below_start_height_errors() { + let tmp_dir = TempDir::new().unwrap(); + + let items = FilterHeader::dummy_batch(0..10); + + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + cache.store_items_at_height(&items, 900).await.unwrap(); + assert_eq!(cache.start_height(), Some(900)); + + // Entirely below the stored range (the #892 shape: a scan start far + // below tip-region storage, within the same segment). + assert!(matches!(cache.get_items(100..200).await, Err(StorageError::InvalidArgument(_)))); + // Straddling the lowest stored height. + assert!(matches!(cache.get_items(895..905).await, Err(StorageError::InvalidArgument(_)))); + // The stored range itself is readable. + assert_eq!(cache.get_items(900..910).await.unwrap(), items); + + // Same result when the never-populated slots live in a lower segment + // than the stored data. + const ITEMS_PER_SEGMENT: u32 = Segment::::ITEMS_PER_SEGMENT; + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + cache.store_items_at_height(&items, ITEMS_PER_SEGMENT + 5).await.unwrap(); + assert!(matches!(cache.get_items(100..200).await, Err(StorageError::InvalidArgument(_)))); + + // An empty cache rejects every range. + let mut cache = + SegmentCache::::load_or_new(tmp_dir.path().join("empty")).await.unwrap(); + assert!(matches!(cache.get_items(0..1).await, Err(StorageError::InvalidArgument(_)))); + } + + #[tokio::test] + async fn test_segment_cache_clear() { + let tmp_dir = TempDir::new().unwrap(); + + const ITEMS_PER_SEGMENT: u32 = Segment::::ITEMS_PER_SEGMENT; + + // Two segments' worth of items, persisted to disk. + let items = FilterHeader::dummy_batch(0..ITEMS_PER_SEGMENT + 5); + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + cache.store_items_at_height(&items, 0).await.unwrap(); + cache.persist(tmp_dir.path()).await; + + let file_0 = tmp_dir.path().join(FilterHeader::segment_file_name(0)); + let file_1 = tmp_dir.path().join(FilterHeader::segment_file_name(1)); + assert!(file_0.exists()); + assert!(file_1.exists()); + + // Reopen (so only the min/max segments are resident) and clear. + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + cache.clear().unwrap(); + + assert_eq!(cache.tip_height(), None); + assert_eq!(cache.start_height(), None); + assert_eq!(cache.get_item(3).await.unwrap(), None); + assert!(cache.get_items(0..5).await.is_err()); + + // The deletion becomes durable on persist. + cache.persist(tmp_dir.path()).await; + assert!(!file_0.exists()); + assert!(!file_1.exists()); + + // Storing a fresh range from a lower origin after clear is sound. + let replacement = FilterHeader::dummy_batch(100..110); + cache.store_items_at_height(&replacement, 100).await.unwrap(); + assert_eq!(cache.start_height(), Some(100)); + assert_eq!(cache.tip_height(), Some(109)); + assert_eq!(cache.get_items(100..110).await.unwrap(), replacement); + + cache.persist(tmp_dir.path()).await; + let mut reloaded = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + assert_eq!(reloaded.start_height(), Some(100)); + assert_eq!(reloaded.tip_height(), Some(109)); + assert_eq!(reloaded.get_items(100..110).await.unwrap(), replacement); + } + + /// `clear` must also drop dirty in-memory state that was never persisted, + /// and queue previously-persisted files of those same segments. + #[tokio::test] + async fn test_segment_cache_clear_unpersisted_state() { + let tmp_dir = TempDir::new().unwrap(); + + let items = FilterHeader::dummy_batch(0..10); + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + cache.store_items_at_height(&items, 0).await.unwrap(); + cache.persist(tmp_dir.path()).await; + + // Dirty the resident segment again so it differs from disk, then clear + // without an intervening persist. + let more = FilterHeader::dummy_batch(100..105); + cache.store_items_at_height(&more, 10).await.unwrap(); + cache.clear().unwrap(); + + assert_eq!(cache.tip_height(), None); + assert_eq!(cache.start_height(), None); + + cache.persist(tmp_dir.path()).await; + assert!(!tmp_dir.path().join(FilterHeader::segment_file_name(0)).exists()); + + let reloaded = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + assert_eq!(reloaded.tip_height(), None); + assert_eq!(reloaded.start_height(), None); + } + + #[test] + fn test_parse_segment_id() { + type Cache = SegmentCache; + + // Round-trips the writer's `{prefix}_{id:04}.{ext}` format for a range + // of ids, including ones wider than the zero-padding. + for id in [0u32, 1, 42, 9999, 10_000, 100_000, u32::MAX] { + assert_eq!(Cache::parse_segment_id(&FilterHeader::segment_file_name(id)), Some(id)); + } + + // Zero-padded low ids parse to their numeric value, not truncated. + assert_eq!(Cache::parse_segment_id("segment_0000.dat"), Some(0)); + assert_eq!(Cache::parse_segment_id("segment_0005.dat"), Some(5)); + + // Ids wider than the padding are accepted whole, never truncated to + // the first four digits. + assert_eq!(Cache::parse_segment_id("segment_100000.dat"), Some(100_000)); + + // Trailing junk between the id and the extension is rejected rather + // than silently parsed as a prefix of the component. + assert_eq!(Cache::parse_segment_id("segment_0000junk.dat"), None); + assert_eq!(Cache::parse_segment_id("segment_12ab.dat"), None); + + // Wrong prefix, wrong extension, missing separator, or a + // non-numeric/empty id component are all rejected. + assert_eq!(Cache::parse_segment_id("segment_0000.tmp"), None); + assert_eq!(Cache::parse_segment_id("other_0000.dat"), None); + assert_eq!(Cache::parse_segment_id("segment0000.dat"), None); + assert_eq!(Cache::parse_segment_id("segment_.dat"), None); + assert_eq!(Cache::parse_segment_id("segment_-1.dat"), None); + assert_eq!(Cache::parse_segment_id("segment_0000.dat.bak"), None); + + // The filter storage's item type round-trips through the same helper. + assert_eq!( + SegmentCache::>::parse_segment_id(&>::segment_file_name(7)), + Some(7) + ); + } + + /// The headline guarantee: a committed, clean segment is dropped from + /// memory on persist, and a later read transparently reloads it from disk + /// byte-identically. + /// + /// This is the leak that pinned a 350k-block backfill in RAM: the scan + /// spans fewer than `MAX_ACTIVE_SEGMENTS`, so the LRU never fires and every + /// item stays resident until the process dies. + #[tokio::test] + async fn test_release_committed_segments_reloads_from_disk() { + let tmp_dir = TempDir::new().unwrap(); + + const ITEMS_PER_SEGMENT: u32 = Segment::::ITEMS_PER_SEGMENT; + + // Dense across segments 0 and 1, plus a few items into segment 2 so the + // frontier lives above the range we expect to be released. + let items = FilterHeader::dummy_batch(0..ITEMS_PER_SEGMENT * 2 + 5); + + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + cache.store_items_at_height(&items, 0).await.unwrap(); + cache.persist(tmp_dir.path()).await; + + // Baseline: without a watermark nothing is released, which is exactly + // the pre-fix behavior. + assert_eq!(cache.committed_height(), None); + assert_eq!(cache.segments.len(), 3, "no watermark declared, so nothing is released"); + + // Segments 0 and 1 are now fully committed (their last heights are + // ITEMS_PER_SEGMENT-1 and 2*ITEMS_PER_SEGMENT-1). + cache.set_committed_height(ITEMS_PER_SEGMENT * 2 - 1); + cache.persist(tmp_dir.path()).await; + + assert!(!cache.segments.contains_key(&0), "committed clean segment 0 must be released"); + assert!(!cache.segments.contains_key(&1), "committed clean segment 1 must be released"); + assert!(cache.segments.contains_key(&2), "frontier segment must stay resident"); + assert_eq!(cache.segments.len(), 1); + + // The cache-level watermarks are unaffected by residency. + assert_eq!(cache.start_height(), Some(0)); + assert_eq!(cache.tip_height(), Some(ITEMS_PER_SEGMENT * 2 + 4)); + + // Reading the released range reloads both segments from disk and + // returns exactly the bytes that were written. + let reread = cache.get_items(0..ITEMS_PER_SEGMENT * 2).await.unwrap(); + assert_eq!(reread, items[0..(ITEMS_PER_SEGMENT * 2) as usize]); + + // Single-item reads across the boundary agree too. + assert_eq!(cache.get_item(0).await.unwrap(), Some(items[0])); + assert_eq!( + cache.get_item(ITEMS_PER_SEGMENT).await.unwrap(), + Some(items[ITEMS_PER_SEGMENT as usize]) + ); + } + + /// Never release a segment that is dirty (its items exist only in memory), + /// the frontier segment (still being written), or a segment only partially + /// below the watermark. + #[tokio::test] + async fn test_release_skips_dirty_frontier_and_partial_segments() { + let tmp_dir = TempDir::new().unwrap(); + + const ITEMS_PER_SEGMENT: u32 = Segment::::ITEMS_PER_SEGMENT; + + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + + // Sparse writes into segments 0, 1 and 2. + cache.store_items_at_height(&FilterHeader::dummy_batch(0..1), 10).await.unwrap(); + cache + .store_items_at_height(&FilterHeader::dummy_batch(1..2), ITEMS_PER_SEGMENT + 10) + .await + .unwrap(); + cache + .store_items_at_height(&FilterHeader::dummy_batch(2..3), ITEMS_PER_SEGMENT * 2 + 10) + .await + .unwrap(); + + // A watermark above everything, but nothing has been persisted yet, so + // all three segments are Dirty and none may be released. + cache.set_committed_height(u32::MAX); + let released = cache.release_committed_segments(); + assert_eq!(released, 0, "dirty segments must never be released"); + assert_eq!(cache.segments.len(), 3); + + cache.persist(tmp_dir.path()).await; + // persist() cleaned all three, then released every non-frontier one. + assert_eq!(cache.segments.keys().copied().collect::>(), vec![2]); + + // Now verify the partial-coverage rule: a watermark inside segment 0 + // does not release it, because its upper heights are not yet committed. + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + let _ = cache.get_segment_mut(&0).await.unwrap(); + assert!(cache.segments.contains_key(&0)); + + cache.set_committed_height(ITEMS_PER_SEGMENT - 2); // one short of the segment's last height + assert_eq!(cache.release_committed_segments(), 0, "partially committed segment must stay"); + assert!(cache.segments.contains_key(&0)); + + cache.set_committed_height(ITEMS_PER_SEGMENT - 1); // exactly the last height + assert_eq!(cache.release_committed_segments(), 1, "fully committed segment is releasable"); + assert!(!cache.segments.contains_key(&0)); + } + + /// The rescan path re-reads heights far below the watermark. Those segments + /// were released, so this exercises the lazy reload under the access + /// pattern `reset_for_rescan` / `start_download` produce. + #[tokio::test] + async fn test_released_segments_serve_a_rescan_reread() { + let tmp_dir = TempDir::new().unwrap(); + + const ITEMS_PER_SEGMENT: u32 = Segment::::ITEMS_PER_SEGMENT; + + let items = FilterHeader::dummy_batch(0..ITEMS_PER_SEGMENT * 2 + 5); + + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + cache.store_items_at_height(&items, 0).await.unwrap(); + cache.set_committed_height(ITEMS_PER_SEGMENT * 2 - 1); + cache.persist(tmp_dir.path()).await; + assert_eq!(cache.segments.len(), 1); + + // A wallet appears behind the scan: the manager rolls the watermark + // back and re-reads from a low height. + cache.set_committed_height(0); + let rescanned = cache.get_items(5..ITEMS_PER_SEGMENT + 5).await.unwrap(); + assert_eq!(rescanned, items[5..(ITEMS_PER_SEGMENT + 5) as usize]); + + // With the watermark rolled back, the re-read segments are retained + // rather than being dropped again underneath the rescan. + cache.persist(tmp_dir.path()).await; + assert!(cache.segments.contains_key(&0), "rolled-back watermark must stop re-release"); + + // And the data still round-trips after a full reopen. + let mut reloaded = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + assert_eq!(reloaded.tip_height(), Some(ITEMS_PER_SEGMENT * 2 + 4)); + assert_eq!( + reloaded.get_items(0..ITEMS_PER_SEGMENT * 2 + 5).await.unwrap(), + items, + "released-then-reloaded data must survive a process restart byte-identically" + ); + } + + /// The watermark must never outlive the data it refers to: `truncate_above` + /// clamps it and `clear` drops it, so a later scan of the same range is not + /// released out from under itself. + #[tokio::test] + async fn test_committed_height_follows_truncate_and_clear() { + let tmp_dir = TempDir::new().unwrap(); + + let items = FilterHeader::dummy_batch(0..30); + + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + cache.store_items_at_height(&items, 0).await.unwrap(); + + cache.set_committed_height(25); + assert_eq!(cache.committed_height(), Some(25)); + + // Truncating above a height below the watermark clamps it. + cache.truncate_above(10).await.unwrap(); + assert_eq!(cache.committed_height(), Some(10)); + + // Truncating above the watermark leaves it alone. + cache.set_committed_height(5); + cache.truncate_above(8).await.unwrap(); + assert_eq!(cache.committed_height(), Some(5)); + + cache.clear().unwrap(); + assert_eq!(cache.committed_height(), None, "an empty cache has nothing committed"); + } + + /// A released segment must still accept new writes into its unused slots, + /// reloading the existing contents first rather than silently starting from + /// a blank segment (which would drop the persisted items on the next write). + #[tokio::test] + async fn test_store_into_released_segment_preserves_existing_items() { + let tmp_dir = TempDir::new().unwrap(); + + const ITEMS_PER_SEGMENT: u32 = Segment::::ITEMS_PER_SEGMENT; + + let mut cache = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + cache.store_items_at_height(&FilterHeader::dummy_batch(0..10), 0).await.unwrap(); + cache + .store_items_at_height(&FilterHeader::dummy_batch(50..55), ITEMS_PER_SEGMENT) + .await + .unwrap(); + + cache.set_committed_height(ITEMS_PER_SEGMENT - 1); + cache.persist(tmp_dir.path()).await; + assert!(!cache.segments.contains_key(&0), "segment 0 released"); + + // Write into a gap in the released segment. + cache.store_items_at_height(&FilterHeader::dummy_batch(90..95), 100).await.unwrap(); + + // Both the reloaded originals and the new items are present. + assert_eq!(cache.get_items(0..10).await.unwrap(), FilterHeader::dummy_batch(0..10)); + assert_eq!(cache.get_items(100..105).await.unwrap(), FilterHeader::dummy_batch(90..95)); + + cache.persist(tmp_dir.path()).await; + let mut reloaded = SegmentCache::::load_or_new(tmp_dir.path()).await.unwrap(); + assert_eq!(reloaded.get_items(0..10).await.unwrap(), FilterHeader::dummy_batch(0..10)); + assert_eq!(reloaded.get_items(100..105).await.unwrap(), FilterHeader::dummy_batch(90..95)); + } + #[test] fn test_segment_insert_get() { let segment_id = 10; diff --git a/dash-spv/src/sync/block_headers/manager.rs b/dash-spv/src/sync/block_headers/manager.rs index 713699f7c..6cefe07f2 100644 --- a/dash-spv/src/sync/block_headers/manager.rs +++ b/dash-spv/src/sync/block_headers/manager.rs @@ -19,6 +19,7 @@ use crate::sync::block_headers::HeadersPipeline; use crate::sync::{BlockHeadersProgress, ProgressPercentage, SyncEvent, SyncManager, SyncState}; use crate::types::HashedBlockHeader; use crate::validation::{BlockHeaderValidator, Validator}; +#[cfg(test)] use dashcore::block::Header; use dashcore::network::message_blockdata::Inventory; use dashcore::BlockHash; @@ -108,7 +109,7 @@ impl BlockHeadersManager { BlockHeaderValidator::new().validate(headers)?; // Store headers - self.header_storage.write().await.store_hashed_headers(headers).await?; + self.header_storage.write().await.store_headers(headers).await?; let tip = self.tip().await?; @@ -122,7 +123,7 @@ impl BlockHeadersManager { /// Handle incoming headers message (used for both initial sync and post-sync). pub(super) async fn handle_headers_pipeline( &mut self, - headers: &[Header], + headers: &[HashedBlockHeader], requests: &RequestSender, ) -> SyncResult> { if !self.pipeline.is_initialized() { @@ -140,12 +141,16 @@ impl BlockHeadersManager { if matched.is_none() && !headers.is_empty() { tracing::debug!( "Headers not matched by pipeline (prev_hash: {}), may be post-sync update", - headers[0].prev_blockhash + headers[0].header().prev_blockhash ); } - // Send more requests during initial sync or active post-sync catch-up. - // Skip for unsolicited headers. + // Process ready-to-store segments + let mut events = Vec::new(); + let ready_batches = self.pipeline.take_ready_to_store(); + + // Draining can expose new segments at the end of the active window, so + // refill only after next_to_store has advanced. Skip unsolicited headers. if was_syncing || !tip_was_complete { let sent = self.pipeline.send_pending(requests)?; if sent > 0 { @@ -153,9 +158,44 @@ impl BlockHeadersManager { } } - // Process ready-to-store segments + events.extend(self.store_ready_batches(ready_batches).await?); + + // After storing unsolicited post-sync headers, mark the tip complete so the next header goes through + // the clean reset path. Don't mark complete during active catch-up. + if !was_syncing && tip_was_complete && !events.is_empty() { + self.pipeline.mark_tip_complete(); + } + + if was_syncing { + events.extend(self.finalize_sync_if_complete(requests).await?); + } + + if matched.is_some() { + self.progress.bump_last_activity(); + } + Ok(events) + } + + /// Write segments that finished downloading into storage. + /// + /// Split out of [`Self::handle_headers_pipeline`] so `tick` can promote + /// headers too. Promotion used to be reachable only from there — i.e. only + /// when a `Headers` message arrived — which leaves a sync no way to finish + /// itself: once the last segment completes, no further `Headers` will ever + /// come, and the already-downloaded, already-validated tail simply stays in + /// memory. A testnet restore was found frozen exactly there, with 1,046,289 + /// headers buffered and the stored tip stuck at 1,474,000 while peers + /// stayed connected and chain locks kept arriving. + /// + /// Takes the batches rather than draining them itself, so callers keep + /// `take_ready_to_store` → `send_pending` → store ordering: draining can + /// expose a new segment at the end of the active window, and the refill + /// should pick it up in the same pass. + pub(super) async fn store_ready_batches( + &mut self, + ready_batches: Vec<(u32, Vec)>, + ) -> SyncResult> { let mut events = Vec::new(); - let ready_batches = self.pipeline.take_ready_to_store(); for (_start_height, batch_headers) in ready_batches { if !batch_headers.is_empty() { @@ -185,37 +225,42 @@ impl BlockHeadersManager { } } - // After storing unsolicited post-sync headers, mark the tip complete so the next header goes through - // the clean reset path. Don't mark complete during active catch-up. - if !was_syncing && tip_was_complete && !events.is_empty() { - self.pipeline.mark_tip_complete(); - } + Ok(events) + } - if was_syncing && self.pipeline.is_complete() { - // If blocks were announced during sync, request them before finalizing the sync - if !self.pending_announcements.is_empty() { - tracing::info!( - "Pipeline complete but {} blocks announced during sync, requesting headers", - self.pending_announcements.len() - ); - self.pipeline.reset_tip_segment(); - self.pipeline.send_pending(requests)?; - } else { - // Synced to the tip and no pending announcements, finalize and emit event - let tip = self.tip().await?; - self.progress.update_target_height(tip.height()); - self.progress.set_state(SyncState::Synced); - tracing::info!("Headers sync complete at height {}", tip.height()); - events.push(SyncEvent::BlockHeaderSyncComplete { - tip_height: tip.height(), - }); - } + /// Close out an initial sync whose pipeline has nothing left to download. + /// + /// Extracted alongside [`Self::store_ready_batches`] and for the same + /// reason: a promotion driven by `tick` has to be able to reach the + /// completion it just made possible, or the manager stores the last + /// segment and then sits in `Syncing` forever. + pub(super) async fn finalize_sync_if_complete( + &mut self, + requests: &RequestSender, + ) -> SyncResult> { + if !self.pipeline.is_complete() { + return Ok(Vec::new()); } - if matched.is_some() { - self.progress.bump_last_activity(); + // If blocks were announced during sync, request them before finalizing the sync + if !self.pending_announcements.is_empty() { + tracing::info!( + "Pipeline complete but {} blocks announced during sync, requesting headers", + self.pending_announcements.len() + ); + self.pipeline.reset_tip_segment(); + self.pipeline.send_pending(requests)?; + return Ok(Vec::new()); } - Ok(events) + + // Synced to the tip and no pending announcements, finalize and emit event + let tip = self.tip().await?; + self.progress.update_target_height(tip.height()); + self.progress.set_state(SyncState::Synced); + tracing::info!("Headers sync complete at height {}", tip.height()); + Ok(vec![SyncEvent::BlockHeaderSyncComplete { + tip_height: tip.height(), + }]) } /// Handle inventory announcements for new blocks. @@ -262,6 +307,8 @@ mod tests { use crate::storage::{ DiskStorageManager, PersistentBlockHeaderStorage, PersistentMetadataStorage, StorageManager, }; + use crate::sync::block_headers::pipeline::ACTIVE_SEGMENT_WINDOW; + use crate::sync::block_headers::segment_state::SegmentState; use crate::sync::{ManagerIdentifier, SyncManager, SyncManagerProgress}; use dashcore::network::message::NetworkMessage; use tokio::sync::mpsc::unbounded_channel; @@ -277,7 +324,12 @@ mod tests { let mut storage = DiskStorageManager::with_temp_dir().await.unwrap(); // Store a genesis header so the manager can initialize let genesis = Header::dummy_batch(0..1); - storage.store_headers(&genesis).await.unwrap(); + storage + .store_headers( + &genesis.iter().map(crate::types::HashedBlockHeader::from).collect::>(), + ) + .await + .unwrap(); let checkpoint_manager = create_test_checkpoint_manager(); BlockHeadersManager::new(storage.block_headers(), storage.metadata(), checkpoint_manager) .await @@ -293,6 +345,80 @@ mod tests { manager } + /// Headers that finished downloading must reach storage even if no further + /// `Headers` message ever arrives. + /// + /// Promotion used to hang off `handle_headers_pipeline` alone, so the last + /// segment of a sync had nothing left to trigger it — the pipeline sat on + /// fully downloaded, fully validated headers forever. Seen in the field on + /// a testnet restore: 1,046,289 headers buffered, stored tip frozen at + /// 1,474,000, peers healthy, chain locks still arriving. + /// + /// Asserts the completion too, not just the store: a regression that keeps + /// the last segment but leaves the manager in `Syncing` is the same stall + /// wearing a different hat. + #[tokio::test] + async fn test_tick_promotes_buffered_headers_with_no_further_messages() { + let mut manager = create_test_manager().await; + let tip = manager.tip().await.unwrap(); + let start_height = tip.height(); + + manager.pipeline.init(start_height, *tip.hash(), start_height + 2); + manager.progress.set_state(SyncState::Syncing); + + let (sender, _rx) = create_test_request_sender(); + // Issue the request the segment expects, so the headers below are a + // legitimate answer to it rather than unsolicited. + manager.pipeline.send_pending(&sender).unwrap(); + + // The headers land in the pipeline without the manager's own + // message-driven promotion running — the state the field stall was in. + let headers: Vec = + Header::dummy_chain(2, *tip.hash()).iter().map(HashedBlockHeader::from).collect(); + manager.pipeline.receive_headers(&headers).unwrap(); + assert!( + manager.pipeline.total_buffered() > 0, + "precondition: headers are downloaded but not yet promoted" + ); + assert_eq!( + manager.tip().await.unwrap().height(), + start_height, + "precondition: storage has not advanced" + ); + + // No further `Headers` will arrive. The periodic tick is the only + // thing left that can finish the job. + let events = manager.tick(&sender).await.unwrap(); + + assert!( + manager.tip().await.unwrap().height() > start_height, + "tick must promote buffered headers into storage" + ); + assert!( + events.iter().any(|e| matches!(e, SyncEvent::BlockHeadersStored { .. })), + "the promotion must be reported, not done silently" + ); + + // Completing the pipeline must also be reachable from a tick: an empty + // response closes the tip segment, and the next tick has to announce + // the sync rather than leaving the manager in `Syncing` forever. + manager.pipeline.send_pending(&sender).unwrap(); + manager.pipeline.receive_headers(&[]).unwrap(); + assert!(manager.pipeline.is_complete(), "precondition: nothing left to download"); + + let events = manager.tick(&sender).await.unwrap(); + + assert_eq!( + manager.state(), + SyncState::Synced, + "a tick over a complete pipeline must finish the sync" + ); + assert!( + events.iter().any(|e| matches!(e, SyncEvent::BlockHeaderSyncComplete { .. })), + "completion must be announced, or downstream managers never start" + ); + } + #[tokio::test] async fn test_block_headers_manager_new() { let manager = create_test_manager().await; @@ -334,7 +460,7 @@ mod tests { } #[tokio::test] - async fn test_unsolicited_post_sync_header_does_not_trigger_get_headers() { + async fn test_unsolicited_post_sync_header_batch_does_not_trigger_get_headers() { let mut manager = create_test_manager().await; let tip = manager.tip().await.unwrap(); let tip_hash = *tip.hash(); @@ -346,16 +472,19 @@ mod tests { let (sender, mut rx) = create_test_request_sender(); - let header = Header::dummy_chain(1, tip_hash).remove(0); + let headers = Header::dummy_chain(2, tip_hash) + .into_iter() + .map(HashedBlockHeader::from) + .collect::>(); - let events = manager.handle_headers_pipeline(&[header], &sender).await.unwrap(); + let events = manager.handle_headers_pipeline(&headers, &sender).await.unwrap(); // Header should have been stored assert_eq!(events.len(), 1); assert!(matches!( events[0], SyncEvent::BlockHeadersStored { - tip_height: 1 + tip_height: 2 } )); @@ -366,6 +495,61 @@ mod tests { assert!(manager.pipeline.is_tip_complete()); } + #[tokio::test] + async fn test_response_cycle_refills_window_after_ordered_drain() { + let mut manager = create_test_manager().await; + let stored_tip = manager.tip().await.unwrap(); + let chain = Header::dummy_chain(ACTIVE_SEGMENT_WINDOW, *stored_tip.hash()); + + let mut segments = Vec::new(); + for (id, header) in chain.iter().enumerate() { + let start_hash = if id == 0 { + *stored_tip.hash() + } else { + chain[id - 1].block_hash() + }; + let target_hash = header.block_hash(); + let mut segment = SegmentState::new( + id, + id as u32, + start_hash, + Some(id as u32 + 1), + Some(target_hash), + ); + if id == 0 { + segment.coordinator.mark_sent(&[start_hash]); + } else { + segment.current_tip_hash = target_hash; + segment.current_height = id as u32 + 1; + segment.complete = true; + segment.buffered_headers.push((*header).into()); + } + segments.push(segment); + } + let next_locator = chain.last().expect("non-empty chain").block_hash(); + segments.push(SegmentState::new( + ACTIVE_SEGMENT_WINDOW, + ACTIVE_SEGMENT_WINDOW as u32, + next_locator, + None, + None, + )); + manager.pipeline.set_segments_for_test(segments); + manager.progress.set_state(SyncState::Syncing); + + let (requests, mut rx) = create_test_request_sender(); + let events = manager.handle_headers_pipeline(&[chain[0].into()], &requests).await.unwrap(); + + assert_eq!(events.len(), ACTIVE_SEGMENT_WINDOW); + match rx.try_recv().expect("response cycle did not refill active window") { + NetworkRequest::SendMessage(NetworkMessage::GetHeaders(request)) => { + assert_eq!(request.locator_hashes[0], next_locator); + } + other => panic!("Expected GetHeaders, got {other:?}"), + } + assert!(rx.try_recv().is_err()); + } + #[tokio::test] async fn test_peer_tip_announcement_lifecycle() { let mut manager = create_synced_manager().await; @@ -457,7 +641,7 @@ mod tests { // segment's current_tip_hash to advanced_hash. let header = Header::dummy_chain(1, initial_locator).remove(0); let advanced_hash = header.block_hash(); - manager.handle_headers_pipeline(&[header], &requests).await.unwrap(); + manager.handle_headers_pipeline(&[header.into()], &requests).await.unwrap(); // Drain the follow-up GetHeaders that send_pending issued. match rx.try_recv().expect("follow-up GetHeaders not sent") { diff --git a/dash-spv/src/sync/block_headers/pipeline.rs b/dash-spv/src/sync/block_headers/pipeline.rs index cce5baca9..286816ddf 100644 --- a/dash-spv/src/sync/block_headers/pipeline.rs +++ b/dash-spv/src/sync/block_headers/pipeline.rs @@ -6,6 +6,7 @@ use std::sync::Arc; +#[cfg(test)] use dashcore::block::Header; use dashcore::BlockHash; @@ -15,6 +16,9 @@ use crate::network::RequestSender; use crate::sync::block_headers::segment_state::SegmentState; use crate::types::HashedBlockHeader; +/// Keep enough checkpoint segments active to saturate decompression while bounding buffered data. +pub(super) const ACTIVE_SEGMENT_WINDOW: usize = 8; + /// Pipeline for parallel header downloads across checkpoint-defined segments. /// /// Divides the blockchain into segments based on checkpoints and downloads @@ -121,7 +125,8 @@ impl HeadersPipeline { /// Returns the number of requests sent. pub fn send_pending(&mut self, requests: &RequestSender) -> SyncResult { let mut sent = 0; - for segment in &mut self.segments { + let window_end = (self.next_to_store + ACTIVE_SEGMENT_WINDOW).min(self.segments.len()); + for segment in &mut self.segments[self.next_to_store..window_end] { // Skip completed segments if segment.complete { continue; @@ -137,7 +142,7 @@ impl HeadersPipeline { /// Try to match incoming headers to the correct segment. /// Returns the segment index if matched, or None if headers don't belong to any segment. /// Returns an error if checkpoint validation fails. - pub fn receive_headers(&mut self, headers: &[Header]) -> SyncResult> { + pub fn receive_headers(&mut self, headers: &[HashedBlockHeader]) -> SyncResult> { if headers.is_empty() { // Empty response means the peer has no more headers after our locator. // Route to the tip segment (target_height is None) if it has in-flight requests. @@ -159,7 +164,7 @@ impl HeadersPipeline { return Ok(None); } - let prev_hash = headers[0].prev_blockhash; + let prev_hash = headers[0].header().prev_blockhash; // Find the segment that matches for (idx, segment) in self.segments.iter_mut().enumerate() { @@ -175,7 +180,9 @@ impl HeadersPipeline { if segment.complete && segment.target_height.is_none() { segment.complete = false; self.next_to_store = idx; - // Mark as in-flight so the coordinator accepts these unsolicited headers + // A headers announcement may contain multiple consecutive headers. The + // coordinator tracks the response by its first previous hash, just like a + // requested headers batch. segment.coordinator.mark_sent(&[prev_hash]); tracing::debug!( "Tip segment {} receiving post-sync headers, reset for continued processing", @@ -189,7 +196,7 @@ impl HeadersPipeline { // Check if these are duplicate headers from another peer. The first // header's hash matches a segment's current tip, meaning we already have it. - let first_hash = headers[0].block_hash(); + let first_hash = *headers[0].hash(); if self.segments.iter().any(|s| s.current_tip_hash == first_hash) { tracing::debug!("Ignoring duplicate header {} from another peer", first_hash); return Ok(None); @@ -333,6 +340,13 @@ impl HeadersPipeline { .find(|s| s.target_height.is_none()) .is_some_and(|s| !s.complete && s.coordinator.active_count() > 0) } + + #[cfg(test)] + pub(super) fn set_segments_for_test(&mut self, segments: Vec) { + self.segments = segments; + self.next_to_store = 0; + self.initialized = true; + } } #[cfg(test)] @@ -409,8 +423,14 @@ mod tests { let sent = pipeline.send_pending(&sender).unwrap(); - // Should send at least one request per segment - assert!(sent >= pipeline.segment_count()); + assert_eq!(sent, ACTIVE_SEGMENT_WINDOW.min(pipeline.segment_count())); + + assert!(pipeline.segments[..sent] + .iter() + .all(|segment| segment.coordinator.active_count() == 1)); + assert!(pipeline.segments[sent..] + .iter() + .all(|segment| segment.coordinator.active_count() == 0)); // Verify messages were queued let mut count = 0; @@ -418,6 +438,16 @@ mod tests { count += 1; } assert_eq!(count, sent); + + if pipeline.segment_count() > ACTIVE_SEGMENT_WINDOW { + pipeline.segments[0].complete = true; + pipeline.next_to_store = 1; + + assert_eq!(pipeline.send_pending(&sender).unwrap(), 1); + assert_eq!(pipeline.segments[ACTIVE_SEGMENT_WINDOW].coordinator.active_count(), 1); + assert!(rx.try_recv().is_ok()); + assert!(rx.try_recv().is_err()); + } } #[test] @@ -464,7 +494,7 @@ mod tests { let mut header = Header::dummy(1); header.prev_blockhash = tip_hash; - let matched = pipeline.receive_headers(&[header]).unwrap(); + let matched = pipeline.receive_headers(&[header.into()]).unwrap(); assert_eq!(matched, Some(0), "Tip segment should accept unsolicited post-sync headers"); assert!(!pipeline.segments[0].complete, "Tip segment should be reset to non-complete"); @@ -472,6 +502,76 @@ mod tests { assert_eq!(pipeline.segments[0].current_height, 1001); } + #[test] + fn test_completed_tip_accepts_unsolicited_header_batch() { + let tip_hash = BlockHash::dummy(99); + let mut tip = SegmentState::new(0, 1000, tip_hash, None, None); + tip.complete = true; + + let cm = create_test_checkpoint_manager(true); + let mut pipeline = HeadersPipeline::new(cm); + pipeline.initialized = true; + pipeline.next_to_store = 1; + pipeline.segments = vec![tip]; + + let headers = Header::dummy_chain(2, tip_hash) + .into_iter() + .map(HashedBlockHeader::from) + .collect::>(); + let matched = pipeline.receive_headers(&headers).unwrap(); + + assert_eq!(matched, Some(0)); + assert!(!pipeline.segments[0].complete); + assert_eq!(pipeline.segments[0].current_tip_hash, *headers[1].hash()); + assert_eq!(pipeline.segments[0].current_height, 1002); + assert_eq!(pipeline.segments[0].buffered_headers, headers); + assert_eq!(pipeline.segments[0].coordinator.active_count(), 0); + assert_eq!(pipeline.next_to_store, 0); + } + + #[test] + fn test_draining_active_window_exposes_next_segment_for_refill() { + let mut segments = Vec::new(); + for id in 0..ACTIVE_SEGMENT_WINDOW { + let start_hash = BlockHash::dummy(id as u32); + let mut segment = + SegmentState::new(id, id as u32, start_hash, Some(id as u32 + 1), None); + segment.complete = true; + segment.buffered_headers.push(Header::dummy(id as u32).into()); + segments.push(segment); + } + segments.push(SegmentState::new( + ACTIVE_SEGMENT_WINDOW, + ACTIVE_SEGMENT_WINDOW as u32, + BlockHash::dummy(ACTIVE_SEGMENT_WINDOW as u32), + None, + None, + )); + + let cm = create_test_checkpoint_manager(true); + let mut pipeline = HeadersPipeline::new(cm); + pipeline.set_segments_for_test(segments); + let (sender, mut rx) = create_test_request_sender(); + + assert_eq!(pipeline.send_pending(&sender).unwrap(), 0); + assert_eq!(pipeline.take_ready_to_store().len(), ACTIVE_SEGMENT_WINDOW); + assert_eq!(pipeline.next_to_store, ACTIVE_SEGMENT_WINDOW); + assert_eq!(pipeline.send_pending(&sender).unwrap(), 1); + + match rx.try_recv().expect("newly exposed segment was not requested") { + NetworkRequest::SendMessage( + dashcore::network::message::NetworkMessage::GetHeaders(request), + ) => { + assert_eq!( + request.locator_hashes[0], + BlockHash::dummy(ACTIVE_SEGMENT_WINDOW as u32) + ) + } + other => panic!("Expected GetHeaders, got {other:?}"), + } + assert!(rx.try_recv().is_err()); + } + #[test] fn test_completed_segment_does_not_steal_next_segment_headers() { // Create two segments which share the checkpoint hash boundary. @@ -502,7 +602,7 @@ mod tests { pipeline.segments[1].coordinator.mark_sent(&[shared_hash]); // Route headers should go to segment 1, not the completed segment 0 - let matched = pipeline.receive_headers(&[header]).unwrap(); + let matched = pipeline.receive_headers(&[header.into()]).unwrap(); assert_eq!(matched, Some(1), "Headers should route to segment 1, not completed segment 0"); // Segment 0 should still have no extra buffered headers @@ -530,7 +630,7 @@ mod tests { // Another peer sends the same header (prev_blockhash is old tip, first // header hash matches the segment's current tip) - let matched = pipeline.receive_headers(&[first_header]).unwrap(); + let matched = pipeline.receive_headers(&[first_header.into()]).unwrap(); assert_eq!(matched, None, "Duplicate headers should be silently ignored"); assert!(pipeline.segments[0].buffered_headers.is_empty()); } @@ -553,7 +653,7 @@ mod tests { mid.coordinator.mark_sent(&[shared_hash]); let mut mid_header = Header::dummy(2); mid_header.prev_blockhash = shared_hash; - mid.receive_headers(&[mid_header]).unwrap(); + mid.receive_headers(&[mid_header.into()]).unwrap(); let mid_preserved_tip = mid.current_tip_hash; let mid_preserved_height = mid.current_height; let mid_preserved_buffered = mid.buffered_headers.len(); diff --git a/dash-spv/src/sync/block_headers/segment_state.rs b/dash-spv/src/sync/block_headers/segment_state.rs index 9829d1250..58a332223 100644 --- a/dash-spv/src/sync/block_headers/segment_state.rs +++ b/dash-spv/src/sync/block_headers/segment_state.rs @@ -2,7 +2,7 @@ use crate::error::{SyncError, SyncResult}; use crate::network::RequestSender; use crate::sync::download_coordinator::{DownloadConfig, DownloadCoordinator}; use crate::types::HashedBlockHeader; -use dashcore::{BlockHash, Header}; +use dashcore::BlockHash; use std::time::Duration; /// Timeout for header requests. @@ -85,7 +85,7 @@ impl SegmentState { /// Process received headers for this segment. /// Returns the number of headers processed, or an error if checkpoint validation fails. - pub(super) fn receive_headers(&mut self, headers: &[Header]) -> SyncResult { + pub(super) fn receive_headers(&mut self, headers: &[HashedBlockHeader]) -> SyncResult { if headers.is_empty() { // Empty response means we've reached the peer's tip for this segment self.complete = true; @@ -110,7 +110,7 @@ impl SegmentState { } // Mark the request as received, reject if we never requested this hash - let prev_hash = headers[0].prev_blockhash; + let prev_hash = headers[0].header().prev_blockhash; if !self.coordinator.receive(&prev_hash) { return Err(SyncError::InvalidState(format!( "Segment {}: received unrequested headers (prev_hash {})", @@ -120,8 +120,7 @@ impl SegmentState { // Process headers let mut processed = 0; - for header in headers { - let hashed = HashedBlockHeader::from(*header); + for hashed in headers { let hash = *hashed.hash(); let height = self.current_height + processed as u32 + 1; @@ -135,7 +134,7 @@ impl SegmentState { self.segment_id, target_height ); - self.buffered_headers.push(hashed); + self.buffered_headers.push(hashed.clone()); processed += 1; self.complete = true; break; @@ -155,13 +154,13 @@ impl SegmentState { } } - self.buffered_headers.push(hashed); + self.buffered_headers.push(hashed.clone()); processed += 1; } // Update current tip for next request if processed > 0 { - self.current_tip_hash = headers[processed - 1].block_hash(); + self.current_tip_hash = *headers[processed - 1].hash(); self.current_height += processed as u32; } @@ -265,7 +264,7 @@ mod tests { let mut first = headers[0]; first.prev_blockhash = hash; - let processed = segment.receive_headers(&[first]).unwrap(); + let processed = segment.receive_headers(&[first.into()]).unwrap(); assert_eq!(processed, 1); assert_eq!(segment.buffered_headers.len(), 1); @@ -292,7 +291,7 @@ mod tests { assert_ne!(*actual_hash, expected_checkpoint_hash); // Receiving this header should fail with a validation error - let result = segment.receive_headers(&[header]); + let result = segment.receive_headers(&[header.into()]); assert!(result.is_err()); let err = result.unwrap_err(); @@ -323,7 +322,7 @@ mod tests { segment.coordinator.mark_sent(&[start_hash]); // Receiving this header should succeed and complete the segment - let result = segment.receive_headers(&[header]); + let result = segment.receive_headers(&[header.into()]); assert!(result.is_ok()); assert_eq!(result.unwrap(), 1); @@ -340,7 +339,7 @@ mod tests { let mut header = Header::dummy(1); header.prev_blockhash = start_hash; - let result = segment.receive_headers(&[header]); + let result = segment.receive_headers(&[header.into()]); assert!(result.is_err()); match result.unwrap_err() { SyncError::InvalidState(msg) => { @@ -365,7 +364,7 @@ mod tests { header.prev_blockhash = start_hash; // Completed segment should return an invalid state error - let result = segment.receive_headers(&[header]); + let result = segment.receive_headers(&[header.into()]); assert!(result.is_err()); match result.unwrap_err() { SyncError::InvalidState(msg) => { @@ -384,7 +383,7 @@ mod tests { let mut header = Header::dummy(1); header.prev_blockhash = start_hash; - segment.receive_headers(&[header]).unwrap(); + segment.receive_headers(&[header.into()]).unwrap(); let preserved_tip_hash = segment.current_tip_hash; let preserved_height = segment.current_height; diff --git a/dash-spv/src/sync/block_headers/sync_manager.rs b/dash-spv/src/sync/block_headers/sync_manager.rs index d47e2cfe8..3acb85c8b 100644 --- a/dash-spv/src/sync/block_headers/sync_manager.rs +++ b/dash-spv/src/sync/block_headers/sync_manager.rs @@ -47,6 +47,17 @@ impl SyncManager for BlockHeadersMana self.announced_peers.clear(); } + fn on_peer_disconnect(&mut self) { + // Only the active sync path reissues header requests. Dropping the + // in-flight marker in any other state would strand the request rather + // than retry it, and would also mask the tip announcement a freshly + // connected peer gets while a catch-up request is outstanding. + if self.state() != SyncState::Syncing { + return; + } + self.pipeline.clear_in_flight(); + } + async fn start_sync(&mut self, requests: &RequestSender) -> SyncResult> { ensure_not_started(self.state(), self.identifier())?; self.progress.set_state(SyncState::Syncing); @@ -93,8 +104,27 @@ impl SyncManager for BlockHeadersMana ) -> SyncResult> { match msg.inner() { NetworkMessage::Headers(headers) => { + let hashed_headers: Vec = + if let Some(hashes) = msg.header_hashes() { + if hashes.len() != headers.len() { + return Err(crate::error::SyncError::InvalidState(format!( + "headers2 hash count mismatch: {} headers, {} hashes", + headers.len(), + hashes.len() + ))); + } + headers + .iter() + .zip(hashes) + .map(|(header, hash)| { + crate::types::HashedBlockHeader::with_trusted_hash(*header, *hash) + }) + .collect() + } else { + headers.iter().map(crate::types::HashedBlockHeader::from).collect() + }; // Always route through pipeline when initialized - self.handle_headers_pipeline(headers, requests).await + self.handle_headers_pipeline(&hashed_headers, requests).await } NetworkMessage::Inv(inv) => { @@ -124,12 +154,27 @@ impl SyncManager for BlockHeadersMana // During initial sync, send more requests and log progress if self.state() == SyncState::Syncing { + // Take, refill, then store — the same order `handle_headers_pipeline` + // uses, since draining can expose a segment the refill should pick + // up in this pass rather than the next one. + let ready_batches = self.pipeline.take_ready_to_store(); + let sent = self.pipeline.send_pending(requests)?; if sent > 0 { tracing::debug!("Tick: pipeline sent {} more requests", sent); } - return Ok(vec![]); + // Promotion is otherwise reachable only from `handle_headers_pipeline`, + // i.e. only when a `Headers` message arrives — so a sync has no way to + // finish itself once the final segment completes, because no further + // `Headers` will come. That is not hypothetical: a testnet restore was + // found frozen with 1,046,289 headers buffered in memory and the stored + // tip stuck at 1,474,000, peers connected and chain locks still + // arriving, until the app was restarted. Retrying on the 100ms tick + // makes a missed promotion self-heal whatever caused it to be missed. + let mut events = self.store_ready_batches(ready_batches).await?; + events.extend(self.finalize_sync_if_complete(requests).await?); + return Ok(events); } // Post-sync: check for stale block announcements @@ -191,6 +236,14 @@ impl SyncManager for BlockHeadersMana address, } => { self.announced_peers.remove(address); + self.on_peer_disconnect(); + if self.state() == SyncState::Syncing { + // Reissue before returning to the task loop. A segment + // rejects headers whose request it no longer tracks, so + // leaving the resend to the next tick would open a window + // in which a surviving peer's reply looks unsolicited. + self.pipeline.send_pending(requests)?; + } } NetworkEvent::PeersUpdated { connected_count, diff --git a/dash-spv/src/sync/blocks/manager.rs b/dash-spv/src/sync/blocks/manager.rs index da9e01220..3c7e50d6e 100644 --- a/dash-spv/src/sync/blocks/manager.rs +++ b/dash-spv/src/sync/blocks/manager.rs @@ -3,6 +3,7 @@ //! Downloads blocks that matched wallet filters and processes them in height order. //! Subscribes to BlockNeeded events and emits BlockProcessed events. +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use tokio::sync::RwLock; @@ -12,7 +13,8 @@ use crate::error::SyncResult; use crate::network::RequestSender; use crate::storage::{BlockHeaderStorage, BlockStorage}; use crate::sync::{BlocksProgress, SyncEvent, SyncManager, SyncState}; -use key_wallet_manager::WalletInterface; +use crate::types::HashedBlock; +use key_wallet_manager::{BlockProcessingResult, WalletId, WalletInterface}; /// Blocks manager for downloading and processing matching blocks. /// @@ -77,60 +79,50 @@ impl BlocksManager SyncResult> { let mut events = Vec::new(); + // Highest height applied in this drain, used below to advance the + // storage's committed watermark exactly once. + let mut last_applied: Option = None; // Process blocks in height order using pipeline's ordering logic while let Some((block, height, interested)) = self.pipeline.take_next_ordered_block() { - let hash = block.block_hash(); + let hash = *block.hash(); // Process the block only for the wallets whose filter matched it. // Already-synced wallets that did not match are not touched. - let mut wallet = self.wallet.write().await; - let result = wallet.process_block_for_wallets(&block, height, &interested).await; - drop(wallet); - - let total_relevant = result.relevant_tx_count(); - let new_scripts_total: usize = result.new_scripts.values().map(|v| v.len()).sum(); - if total_relevant > 0 { - tracing::info!( - "Found {} relevant transactions ({} new, {} existing) {} at height {}, new scripts: {}", - total_relevant, - result.new_txids.len(), - result.existing_txids.len(), - hash, - height, - new_scripts_total - ); - } - - // Collect confirmed txids before moving new_scripts out of result - let confirmed_txids: Vec<_> = result.relevant_txids().cloned().collect(); - - // Collect new scripts for gap limit rescanning - let new_scripts = result.new_scripts; - if new_scripts_total > 0 { - tracing::debug!( - "Block {} generated {} new scripts for gap limit maintenance across {} wallets", - height, - new_scripts_total, - new_scripts.len() - ); - } + let result = self.apply_block(&block, height, &interested).await; self.progress.add_processed(1); - if total_relevant > 0 { + if result.relevant_tx_count() > 0 { self.progress.add_relevant(1); } - // Only count new transactions to avoid double-counting during rescans - self.progress.add_transactions(result.new_txids.len() as u32); + // Collect confirmed txids before moving new_scripts out of result + let confirmed_txids: Vec<_> = result.relevant_txids().cloned().collect(); self.progress.update_last_processed(height); + last_applied = Some(height); events.push(SyncEvent::BlockProcessed { block_hash: hash, height, wallets: interested, - new_scripts, + new_scripts: result.new_scripts, confirmed_txids, }); + + self.reapply_blocks(result.reapply_heights).await; + } + + // Blocks are drained in strict height order, so `last_applied` is the + // high-water mark of blocks now applied to every interested wallet. + // Telling storage lets it release those block bodies from memory on the + // next persist instead of pinning the whole backfill; they stay + // readable via `load_block`, which reloads them from disk. + // + // Set once per drain rather than per block: this takes the block + // storage write lock, and the loop above can run thousands of times. + // No other guard is held here, and the wallet lock inside the loop was + // dropped before this point. + if let Some(height) = last_applied { + self.block_storage.write().await.set_committed_height(height).await; } // Check if pipeline is empty @@ -150,6 +142,70 @@ impl BlocksManager, + ) -> BlockProcessingResult { + let hash = *block.hash(); + let mut wallet = self.wallet.write().await; + let result = wallet.process_block_for_wallets(block.block(), hash, height, wallets).await; + drop(wallet); + + let total_relevant = result.relevant_tx_count(); + let new_scripts_total: usize = result.new_scripts.values().map(|v| v.len()).sum(); + if total_relevant > 0 { + tracing::info!( + "Found {} relevant transactions ({} new, {} existing) {} at height {}, new scripts: {}", + total_relevant, + result.new_txids.len(), + result.existing_txids.len(), + hash, + height, + new_scripts_total + ); + } + if new_scripts_total > 0 { + tracing::debug!( + "Block {} generated {} new scripts for gap limit maintenance across {} wallets", + height, + new_scripts_total, + result.new_scripts.len() + ); + } + + // Only count new transactions to avoid double-counting during rescans + self.progress.add_transactions(result.new_txids.len() as u32); + result + } + + async fn reapply_blocks(&mut self, reapply: BTreeMap>) { + let mut queue: BTreeSet<(u32, WalletId)> = reapply + .into_iter() + .flat_map(|(wallet_id, heights)| heights.into_iter().map(move |h| (h, wallet_id))) + .collect(); + while let Some((height, wallet_id)) = queue.pop_first() { + let block = match self.block_storage.read().await.load_block(height).await { + Ok(Some(block)) => block, + Ok(None) => { + tracing::warn!("Cannot re-apply block at height {}: not in storage", height); + continue; + } + Err(e) => { + tracing::warn!("Cannot re-apply block at height {}: {}", height, e); + continue; + } + }; + let result = self.apply_block(&block, height, &BTreeSet::from([wallet_id])).await; + queue.extend( + result.reapply_heights.into_iter().flat_map(|(wallet_id, heights)| { + heights.into_iter().map(move |h| (h, wallet_id)) + }), + ); + } + } } impl std::fmt::Debug @@ -166,15 +222,20 @@ impl std::fmt::Debug #[cfg(test)] mod tests { use super::*; - use crate::network::{MessageType, NetworkManager}; + use crate::network::{MessageType, NetworkEvent, NetworkManager, NetworkRequest}; use crate::storage::{ DiskStorageManager, PersistentBlockHeaderStorage, PersistentBlockStorage, StorageManager, }; use crate::sync::{ManagerIdentifier, SyncEvent, SyncManagerProgress}; - use crate::test_utils::MockNetworkManager; + use crate::test_utils::{test_socket_address, MockNetworkManager}; + use crate::types::HashedBlock; + use dashcore::network::message::NetworkMessage; + use dashcore::network::message_blockdata::Inventory; + use dashcore::BlockHash; use key_wallet_manager::test_utils::{MockWallet, MOCK_WALLET_ID}; - use key_wallet_manager::FilterMatchKey; + use key_wallet_manager::{FilterMatchKey, WalletId}; use std::collections::{BTreeMap, BTreeSet}; + use tokio::sync::mpsc; type TestBlocksManager = BlocksManager; @@ -232,6 +293,61 @@ mod tests { assert!(events.is_empty()); } + /// Losing one peer of several must put that peer's `getdata` back on the + /// queue right away instead of waiting out the block download timeout. + #[tokio::test] + async fn test_peer_disconnect_reissues_in_flight_block_requests() { + let mut manager = create_test_manager().await; + let (tx, mut rx) = mpsc::unbounded_channel(); + let requests = RequestSender::new(tx); + + let blocks: BTreeMap> = (100..103) + .map(|height| { + ( + FilterMatchKey::new(height, BlockHash::dummy(height)), + BTreeSet::from([MOCK_WALLET_ID]), + ) + }) + .collect(); + let event = SyncEvent::BlocksNeeded { + blocks, + }; + manager.handle_sync_event(&event, &requests).await.unwrap(); + + let requested = drain_requested_blocks(&mut rx); + assert_eq!(requested.len(), 3); + + let disconnect = NetworkEvent::PeerDisconnected { + address: test_socket_address(1), + }; + manager.handle_network_event(&disconnect, &requests).await.unwrap(); + + manager.tick(&requests).await.unwrap(); + assert_eq!(drain_requested_blocks(&mut rx), requested); + } + + fn drain_requested_blocks( + rx: &mut mpsc::UnboundedReceiver, + ) -> BTreeSet { + let mut hashes = BTreeSet::new(); + while let Ok(request) = rx.try_recv() { + match request { + NetworkRequest::SendMessage(NetworkMessage::GetData(inventory)) => { + for item in inventory { + match item { + Inventory::Block(hash) => { + hashes.insert(hash); + } + other => panic!("Expected a block inventory item, got {:?}", other), + } + } + } + other => panic!("Expected GetData, got {:?}", other), + } + } + hashes + } + /// `process_buffered_blocks` must call `process_block_for_wallets` with /// the exact wallet set carried in the pipeline so already-synced /// wallets are not touched by routing logic. @@ -256,7 +372,11 @@ mod tests { header, txdata: vec![], }; - manager.pipeline.add_from_storage(block.clone(), 100, BTreeSet::from([MOCK_WALLET_ID])); + manager.pipeline.add_from_storage( + HashedBlock::from(&block), + 100, + BTreeSet::from([MOCK_WALLET_ID]), + ); let events = manager.process_buffered_blocks().await.unwrap(); assert!(matches!(events.first(), Some(SyncEvent::BlockProcessed { .. }))); @@ -270,6 +390,41 @@ mod tests { assert_eq!(processed[0].1, 100); } + #[tokio::test] + async fn test_process_buffered_blocks_reapplies_requested_stored_block() { + let storage = DiskStorageManager::with_temp_dir().await.unwrap(); + let mut wallet = MockWallet::new(); + wallet.set_reapply_heights(100, BTreeSet::from([200])); + let wallet = Arc::new(RwLock::new(wallet)); + let mut manager: TestBlocksManager = + BlocksManager::new(wallet.clone(), storage.block_headers(), storage.blocks()).await; + manager.progress.set_state(SyncState::Syncing); + + manager + .block_storage + .write() + .await + .store_block(200, HashedBlock::dummy(200, vec![])) + .await + .unwrap(); + manager.pipeline.add_from_storage( + HashedBlock::dummy(100, vec![]), + 100, + BTreeSet::from([MOCK_WALLET_ID]), + ); + + let events = manager.process_buffered_blocks().await.unwrap(); + assert_eq!( + events.iter().filter(|e| matches!(e, SyncEvent::BlockProcessed { .. })).count(), + 1 + ); + + let processed = wallet.read().await.processed_blocks(); + let heights: Vec = processed.lock().await.iter().map(|(_, h)| *h).collect(); + assert_eq!(heights, vec![100, 200]); + assert_eq!(manager.progress.processed(), 1); + } + /// A wallet that is NOT in the pipeline's interested set must not be /// routed the block. Two wallets are registered, but only `wallet_in` /// appears in the routed set; the other wallet's processed log must @@ -312,7 +467,11 @@ mod tests { txdata: vec![], }; // Only wallet_in is in the routed set. - manager.pipeline.add_from_storage(block.clone(), 100, BTreeSet::from([wallet_in])); + manager.pipeline.add_from_storage( + HashedBlock::from(&block), + 100, + BTreeSet::from([wallet_in]), + ); let _ = manager.process_buffered_blocks().await.unwrap(); @@ -357,7 +516,11 @@ mod tests { }; // Already-downloaded block sitting in the pipeline. - manager.pipeline.add_from_storage(block.clone(), 200, BTreeSet::from([MOCK_WALLET_ID])); + manager.pipeline.add_from_storage( + HashedBlock::from(&block), + 200, + BTreeSet::from([MOCK_WALLET_ID]), + ); manager.on_disconnect(); diff --git a/dash-spv/src/sync/blocks/pipeline.rs b/dash-spv/src/sync/blocks/pipeline.rs index a8c4edcb1..8610d33b8 100644 --- a/dash-spv/src/sync/blocks/pipeline.rs +++ b/dash-spv/src/sync/blocks/pipeline.rs @@ -9,7 +9,7 @@ use std::time::Duration; use crate::error::SyncResult; use crate::network::RequestSender; use crate::sync::download_coordinator::{DownloadConfig, DownloadCoordinator}; -use dashcore::blockdata::block::Block; +use crate::types::HashedBlock; use dashcore::BlockHash; use key_wallet_manager::{FilterMatchKey, WalletId}; @@ -17,7 +17,12 @@ use key_wallet_manager::{FilterMatchKey, WalletId}; const MAX_CONCURRENT_BLOCK_DOWNLOADS: usize = 20; /// Timeout for block downloads before retry. -const BLOCK_TIMEOUT: Duration = Duration::from_secs(30); +/// +/// A filter batch cannot commit while any block it matched is outstanding, so +/// this is the price of a single peer dropping a request, paid with filter sync +/// stopped. It stays comfortably above the transfer time of a full block on a +/// slow link, but no longer treats a peer that will never answer as merely slow. +const BLOCK_TIMEOUT: Duration = Duration::from_secs(15); /// Maximum blocks per GetData request, kept a bit lower for better download distribution to multiple peers const BLOCKS_PER_REQUEST: usize = 8; @@ -32,8 +37,8 @@ pub(super) struct BlocksPipeline { coordinator: DownloadCoordinator, /// Heights queued or in-flight (waiting for download). pending_heights: BTreeSet, - /// Downloaded blocks ready to process (height -> Block). - downloaded: BTreeMap, + /// Downloaded blocks ready to process (height -> block, with its cached hash). + downloaded: BTreeMap, /// Map hash -> height for looking up height when block arrives. hash_to_height: HashMap, /// Per-block interested wallets, populated when the block is queued. @@ -138,8 +143,8 @@ impl BlocksPipeline { /// Looks up the height from the internal hash_to_height map and stores /// the block in the downloaded buffer for height-ordered processing. /// Returns `true` if this was a tracked block, `false` if unrequested. - pub(super) fn receive_block(&mut self, block: &Block) -> bool { - let hash = block.block_hash(); + pub(super) fn receive_block(&mut self, block: &HashedBlock) -> bool { + let hash = *block.hash(); if !self.coordinator.receive(&hash) { tracing::debug!("Ignoring unrequested block: {}", hash); return false; @@ -158,7 +163,9 @@ impl BlocksPipeline { /// Returns None if: /// - No downloaded blocks available, or /// - Waiting for a lower-height block still pending - pub(super) fn take_next_ordered_block(&mut self) -> Option<(Block, u32, BTreeSet)> { + pub(super) fn take_next_ordered_block( + &mut self, + ) -> Option<(HashedBlock, u32, BTreeSet)> { let lowest_downloaded = *self.downloaded.keys().next()?; // Check if any pending blocks have lower heights @@ -169,7 +176,7 @@ impl BlocksPipeline { } let block = self.downloaded.remove(&lowest_downloaded).unwrap(); - let wallets = self.hash_to_wallets.remove(&block.block_hash()).unwrap_or_default(); + let wallets = self.hash_to_wallets.remove(block.hash()).unwrap_or_default(); Some((block, lowest_downloaded, wallets)) } @@ -178,11 +185,11 @@ impl BlocksPipeline { /// Used when blocks are already persisted from a previous sync. pub(super) fn add_from_storage( &mut self, - block: Block, + block: HashedBlock, height: u32, wallets: BTreeSet, ) { - let hash = block.block_hash(); + let hash = *block.hash(); self.hash_to_wallets.entry(hash).or_default().extend(wallets); self.downloaded.insert(height, block); } @@ -204,6 +211,7 @@ impl BlocksPipeline { #[cfg(test)] mod tests { + use dashcore::blockdata::block::Block; use dashcore_hashes::Hash; use super::*; @@ -281,11 +289,11 @@ mod tests { assert_eq!(pipeline.coordinator.active_count(), 1); // Receive block - assert!(pipeline.receive_block(&block)); + assert!(pipeline.receive_block(&HashedBlock::from(&block))); assert_eq!(pipeline.coordinator.active_count(), 0); assert_eq!(pipeline.downloaded.len(), 1); assert!(pipeline.pending_heights.is_empty()); - assert_eq!(pipeline.downloaded.get(&100).unwrap().block_hash(), hash); + assert_eq!(*pipeline.downloaded.get(&100).unwrap().hash(), hash); } #[test] @@ -293,7 +301,7 @@ mod tests { let mut pipeline = BlocksPipeline::new(); let block = make_test_block(1); - assert!(!pipeline.receive_block(&block)); + assert!(!pipeline.receive_block(&HashedBlock::from(&block))); assert!(pipeline.downloaded.is_empty()); } @@ -332,7 +340,7 @@ mod tests { assert_eq!(pipeline.coordinator.active_count(), 1); // B: already received, sitting in `downloaded` — must survive requeue. - pipeline.add_from_storage(block_b.clone(), 200, BTreeSet::from([[2u8; 32]])); + pipeline.add_from_storage(HashedBlock::from(&block_b), 200, BTreeSet::from([[2u8; 32]])); pipeline.requeue_in_flight(); @@ -385,7 +393,7 @@ mod tests { // Use add_from_storage to test ordering logic without network // Add block 2 first (out of order) - pipeline.add_from_storage(block2.clone(), 101, BTreeSet::new()); + pipeline.add_from_storage(HashedBlock::from(&block2), 101, BTreeSet::new()); // Also track height 100 as pending to simulate waiting pipeline.pending_heights.insert(100); @@ -394,17 +402,17 @@ mod tests { // Add block 1 pipeline.pending_heights.remove(&100); - pipeline.add_from_storage(block1.clone(), 100, BTreeSet::new()); + pipeline.add_from_storage(HashedBlock::from(&block1), 100, BTreeSet::new()); // Now block 1 is ready (lowest height) let (block, height, _) = pipeline.take_next_ordered_block().unwrap(); assert_eq!(height, 100); - assert_eq!(block.block_hash(), hash1); + assert_eq!(*block.hash(), hash1); // Block 2 is now ready let (block, height, _) = pipeline.take_next_ordered_block().unwrap(); assert_eq!(height, 101); - assert_eq!(block.block_hash(), hash2); + assert_eq!(*block.hash(), hash2); // No more blocks assert!(pipeline.take_next_ordered_block().is_none()); @@ -417,7 +425,7 @@ mod tests { // Add block at height 101, but height 100 is still pending pipeline.pending_heights.insert(100); - pipeline.add_from_storage(block2.clone(), 101, BTreeSet::new()); + pipeline.add_from_storage(HashedBlock::from(&block2), 101, BTreeSet::new()); // Cannot take block 2 - block at height 100 is still pending assert!(pipeline.take_next_ordered_block().is_none()); @@ -436,13 +444,13 @@ mod tests { let block = make_test_block(1); let hash = block.block_hash(); - pipeline.add_from_storage(block.clone(), 100, BTreeSet::new()); + pipeline.add_from_storage(HashedBlock::from(&block), 100, BTreeSet::new()); assert_eq!(pipeline.downloaded.len(), 1); let (taken_block, height, _) = pipeline.take_next_ordered_block().unwrap(); assert_eq!(height, 100); - assert_eq!(taken_block.block_hash(), hash); + assert_eq!(*taken_block.hash(), hash); } #[test] @@ -452,7 +460,7 @@ mod tests { // Adding to downloaded makes it incomplete let block = make_test_block(1); - pipeline.add_from_storage(block, 100, BTreeSet::new()); + pipeline.add_from_storage(HashedBlock::from(&block), 100, BTreeSet::new()); assert!(!pipeline.is_complete()); // Take the block @@ -487,10 +495,10 @@ mod tests { // Drive the block through receive_block to land it in `downloaded`. let hashes = pipeline.coordinator.take_pending(1); pipeline.coordinator.mark_sent(&hashes); - assert!(pipeline.receive_block(&block)); + assert!(pipeline.receive_block(&HashedBlock::from(&block))); let (taken_block, height, taken_wallets) = pipeline.take_next_ordered_block().unwrap(); - assert_eq!(taken_block.block_hash(), hash); + assert_eq!(*taken_block.hash(), hash); assert_eq!(height, 100); assert_eq!(taken_wallets, wallets); } @@ -516,7 +524,7 @@ mod tests { let hashes = pipeline.coordinator.take_pending(1); assert_eq!(hashes.len(), 1); pipeline.coordinator.mark_sent(&hashes); - assert!(pipeline.receive_block(&block)); + assert!(pipeline.receive_block(&HashedBlock::from(&block))); let (_, _, taken_wallets) = pipeline.take_next_ordered_block().unwrap(); let mut expected = wallets_a; @@ -550,7 +558,7 @@ mod tests { assert_eq!(pipeline.coordinator.active_count(), 1); // Late wallet ids are still merged for when the block arrives. - assert!(pipeline.receive_block(&block)); + assert!(pipeline.receive_block(&HashedBlock::from(&block))); let (_, _, taken_wallets) = pipeline.take_next_ordered_block().unwrap(); let mut expected = wallets_a; expected.extend(wallets_b); @@ -571,7 +579,7 @@ mod tests { pipeline.queue([(FilterMatchKey::new(100, hash), wallets_a.clone())]); let hashes = pipeline.coordinator.take_pending(1); pipeline.coordinator.mark_sent(&hashes); - assert!(pipeline.receive_block(&block)); + assert!(pipeline.receive_block(&HashedBlock::from(&block))); assert_eq!(pipeline.downloaded.len(), 1); assert_eq!(pipeline.coordinator.pending_count(), 0); assert_eq!(pipeline.coordinator.active_count(), 0); @@ -598,8 +606,8 @@ mod tests { let wallets_a: BTreeSet = BTreeSet::from([[1u8; 32]]); let wallets_b: BTreeSet = BTreeSet::from([[2u8; 32]]); - pipeline.add_from_storage(block.clone(), 100, wallets_a.clone()); - pipeline.add_from_storage(block.clone(), 100, wallets_b.clone()); + pipeline.add_from_storage(HashedBlock::from(&block), 100, wallets_a.clone()); + pipeline.add_from_storage(HashedBlock::from(&block), 100, wallets_b.clone()); let (_, _, taken_wallets) = pipeline.take_next_ordered_block().unwrap(); let mut expected = wallets_a; @@ -618,12 +626,12 @@ mod tests { pipeline.coordinator.mark_sent(&hashes); // First receive - let result = pipeline.receive_block(&block); + let result = pipeline.receive_block(&HashedBlock::from(&block)); assert!(result); assert_eq!(pipeline.downloaded.len(), 1); // Duplicate receive (not tracked anymore since already completed) - let result = pipeline.receive_block(&block); + let result = pipeline.receive_block(&HashedBlock::from(&block)); assert!(!result); assert_eq!(pipeline.downloaded.len(), 1); } diff --git a/dash-spv/src/sync/blocks/sync_manager.rs b/dash-spv/src/sync/blocks/sync_manager.rs index 871d78f23..9eeeda96d 100644 --- a/dash-spv/src/sync/blocks/sync_manager.rs +++ b/dash-spv/src/sync/blocks/sync_manager.rs @@ -62,6 +62,10 @@ impl SyncM self.pipeline.requeue_in_flight(); } + fn on_peer_disconnect(&mut self) { + self.pipeline.requeue_in_flight(); + } + async fn handle_message( &mut self, msg: Message, @@ -74,7 +78,7 @@ impl SyncM let hashed_block = HashedBlock::from(block); // Check if this is a block we requested (pipeline handles buffering with height) - if !self.pipeline.receive_block(block) { + if !self.pipeline.receive_block(&hashed_block) { tracing::debug!("Received unrequested block {}", hashed_block.hash()); return Ok(vec![]); } @@ -146,11 +150,7 @@ impl SyncM ))); } // Block loaded from storage, add to pipeline for processing - self.pipeline.add_from_storage( - hashed_block.block().clone(), - key.height(), - wallets.clone(), - ); + self.pipeline.add_from_storage(hashed_block, key.height(), wallets.clone()); self.progress.add_from_storage(1); continue; } diff --git a/dash-spv/src/sync/chainlock/manager.rs b/dash-spv/src/sync/chainlock/manager.rs index 4b55d840f..23993c78f 100644 --- a/dash-spv/src/sync/chainlock/manager.rs +++ b/dash-spv/src/sync/chainlock/manager.rs @@ -227,7 +227,7 @@ impl ChainLockManager { async fn verify_block_hash(&self, chainlock: &ChainLock) -> bool { let storage = self.header_storage.read().await; match storage.get_header(chainlock.block_height).await { - Ok(Some(header)) => header.block_hash() == chainlock.block_hash, + Ok(Some(header)) => *header.hash() == chainlock.block_hash, Ok(None) => { // Don't reject if we don't have the header yet true @@ -300,6 +300,7 @@ mod tests { use crate::Network; use dashcore::bls_sig_utils::BLSSignature; use dashcore::hashes::Hash; + use dashcore::sml::masternode_list::MasternodeList; use dashcore::BlockHash; type TestChainLockManager = @@ -412,7 +413,7 @@ mod tests { .block_headers() .write() .await - .store_headers_at_height(&[header], 100) + .store_headers_at_height(&[crate::types::HashedBlockHeader::from(header)], 100) .await .expect("store header at 100"); @@ -454,6 +455,30 @@ mod tests { assert_eq!(manager.progress.valid(), 0); } + #[tokio::test] + async fn test_chainlock_signed_above_the_newest_list_is_not_accepted() { + let mut manager = create_test_manager().await; + manager + .masternode_engine + .write() + .await + .masternode_lists + .insert(100, MasternodeList::empty(BlockHash::all_zeros(), 100)); + let _ = manager.on_masternode_ready().await; + + let events = manager.process_chainlock(&create_test_chainlock(1_000)).await.unwrap(); + + assert!(matches!( + events.as_slice(), + [SyncEvent::ChainLockReceived { + validated: false, + .. + }] + )); + assert!(manager.best_chainlock().is_none()); + assert_eq!(manager.progress.invalid(), 1); + } + #[tokio::test] async fn test_chainlock_keeps_only_best() { let mut manager = create_test_manager().await; diff --git a/dash-spv/src/sync/download_coordinator.rs b/dash-spv/src/sync/download_coordinator.rs index e36753b6d..6acb0f3ca 100644 --- a/dash-spv/src/sync/download_coordinator.rs +++ b/dash-spv/src/sync/download_coordinator.rs @@ -375,6 +375,21 @@ mod tests { assert_eq!(coord.retry_counts.get(&7), Some(&1)); assert!(!coord.is_in_flight(&7)); assert_eq!(coord.pending_count(), 1); + + // A peer disconnect is not an attempt against the item's budget, so + // repeated requeues must not spend it. + let items = coord.take_pending(1); + coord.mark_sent(&items); + coord.requeue_in_flight(); + assert_eq!(coord.retry_counts.get(&7), Some(&1)); + + // A genuine timeout still counts, and delivery still clears the budget. + let items = coord.take_pending(1); + coord.mark_sent(&items); + coord.enqueue_retry(7); + assert_eq!(coord.retry_counts.get(&7), Some(&2)); + assert!(coord.receive(&7)); + assert_eq!(coord.retry_counts.get(&7), None); } #[test] diff --git a/dash-spv/src/sync/events.rs b/dash-spv/src/sync/events.rs index 2fbf24217..76f4afccd 100644 --- a/dash-spv/src/sync/events.rs +++ b/dash-spv/src/sync/events.rs @@ -1,3 +1,4 @@ +use crate::sync::mempool::BroadcastResult; use crate::sync::ManagerIdentifier; use dashcore::ephemerealdata::chain_lock::ChainLock; use dashcore::ephemerealdata::instant_lock::InstantLock; @@ -162,6 +163,20 @@ pub enum SyncEvent { validated: bool, }, + /// The network-level outcome of a self-broadcast transaction was + /// determined: accepted (echoed back by non-recipient peers, InstantSend + /// locked, or confirmed) or uncertain (no signal within the timeout). + /// An `Uncertain` outcome may later be followed by an `Accepted` one. + /// + /// Emitted by: `MempoolManager` + /// Consumed by: External listeners, `broadcast_transaction_and_wait` + TransactionBroadcastResult { + /// The broadcast transaction's txid + txid: Txid, + /// The determined outcome + result: BroadcastResult, + }, + /// Sync has reached the chain tip (all managers idle). /// /// Emitted on every not-synced to synced transition. Cycle 0 is the @@ -226,12 +241,13 @@ impl fmt::Display for SyncEvent { } => match qr_info_result { Some(s) => write!( f, - "MasternodeStateUpdated(height={}, qr_info={{stored_cycle_height={:?}, verified={}/{}, newly_qualified={}}})", + "MasternodeStateUpdated(height={}, qr_info={{stored_cycle_height={:?}, verified={}/{}, newly_qualified={}, previous_cycle_invalid={}}})", height, s.stored_cycle_height, s.fully_verified_count, s.rotated_quorum_count, s.newly_qualified_count, + s.previous_cycle_invalid_count, ), None => write!(f, "MasternodeStateUpdated(height={})", height), }, @@ -256,6 +272,10 @@ impl fmt::Display for SyncEvent { "InstantLockReceived(txid={}, validated={})", instant_lock.txid, validated ), + SyncEvent::TransactionBroadcastResult { + txid, + result, + } => write!(f, "TransactionBroadcastResult(txid={}, result={})", txid, result), SyncEvent::SyncComplete { header_tip, cycle, diff --git a/dash-spv/src/sync/filter_headers/manager.rs b/dash-spv/src/sync/filter_headers/manager.rs index eb4f3600f..d2f1e5c90 100644 --- a/dash-spv/src/sync/filter_headers/manager.rs +++ b/dash-spv/src/sync/filter_headers/manager.rs @@ -43,6 +43,17 @@ pub struct FilterHeadersManager } impl FilterHeadersManager { + /// The tip height block-header storage holds right now. + /// + /// Distinct from `progress.block_header_tip_height()`, which is only ever + /// as fresh as the last `BlockHeadersStored` / `BlockHeaderSyncComplete` + /// this manager was handed. Storage can move without either arriving — + /// an out-of-order segment completing promotes a run of buffered headers — + /// so the tick reads storage directly to notice. + pub(super) async fn stored_block_header_tip(&self) -> Option { + self.header_storage.read().await.get_tip_height().await + } + /// Transition to `Synced` and return `FilterHeadersSyncComplete` if block headers /// are done and filter headers have reached the target. Returns `None` if already /// `Synced` or conditions are not met. @@ -249,12 +260,16 @@ impl std::fmt::Debug #[cfg(test)] mod tests { use super::*; - use crate::network::MessageType; + use crate::network::{MessageType, NetworkRequest}; use crate::storage::{ DiskStorageManager, PersistentBlockHeaderStorage, PersistentFilterHeaderStorage, StorageManager, }; use crate::sync::{ManagerIdentifier, SyncManagerProgress}; + use crate::types::HashedBlockHeader; + use dashcore::network::message::NetworkMessage; + use dashcore::{block::Version, BlockHash, CompactTarget, Header as BlockHeader}; + use dashcore_hashes::Hash; type TestFilterHeadersManager = FilterHeadersManager; @@ -379,6 +394,96 @@ mod tests { assert!(!events.iter().any(|e| matches!(e, SyncEvent::FilterHeadersSyncComplete { .. }))); } + /// The tick must notice block-header storage advancing on its own. + /// + /// `handle_new_headers` — the only path that extends the CFHeaders queue — + /// runs off `BlockHeadersStored` / `BlockHeaderSyncComplete`. Storage can + /// move without either arriving, because a segment completing out of order + /// promotes a run of buffered headers. When that happened on a mainnet + /// restore the queue stayed at its old target and filter headers stopped + /// permanently, while block headers and ChainLocks carried on. + #[tokio::test] + async fn test_tick_extends_when_storage_tip_advanced_without_an_event() { + let storage = DiskStorageManager::with_temp_dir().await.unwrap(); + let header_storage = storage.block_headers(); + let mut manager = + FilterHeadersManager::new(header_storage.clone(), storage.filter_headers()) + .await + .expect("Failed to create FilterHeadersManager"); + let (sender, mut rx) = create_test_request_sender(); + + // Mid-sync: this manager was last told the tip was 1000. + manager.progress.update_current_height(1000); + manager.progress.update_target_height(1000); + manager.progress.update_block_header_tip_height(1000); + manager.set_state(SyncState::Syncing); + + // Storage moves past that on its own — no event is delivered. + let mut headers = Vec::new(); + let mut prev = BlockHash::from_byte_array([0u8; 32]); + for nonce in 0..1200u32 { + let header = HashedBlockHeader::from(BlockHeader { + version: Version::from_consensus(1), + prev_blockhash: prev, + merkle_root: dashcore::TxMerkleNode::all_zeros(), + time: 0, + bits: CompactTarget::from_consensus(0x2100ffff), + nonce, + }); + prev = *header.hash(); + headers.push(header); + } + header_storage.write().await.store_headers_at_height(&headers, 0).await.unwrap(); + let stored_tip = manager.stored_block_header_tip().await.expect("storage has a tip"); + assert!(stored_tip > 1000, "test setup: storage tip must exceed the known tip"); + + let manager_ref: &mut TestSyncManager = &mut manager; + manager_ref.tick(&sender).await.unwrap(); + + assert_eq!( + manager.progress.block_header_tip_height(), + stored_tip, + "tick must pick up a tip that advanced without an event" + ); + + // The tip alone is not the fix. `handle_new_headers` updates that + // field before it touches the pipeline, so a version that noticed the + // advance and then failed to queue anything would still satisfy the + // assertion above — and would leave sync exactly as stuck as before. + // What has to be true is that a request for the new range went out. + let mut requested_stops = Vec::new(); + while let Ok(request) = rx.try_recv() { + match request { + NetworkRequest::SendMessage(NetworkMessage::GetCFHeaders(get)) => { + requested_stops.push(get.stop_hash); + } + NetworkRequest::SendMessageToPeer(NetworkMessage::GetCFHeaders(get), _) => { + requested_stops.push(get.stop_hash); + } + _ => {} + } + } + assert!( + !requested_stops.is_empty(), + "tick must queue and send CFHeaders requests for the newly available range" + ); + // Every stop hash must be a header the new range actually contains, so + // a request built from the stale target cannot pass this. + let storage = header_storage.read().await; + for stop in &requested_stops { + let mut found = false; + for height in 1001..=stored_tip { + if let Ok(Some(header)) = storage.get_header(height).await { + if header.hash() == stop { + found = true; + break; + } + } + } + assert!(found, "CFHeaders stop hash {stop} is not in the newly discovered range"); + } + } + #[tokio::test] async fn test_on_disconnect() { let mut manager = create_test_manager().await; diff --git a/dash-spv/src/sync/filter_headers/pipeline.rs b/dash-spv/src/sync/filter_headers/pipeline.rs index 0640f4d53..97911cbad 100644 --- a/dash-spv/src/sync/filter_headers/pipeline.rs +++ b/dash-spv/src/sync/filter_headers/pipeline.rs @@ -87,13 +87,10 @@ impl FilterHeadersPipeline { let batch_end = (current + FILTER_HEADERS_BATCH_SIZE - 1).min(new_target); // Get stop hash for this batch - let stop_hash = storage - .get_header(batch_end) - .await? - .ok_or_else(|| { + let stop_hash = + storage.get_header(batch_end).await?.map(|h| *h.hash()).ok_or_else(|| { SyncError::Storage(format!("Missing header at height {}", batch_end)) - })? - .block_hash(); + })?; self.coordinator.enqueue([stop_hash]); self.batch_starts.insert(stop_hash, current); @@ -145,13 +142,10 @@ impl FilterHeadersPipeline { let batch_end = (current + FILTER_HEADERS_BATCH_SIZE - 1).min(target_height); // Get stop hash for this batch - let stop_hash = storage - .get_header(batch_end) - .await? - .ok_or_else(|| { + let stop_hash = + storage.get_header(batch_end).await?.map(|h| *h.hash()).ok_or_else(|| { SyncError::Storage(format!("Missing header at height {}", batch_end)) - })? - .block_hash(); + })?; self.coordinator.enqueue([stop_hash]); self.batch_starts.insert(stop_hash, current); @@ -260,6 +254,16 @@ impl FilterHeadersPipeline { ready } + /// Move in-flight `getcfheaders` requests back to pending after a peer + /// disconnect so the next `send_pending` reissues them. + /// + /// `batch_starts` must survive, since `send_pending` errors out on a pending + /// stop hash with no start height. `next_expected` and the out-of-order + /// buffer survive too, so already-received batches are not re-downloaded. + pub(super) fn requeue_in_flight(&mut self) { + self.coordinator.requeue_in_flight(); + } + /// Re-enqueue timed out requests for retry. pub(super) fn handle_timeouts(&mut self) { for stop_hash in self.coordinator.check_timeouts() { @@ -273,6 +277,10 @@ mod tests { use dashcore_hashes::Hash; use super::*; + use crate::network::NetworkRequest; + use dashcore::hash_types::{FilterHash, FilterHeader}; + use dashcore::network::message_filter::GetCFHeaders; + use tokio::sync::mpsc::unbounded_channel; #[test] fn test_cfheaders_pipeline_new() { @@ -422,6 +430,60 @@ mod tests { assert!(matches!(err, SyncError::InvalidState(_))); } + /// A peer disconnect requeues in-flight batches without discarding what the + /// pipeline has already made of the ones that came back, so the reissued + /// requests carry their original start heights and nothing is re-downloaded. + #[test] + fn test_requeue_in_flight_reissues_batches_and_keeps_progress() { + let mut pipeline = FilterHeadersPipeline::new(); + pipeline.next_expected = 1; + pipeline.target_height = 6000; + + let hash1 = BlockHash::from_byte_array([0x01; 32]); + let hash2 = BlockHash::from_byte_array([0x02; 32]); + pipeline.coordinator.mark_sent(&[hash1, hash2]); + pipeline.batch_starts.insert(hash1, 1); + pipeline.batch_starts.insert(hash2, 2001); + + // A third batch already came back out of order and is waiting for the + // two above to be processed first. + pipeline.buffered.insert( + 4001, + CFHeaders { + filter_type: 0, + stop_hash: BlockHash::from_byte_array([0x03; 32]), + previous_filter_header: FilterHeader::all_zeros(), + filter_hashes: vec![FilterHash::all_zeros()], + }, + ); + + pipeline.requeue_in_flight(); + assert_eq!(pipeline.coordinator.active_count(), 0); + assert_eq!(pipeline.coordinator.pending_count(), 2); + + let (tx, mut rx) = unbounded_channel(); + let requests = RequestSender::new(tx); + assert_eq!(pipeline.send_pending(&requests).unwrap(), 2); + + let mut reissued = Vec::new(); + while let Ok(request) = rx.try_recv() { + match request { + NetworkRequest::SendMessage(NetworkMessage::GetCFHeaders(GetCFHeaders { + start_height, + stop_hash, + .. + })) => reissued.push((start_height, stop_hash)), + other => panic!("Expected GetCFHeaders, got {:?}", other), + } + } + reissued.sort(); + assert_eq!(reissued, vec![(1, hash1), (2001, hash2)]); + + assert_eq!(pipeline.next_expected(), 1); + assert_eq!(pipeline.buffered.len(), 1); + assert_eq!(pipeline.target_height, 6000); + } + #[test] fn test_handle_timeouts_multiple_batches() { use std::time::Duration; diff --git a/dash-spv/src/sync/filter_headers/sync_manager.rs b/dash-spv/src/sync/filter_headers/sync_manager.rs index eae554d57..353bf2b8a 100644 --- a/dash-spv/src/sync/filter_headers/sync_manager.rs +++ b/dash-spv/src/sync/filter_headers/sync_manager.rs @@ -37,6 +37,10 @@ impl SyncManager for FilterHeade self.block_headers_synced = false; } + fn on_peer_disconnect(&mut self) { + self.pipeline.requeue_in_flight(); + } + async fn handle_message( &mut self, msg: Message, @@ -152,6 +156,30 @@ impl SyncManager for FilterHeade // Send pending requests (including retries) self.pipeline.send_pending(requests)?; + // Re-read the block-header tip and extend from here if it moved. + // + // `handle_new_headers` — the only thing that ever extends the + // CFHeaders queue — runs solely off `BlockHeaderSyncComplete` and + // `BlockHeadersStored`. Header storage can advance without either + // reaching this manager: a segment that completes out of order + // promotes a run of buffered headers, and on a mainnet scan that is + // how the last stretch of the chain lands. When it does, this + // manager keeps the target it was last told about, its queue drains, + // and nothing re-arms it — filter headers stop for good while block + // headers, ChainLocks and inv announcements carry on, so the client + // looks alive while sync is frozen. Observed on a mainnet restore: + // the queue was last extended to height 2_398_000, block headers + // reached 2_523_515 twenty-four minutes later, and filter headers + // never moved again. + // + // Same fix as promoting finished header segments from the tick + // (#960): trust the tick, not the message. + if let Some(tip) = self.stored_block_header_tip().await { + if tip > self.progress.block_header_tip_height() { + return self.handle_new_headers(tip, requests).await; + } + } + Ok(vec![]) } diff --git a/dash-spv/src/sync/filters/batch.rs b/dash-spv/src/sync/filters/batch.rs index 386e0d16e..3fd609b79 100644 --- a/dash-spv/src/sync/filters/batch.rs +++ b/dash-spv/src/sync/filters/batch.rs @@ -1,7 +1,7 @@ use dashcore::bip158::BlockFilter; use dashcore::ScriptBuf; use key_wallet_manager::{FilterMatchKey, WalletId}; -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; /// A completed batch of compact block filters ready for verification. /// @@ -24,14 +24,21 @@ pub(super) struct FiltersBatch { pending_blocks: u32, /// Whether rescan has been completed for this batch. rescan_complete: bool, - /// Wallets that were behind for this batch's height range at scan time and - /// therefore need their `synced_height` advanced when the batch commits. - /// Already-synced wallets must not be touched. - scanned_wallets: BTreeSet, + /// Wallets that were behind for this batch's height range at scan time — + /// and therefore need their `synced_height` advanced when the batch + /// commits — each mapped to the wallet's `account_generation` at scan + /// time. Commit refuses to advance a wallet whose generation has changed + /// since the scan: an account added mid-flight means the scan did not test + /// the new account's scripts, so the batch cannot certify coverage for the + /// current account set (dashpay/rust-dashcore#649). Already-synced wallets + /// must not be touched. + scanned_wallets: BTreeMap, /// Cached scriptPubKeys discovered during block processing that still /// need rescan, attributed per wallet so we can rerun matching only /// against the wallet that produced each new script. collected_scripts: HashMap>, + /// Every script already matched against this batch's filters, per wallet. + tested_scripts: HashMap>, } impl FiltersBatch { @@ -49,8 +56,9 @@ impl FiltersBatch { scanned: false, pending_blocks: 0, rescan_complete: false, - scanned_wallets: BTreeSet::new(), + scanned_wallets: BTreeMap::new(), collected_scripts: HashMap::new(), + tested_scripts: HashMap::new(), } } /// Start height of this batch (inclusive). @@ -114,17 +122,37 @@ impl FiltersBatch { ) { self.collected_scripts.entry(wallet_id).or_default().extend(scripts); } + /// Record that `scripts` have been matched against this batch's filters. + pub(super) fn mark_tested>( + &mut self, + wallet_id: WalletId, + scripts: I, + ) { + self.tested_scripts.entry(wallet_id).or_default().extend(scripts); + } + + /// The wallet's scripts that this batch has never been matched against. + pub(super) fn untested<'a>( + &'a self, + wallet_id: &WalletId, + monitored: &'a [ScriptBuf], + ) -> impl Iterator { + let tested = self.tested_scripts.get(wallet_id); + monitored.iter().filter(move |script| tested.is_none_or(|t| !t.contains(*script))) + } + /// Take collected per-wallet scripts for rescan, leaving the map empty. pub(super) fn take_collected_scripts(&mut self) -> HashMap> { std::mem::take(&mut self.collected_scripts) } - /// Record the set of wallets that were behind for this batch at scan time. - pub(super) fn set_scanned_wallets(&mut self, wallets: BTreeSet) { + /// Record the wallets that were behind for this batch at scan time, each + /// with its `account_generation` snapshot. + pub(super) fn set_scanned_wallets(&mut self, wallets: BTreeMap) { self.scanned_wallets = wallets; } - /// Wallets that were behind at scan time and must have their synced_height - /// advanced when this batch commits. - pub(super) fn scanned_wallets(&self) -> &BTreeSet { + /// Wallets that were behind at scan time (with their generation snapshot) + /// and must have their synced_height advanced when this batch commits. + pub(super) fn scanned_wallets(&self) -> &BTreeMap { &self.scanned_wallets } } diff --git a/dash-spv/src/sync/filters/block_match_tracker.rs b/dash-spv/src/sync/filters/block_match_tracker.rs index 6f9301491..eb81d5344 100644 --- a/dash-spv/src/sync/filters/block_match_tracker.rs +++ b/dash-spv/src/sync/filters/block_match_tracker.rs @@ -92,6 +92,28 @@ impl BlockMatchTracker { } } + /// Like [`Self::track`], but for matches driven by newly derived + /// scripts: the prior processing of this block predates those scripts, + /// so processed records do not cover them and the block must be + /// re-applied to the full candidate set. Never returns + /// `AlreadyProcessed`. + pub(super) fn track_for_new_scripts( + &mut self, + key: &FilterMatchKey, + batch_start: u32, + candidate_wallets: BTreeSet, + ) -> BlockTrackResult { + if self.blocks_remaining.contains_key(key.hash()) { + return BlockTrackResult::InFlight { + wallets: candidate_wallets, + }; + } + self.blocks_remaining.insert(*key.hash(), (key.height(), batch_start)); + BlockTrackResult::NewlyTracked { + wallets: candidate_wallets, + } + } + /// Record that `wallets` have had the block at `(height, hash)` applied /// to their state. Idempotent: existing entries merge, never shrink. pub(super) fn record_processed( @@ -126,6 +148,11 @@ impl BlockMatchTracker { self.processed_blocks_per_wallet.split_off(&(height + 1)); } + /// True while matched blocks are still awaiting their `BlockProcessed`. + pub(super) fn has_blocks_in_flight(&self) -> bool { + !self.blocks_remaining.is_empty() + } + /// True when there is no in-flight or processed-record state. pub(super) fn is_empty(&self) -> bool { self.blocks_remaining.is_empty() && self.processed_blocks_per_wallet.is_empty() @@ -210,6 +237,23 @@ mod tests { tracker.track(&key, 5000, BTreeSet::from([wallet_a, wallet_b])), BlockTrackResult::AlreadyProcessed ); + + // A match on newly derived scripts re-queues the fully-processed + // block anyway: the prior processing predates those scripts. + assert_eq!( + tracker.track_for_new_scripts(&key, 5000, BTreeSet::from([wallet_a])), + BlockTrackResult::NewlyTracked { + wallets: BTreeSet::from([wallet_a]) + } + ); + // While re-queued the block reports InFlight, preserving the + // one-increment-one-decrement pending accounting. + assert_eq!( + tracker.track_for_new_scripts(&key, 5000, BTreeSet::from([wallet_a])), + BlockTrackResult::InFlight { + wallets: BTreeSet::from([wallet_a]) + } + ); } /// `prune_at_or_below` drops every entry at or below the given height diff --git a/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs b/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs new file mode 100644 index 000000000..bf064fd97 --- /dev/null +++ b/dash-spv/src/sync/filters/coinjoin_gap_discovery_tests.rs @@ -0,0 +1,586 @@ +//! CoinJoin gap-limit discovery repros against the real filter → block → +//! wallet pipeline (`FiltersManager` + `BlockMatchTracker` + a real +//! `WalletManager` with a CoinJoin account). +//! +//! Background: a wallet with dense CoinJoin usage (dozens of consecutive +//! external indices funded per block) historically stalled at +//! `highest_used = 59` (initial 30-address watch window + one gap-limit +//! extension) because a matched block was applied to a wallet exactly once — +//! the `BlockMatchTracker` gate was keyed by `(wallet, block)`, not +//! `(wallet, address)` — so outputs paying addresses derived *from that same +//! block's matches* were never re-tested. PR #820 fixed the in-flight part: +//! `rescan_batch` now re-queues already-processed blocks of ACTIVE batches +//! against newly derived scripts, to a fixpoint. +//! +//! These tests pin both the fixed behaviour and the remaining hole: +//! +//! 1. [`coinjoin_gap_limit_dense_same_batch_recovers`] — the empirical +//! stall-at-59 shape (two dense blocks in one batch). GREEN since #820; +//! kept as a regression guard. +//! 2. [`coinjoin_gap_limit_inversion_within_batch_recovers`] — a gap-window +//! output in an EARLIER block of the SAME (still-active) batch is +//! recovered by the commit-time rescan. GREEN since #820. +//! 3. [`coinjoin_gap_limit_stall_across_committed_batch`] — the SAME shape +//! with the earlier block in an already-COMMITTED batch (#846): rescans +//! only reach `active_batches`, and committed batches are gone. GREEN +//! since `rescan_committed_range` re-tests newly derived scripts against +//! the STORED filters below the committing batch. +//! 4. [`committed_range_sweep_coalesces_across_batch_commits`] — the cost +//! side of #846's fix: N script-carrying commits share one drain-time +//! committed-range sweep instead of walking the stored history once per +//! commit, while a late-derived script still finds its +//! committed-prefix block before `FiltersSyncComplete`. +//! +//! Each test drives the manager exactly the way the production event loop +//! does: `try_process_batch` → `BlocksNeeded` → (blocks-manager stand-in) +//! `WalletInterface::process_block_for_wallets` → `BlockProcessed` (with the +//! wallet's freshly derived scripts) → `handle_sync_event` → commit-time +//! rescan, iterated to quiescence. Deterministic: fixed mnemonic, synthetic +//! blocks, real BIP-158 filters, no network. + +use super::*; +use crate::storage::{ + DiskStorageManager, PersistentBlockHeaderStorage, PersistentFilterHeaderStorage, + PersistentFilterStorage, StorageManager, +}; +use dashcore::{ + Address, Block, BlockHash, Network, OutPoint, Transaction, TxIn, TxOut, Txid, Witness, +}; +use key_wallet::account::ManagedAccountTrait; +use key_wallet::gap_limit::DEFAULT_COINJOIN_GAP_LIMIT; +use key_wallet::managed_account::address_pool::{AddressPool, AddressPoolType, KeySource}; +use key_wallet::wallet::initialization::WalletAccountCreationOptions; +use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; +use key_wallet_manager::WalletManager; +use tokio::sync::mpsc::unbounded_channel; + +/// Deterministic test wallet seed (standard BIP-39 test vector mnemonic). +const TEST_MNEMONIC: &str = + "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about"; + +/// CoinJoin-denomination-shaped output value (0.001 DASH + per-round fee). +/// The matcher is purely script-based; the value is for realism only. +const DENOMINATION: u64 = 100_001; + +/// The initial CoinJoin watch window is `0..G`. Every funded index range +/// below is expressed relative to `G` so the tests keep pinning the +/// beyond-the-window shapes they were written for even when the default gap +/// changes (it moved 30 -> 100 in #868, which would otherwise have silently +/// turned the cross-commit repro into a trivially-green test). +const G: usize = DEFAULT_COINJOIN_GAP_LIMIT as usize; + +type CjFiltersManager = FiltersManager< + PersistentBlockHeaderStorage, + PersistentFilterHeaderStorage, + PersistentFilterStorage, + WalletManager, +>; + +/// Real wallet manager with one seed wallet (Default accounts incl. CoinJoin +/// account 0, external pool pre-generated to `DEFAULT_COINJOIN_GAP_LIMIT`), +/// wired into a `FiltersManager` over temp-dir storage. +async fn setup() -> (CjFiltersManager, Arc>>, WalletId) { + let mut wm = WalletManager::::new(Network::Regtest); + let wallet_id = wm + .create_wallet_from_mnemonic(TEST_MNEMONIC, 0, WalletAccountCreationOptions::Default) + .expect("create deterministic test wallet"); + let wallet = Arc::new(RwLock::new(wm)); + + let storage = DiskStorageManager::with_temp_dir().await.unwrap(); + let mut manager = FiltersManager::new( + Arc::clone(&wallet), + storage.block_headers(), + storage.filter_headers(), + storage.filters(), + ) + .await; + manager.set_state(SyncState::Syncing); + (manager, wallet, wallet_id) +} + +/// Derive the wallet's first `count` CoinJoin-account-0 External addresses +/// OFFLINE (via a standalone pool over the same xpub + base path), so test +/// blocks can pay indices far beyond the wallet's live watch window without +/// touching the wallet's own state. +async fn coinjoin_external_addresses( + wallet: &Arc>>, + wallet_id: &WalletId, + count: u32, +) -> Vec
{ + let wm = wallet.read().await; + let signing_wallet = wm.get_wallet(wallet_id).expect("wallet present"); + let xpub = signing_wallet + .accounts + .coinjoin_accounts + .get(&0) + .expect("CoinJoin account 0 (Default options)") + .account_xpub; + let info = wm.get_wallet_info(wallet_id).expect("wallet info present"); + let live_pool = coinjoin_external_pool(info); + let mut pool = AddressPool::new( + live_pool.base_path.clone(), + AddressPoolType::External, + count, + Network::Regtest, + &KeySource::Public(xpub), + ) + .expect("derive standalone CoinJoin External pool"); + // `AddressPool::new` pre-generates `count` addresses; extend defensively + // in case the constructor semantics change. + if pool.highest_generated.map(|h| h + 1).unwrap_or(0) < count { + let missing = count - pool.highest_generated.map(|h| h + 1).unwrap_or(0); + pool.generate_addresses(missing, &KeySource::Public(xpub), true) + .expect("extend standalone pool"); + } + (0..count).map(|i| pool.address_at_index(i).expect("address generated")).collect() +} + +/// The wallet's LIVE CoinJoin-account-0 External pool. +fn coinjoin_external_pool(info: &ManagedWalletInfo) -> &AddressPool { + let account = + info.accounts.coinjoin_accounts.get(&0).expect("CoinJoin account 0 (Default options)"); + account + .managed_account_type() + .address_pools() + .into_iter() + .find(|p| p.pool_type == AddressPoolType::External) + .expect("CoinJoin External pool") +} + +/// `(highest_used, highest_generated, used_count)` of the live CoinJoin +/// External pool — the discovery frontier the assertions pin. +async fn coinjoin_pool_state( + wallet: &Arc>>, + wallet_id: &WalletId, +) -> (Option, Option, usize) { + let wm = wallet.read().await; + let info = wm.get_wallet_info(wallet_id).expect("wallet info present"); + let pool = coinjoin_external_pool(info); + (pool.highest_used, pool.highest_generated, pool.used_indices.len()) +} + +/// One block at `height` whose single transaction pays every address in +/// `addresses` one denomination output. Returns the block, its real BIP-158 +/// filter, and the filter match key. +fn block_paying(height: u32, addresses: &[Address]) -> (Block, BlockFilter, FilterMatchKey) { + let mut prev_txid = [0xABu8; 32]; + prev_txid[..4].copy_from_slice(&height.to_le_bytes()); + let tx = Transaction { + version: 1, + lock_time: 0, + // One non-null input so the tx is not coinbase-shaped. + input: vec![TxIn { + previous_output: OutPoint::new(Txid::from(prev_txid), 0), + script_sig: ScriptBuf::new(), + sequence: 0xffffffff, + witness: Witness::new(), + }], + output: addresses + .iter() + .map(|a| TxOut { + value: DENOMINATION, + script_pubkey: a.script_pubkey(), + }) + .collect(), + special_transaction_payload: None, + }; + let block = Block::dummy(height, vec![tx]); + let filter = BlockFilter::dummy(&block); + let key = FilterMatchKey::new(height, block.block_hash()); + (block, filter, key) +} + +/// Drive the production event loop to quiescence: consume `BlocksNeeded` by +/// processing the named blocks through the real wallet (the blocks-manager's +/// job), feed the resulting `BlockProcessed` events (carrying the wallet's +/// freshly derived scripts) back into the filters manager, and repeat until +/// no more blocks are requested. Blocks are processed in height order, like +/// the real `BlocksPipeline`. +async fn drive_to_quiescence( + manager: &mut CjFiltersManager, + wallet: &Arc>>, + blocks: &HashMap, + initial_events: Vec, +) { + let (tx, _rx) = unbounded_channel(); + let requests = RequestSender::new(tx); + + let mut events = initial_events; + for _round in 0..64 { + let mut pending: BTreeMap<(u32, BlockHash), BTreeSet> = BTreeMap::new(); + for event in events.drain(..) { + if let SyncEvent::BlocksNeeded { + blocks: needed, + } = event + { + for (key, wallets) in needed { + pending.entry((key.height(), *key.hash())).or_default().extend(wallets); + } + } + } + if pending.is_empty() { + return; + } + + let mut next_events = Vec::new(); + for ((height, block_hash), wallets) in pending { + let block = blocks.get(&block_hash).expect("BlocksNeeded for an unknown test block"); + let result = wallet + .write() + .await + .process_block_for_wallets(block, block_hash, height, &wallets) + .await; + let confirmed_txids = result.relevant_txids().cloned().collect(); + let event = SyncEvent::BlockProcessed { + block_hash, + height, + wallets, + new_scripts: result.new_scripts, + confirmed_txids, + }; + next_events.extend( + manager.handle_sync_event(&event, &requests).await.expect("BlockProcessed"), + ); + } + events = next_events; + } + panic!("drive_to_quiescence did not settle within 64 rounds — runaway rescan loop?"); +} + +/// The empirical dense stall shape (stall-at-59 under the original gap of +/// 30): block A funds CoinJoin External indices 0..=51, the NEXT block funds +/// 52..=G+39 — the tail extends a full window past the initial one, so +/// discovery must climb through at least one gap extension. Before PR #820 +/// the `(wallet, block)` gate stopped discovery at the first extension; the +/// commit-time fixpoint rescan now climbs the whole dense range. Regression +/// guard for #820. +#[tokio::test] +async fn coinjoin_gap_limit_dense_same_batch_recovers() { + let (mut manager, wallet, wallet_id) = setup().await; + let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 40) as u32).await; + + let (highest_used, highest_generated, _) = coinjoin_pool_state(&wallet, &wallet_id).await; + assert_eq!(highest_used, None, "fresh wallet must have no used CoinJoin indices"); + assert_eq!( + highest_generated, + Some(DEFAULT_COINJOIN_GAP_LIMIT - 1), + "fresh wallet must watch exactly the initial CoinJoin gap window" + ); + + let (block_a, filter_a, key_a) = block_paying(10, &addresses[0..=51]); + let (block_b, filter_b, key_b) = block_paying(11, &addresses[52..=(G + 39)]); + let blocks: HashMap = + HashMap::from([(block_a.block_hash(), block_a), (block_b.block_hash(), block_b)]); + + let mut batch = FiltersBatch::new(0, 99, HashMap::from([(key_a, filter_a), (key_b, filter_b)])); + batch.mark_verified(); + manager.active_batches.insert(0, batch); + manager.progress.update_stored_height(99); + + let initial_events = manager.try_process_batch().await.unwrap(); + drive_to_quiescence(&mut manager, &wallet, &blocks, initial_events).await; + + let (highest_used, _, used_count) = coinjoin_pool_state(&wallet, &wallet_id).await; + assert_eq!( + highest_used, + Some((G + 39) as u32), + "dense same-batch CoinJoin range must be fully discovered via the commit-time \ + fixpoint rescan (PR #820); a stall at the first gap extension means the \ + (wallet, block) once-per-block gate regressed. used_count={used_count}" + ); + assert_eq!(used_count, G + 40, "every funded index 0..=G+39 must be marked used"); +} + +/// Height inversion INSIDE one active batch: indices G+10..=G+21 (beyond the +/// initial watch window) are funded at height 10, in-window indices 0..=29 +/// only at height 20. The initial scan cannot see block A (nothing watched +/// matches it), but once block B's processing extends the window past G+21 +/// the commit-time rescan re-tests block A's filter and recovers it. GREEN +/// since #820 — contrast for the cross-batch test below, isolating the +/// commit boundary as the broken seam. +#[tokio::test] +async fn coinjoin_gap_limit_inversion_within_batch_recovers() { + let (mut manager, wallet, wallet_id) = setup().await; + let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 22) as u32).await; + + let (block_a, filter_a, key_a) = block_paying(10, &addresses[(G + 10)..=(G + 21)]); + let (block_b, filter_b, key_b) = block_paying(20, &addresses[0..=29]); + let blocks: HashMap = + HashMap::from([(block_a.block_hash(), block_a), (block_b.block_hash(), block_b)]); + + let mut batch = FiltersBatch::new(0, 99, HashMap::from([(key_a, filter_a), (key_b, filter_b)])); + batch.mark_verified(); + manager.active_batches.insert(0, batch); + manager.progress.update_stored_height(99); + + let initial_events = manager.try_process_batch().await.unwrap(); + drive_to_quiescence(&mut manager, &wallet, &blocks, initial_events).await; + + let (highest_used, _, used_count) = coinjoin_pool_state(&wallet, &wallet_id).await; + assert_eq!( + highest_used, + Some((G + 21) as u32), + "a gap-window output in an earlier block of the SAME active batch must be \ + recovered by the commit-time rescan (PR #820). used_count={used_count}" + ); +} + +/// Gap-window outputs in an already-COMMITTED batch (#846). +/// +/// Same funding shape as the within-batch inversion test, but the early +/// block (indices G+10..=G+21, height 10) sits in batch 0..=99 while the +/// in-window block (indices 0..=29) sits at height 110 in batch 100..=199. +/// Batch 0 scans clean (nothing watched matches) and commits. Processing the +/// height-110 block extends the window past G+21, and those scripts DO match +/// block 10's filter — but `rescan_batch` only reaches `active_batches`, and +/// committed batches are gone (`try_commit_batches` removes them; the +/// tracker prunes at-or-below the committed height). Indices G+10..=G+21 — +/// squarely inside the BIP-44/CoinJoin gap-limit recovery contract +/// (G+21 < 29 + 1 + G) — used to stay invisible forever, along with their +/// funds; a fresh re-sync from genesis hit the same wall deterministically. +/// +/// GREEN since `rescan_committed_range`: newly derived scripts are re-tested +/// against the persisted filters below the committing batch (BIP-158 filters +/// are address-independent, so re-matching needs no re-download), and hits +/// flow through the `track_for_new_scripts` re-download path to the same +/// commit-time fixpoint. `highest_used` reaches G+21. +#[tokio::test] +async fn coinjoin_gap_limit_stall_across_committed_batch() { + let (mut manager, wallet, wallet_id) = setup().await; + let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 22) as u32).await; + + let (block_a, filter_a, key_a) = block_paying(10, &addresses[(G + 10)..=(G + 21)]); + let (block_b, filter_b, key_b) = block_paying(110, &addresses[0..=29]); + + // Uphold the production invariant the injected batches imply: every + // height at or below `stored_height` has its header and filter + // persisted (store_and_match_batches stores a batch's filters before + // stored_height advances past it). The committed-range recovery path + // re-tests exactly this stored data, so the invariant is load-bearing + // here: batch 0 commits before block B's processing derives the missing + // scripts, and by then its in-memory filters are gone. + { + let mut header_storage = manager.header_storage.write().await; + let mut filter_storage = manager.filter_storage.write().await; + for height in 0..=99u32 { + let (header, filter_bytes) = if height == 10 { + (block_a.header, filter_a.content.clone()) + } else { + let filler = Block::dummy(height, vec![]); + let filter = BlockFilter::dummy(&filler); + (filler.header, filter.content) + }; + header_storage + .store_headers_at_height(&[header.into()], height) + .await + .expect("seed header"); + filter_storage.store_filter(height, &filter_bytes).await.expect("seed filter"); + } + } + + let blocks: HashMap = + HashMap::from([(block_a.block_hash(), block_a), (block_b.block_hash(), block_b)]); + + let mut batch_0 = FiltersBatch::new(0, 99, HashMap::from([(key_a, filter_a)])); + batch_0.mark_verified(); + manager.active_batches.insert(0, batch_0); + let mut batch_1 = FiltersBatch::new(100, 199, HashMap::from([(key_b, filter_b)])); + batch_1.mark_verified(); + manager.active_batches.insert(100, batch_1); + manager.progress.update_stored_height(199); + + let initial_events = manager.try_process_batch().await.unwrap(); + drive_to_quiescence(&mut manager, &wallet, &blocks, initial_events).await; + + let (highest_used, highest_generated, used_count) = + coinjoin_pool_state(&wallet, &wallet_id).await; + // Sanity: the in-window block was found and the gap window extended past + // index G+21, so the missed indices ARE inside the watched range by now. + assert!( + highest_generated >= Some((G + 21) as u32), + "gap maintenance must have extended the watch window past index G+21 \ + (got {highest_generated:?})" + ); + assert_eq!( + highest_used, + Some((G + 21) as u32), + "CoinJoin External indices G+10..=G+21 were funded at height 10 in a batch that \ + committed before their scripts were derived, and the new-script rescan never \ + looks below the committed boundary (rescan_batch only reaches active_batches; \ + BlockMatchTracker/commit pruning drops the range). The addresses are within \ + the gap-limit recovery contract and are watched now (highest_generated = \ + {highest_generated:?}), yet their outputs stay invisible: highest_used stalls \ + at {highest_used:?}, used_count={used_count}. Fix direction: key re-scan \ + suppression by (wallet, address/script) instead of block/commit progress, or \ + trigger a below-committed-height rescan for a wallet whose gap maintenance \ + derives scripts mid-sync." + ); +} + +/// Committed-range sweeps coalesce across batch commits. +/// +/// Four batches; the last three each contain one in-window block whose +/// processing derives new scripts (each funds the next run of CoinJoin +/// indices, so gap maintenance extends the watch window at every commit). +/// Per-commit sweeping walks the entire committed prefix once per +/// script-carrying commit — on a real mainnet restore that shape produced +/// 191 full-prefix sweeps totalling ~14.5 minutes. Coalesced, the +/// accumulated scripts cross the stored history when the forward pipeline +/// drains: one sweep, plus one follow-up round for the scripts derived from +/// the block that sweep recovers. +/// +/// The early block (height 10, beyond-window indices G+10..=G+21, unwatched +/// when its batch scans and commits) pins the #846 correctness contract at +/// the same time: the deferred sweep must still find it, and +/// `FiltersSyncComplete` must not be emitted before it has been found and +/// applied. `committed_range_sweeps` counts sweeps that reach the chunk walk +/// in `rescan_committed_range`. +#[tokio::test] +async fn committed_range_sweep_coalesces_across_batch_commits() { + let (mut manager, wallet, wallet_id) = setup().await; + let addresses = coinjoin_external_addresses(&wallet, &wallet_id, (G + 22) as u32).await; + + // Beyond-window block in the range that commits first (#846 shape). + let (block_early, filter_early, key_early) = block_paying(10, &addresses[(G + 10)..=(G + 21)]); + // One in-window block per later batch. Each extends `highest_used` by 30, + // so every one of these batches carries newly derived scripts into its + // commit. After block 110 the generated window reaches 29 + G >= G + 21, + // so the early block's scripts are among the first commit's derivations. + let (block_1, filter_1, key_1) = block_paying(110, &addresses[0..=29]); + let (block_2, filter_2, key_2) = block_paying(210, &addresses[30..=59]); + let (block_3, filter_3, key_3) = block_paying(310, &addresses[60..=89]); + + // Persist headers and filters for the committed prefix 0..=299 — the + // range the drain-time sweep reloads from storage. Real data at the + // three block heights, filler elsewhere. + { + let mut header_storage = manager.header_storage.write().await; + let mut filter_storage = manager.filter_storage.write().await; + for height in 0..=299u32 { + let (header, filter_bytes) = match height { + 10 => (block_early.header, filter_early.content.clone()), + 110 => (block_1.header, filter_1.content.clone()), + 210 => (block_2.header, filter_2.content.clone()), + _ => { + let filler = Block::dummy(height, vec![]); + let filter = BlockFilter::dummy(&filler); + (filler.header, filter.content) + } + }; + header_storage + .store_headers_at_height(&[header.into()], height) + .await + .expect("seed header"); + filter_storage.store_filter(height, &filter_bytes).await.expect("seed filter"); + } + } + + let blocks: HashMap = HashMap::from([ + (block_early.block_hash(), block_early), + (block_1.block_hash(), block_1), + (block_2.block_hash(), block_2), + (block_3.block_hash(), block_3), + ]); + + for (start, filters) in [ + (0u32, HashMap::from([(key_early, filter_early)])), + (100, HashMap::from([(key_1, filter_1)])), + (200, HashMap::from([(key_2, filter_2)])), + (300, HashMap::from([(key_3, filter_3)])), + ] { + let mut batch = FiltersBatch::new(start, start + 99, filters); + batch.mark_verified(); + manager.active_batches.insert(start, batch); + } + manager.progress.update_stored_height(399); + // Both halves of the drain gate have to be live, or the test only proves + // the `active_batches.len() == 1` half: with the tips left at their + // default 0, `end_height() >= filter_header_tip_height()` is trivially + // true and the comparison that keeps the sweep from firing while more + // batches are still to come is never exercised. Same for the target + // height, which `FiltersSyncComplete` is checked against below. + manager.progress.update_filter_header_tip_height(399); + manager.progress.update_target_height(399); + + // Drive the production event loop to quiescence like `drive_to_quiescence` + // does, additionally watching for `FiltersSyncComplete` so the completion + // contract can be asserted at the moment it is emitted. + let (tx, _rx) = unbounded_channel(); + let requests = RequestSender::new(tx); + let mut events = manager.try_process_batch().await.unwrap(); + let mut sync_complete_seen = false; + 'rounds: for _round in 0..64 { + let mut pending: BTreeMap<(u32, BlockHash), BTreeSet> = BTreeMap::new(); + for event in events.drain(..) { + match event { + SyncEvent::BlocksNeeded { + blocks: needed, + } => { + for (key, wallets) in needed { + pending.entry((key.height(), *key.hash())).or_default().extend(wallets); + } + } + SyncEvent::FiltersSyncComplete { + .. + } => { + let (highest_used, _, _) = coinjoin_pool_state(&wallet, &wallet_id).await; + assert_eq!( + highest_used, + Some((G + 21) as u32), + "FiltersSyncComplete emitted before the deferred committed-range \ + sweep recovered the height-10 block: a script derived after its \ + range committed was never tested against the committed prefix" + ); + sync_complete_seen = true; + } + _ => {} + } + } + if pending.is_empty() { + break 'rounds; + } + for ((height, block_hash), wallets) in pending { + let block = blocks.get(&block_hash).expect("BlocksNeeded for an unknown test block"); + let result = wallet + .write() + .await + .process_block_for_wallets(block, block_hash, height, &wallets) + .await; + let confirmed_txids = result.relevant_txids().cloned().collect(); + let event = SyncEvent::BlockProcessed { + block_hash, + height, + wallets, + new_scripts: result.new_scripts, + confirmed_txids, + }; + events.extend( + manager.handle_sync_event(&event, &requests).await.expect("BlockProcessed"), + ); + } + } + + assert!(sync_complete_seen, "the run must reach FiltersSyncComplete"); + + let (highest_used, _, used_count) = coinjoin_pool_state(&wallet, &wallet_id).await; + assert_eq!( + highest_used, + Some((G + 21) as u32), + "the height-10 block's beyond-window outputs must be recovered by the \ + drain-time committed-range sweep (used_count={used_count})" + ); + assert_eq!(used_count, 90 + 12, "indices 0..=89 and G+10..=G+21 must all be marked used"); + + assert!( + manager.committed_range_sweeps >= 1, + "the deferred committed-range sweep must still run before completion" + ); + assert!( + manager.committed_range_sweeps <= 2, + "three script-carrying batch commits must share the committed-range sweep \ + (one drain-time sweep plus one follow-up round for the scripts derived from \ + the recovered block); got {} sweeps — one sweep per commit means the \ + coalescing regressed", + manager.committed_range_sweeps + ); +} diff --git a/dash-spv/src/sync/filters/manager.rs b/dash-spv/src/sync/filters/manager.rs index 2464466b2..f5e0cdf05 100644 --- a/dash-spv/src/sync/filters/manager.rs +++ b/dash-spv/src/sync/filters/manager.rs @@ -7,7 +7,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::sync::Arc; -use dashcore::bip158::BlockFilter; +use dashcore::bip158::{BlockFilter, FilterQuery}; use dashcore::ScriptBuf; use super::batch::FiltersBatch; @@ -23,13 +23,33 @@ use crate::validation::{FilterValidationInput, FilterValidator, Validator}; use crate::sync::progress::ProgressPercentage; use dashcore::hash_types::FilterHeader; use key_wallet_manager::WalletInterface; -use key_wallet_manager::{check_compact_filters_for_script_pubkeys, FilterMatchKey, WalletId}; +use key_wallet_manager::{check_compact_filters_for_elements, FilterMatchKey, WalletId}; use tokio::sync::RwLock; /// Batch size for processing filters. const BATCH_PROCESSING_SIZE: u32 = 5000; +/// Snapshot of a behind wallet's compact-filter query inputs for a batch scan. +struct WalletScanState { + /// The wallet these inputs belong to. + id: WalletId, + /// The wallet's committed sync checkpoint; heights at or below it are skipped. + synced: u32, + /// Monitored scriptPubKeys. + scripts: Vec, + /// Bare `hash160` filter elements (owner/voting key hashes) a compact + /// filter carries beyond the scriptPubKeys. + elements: Vec>, +} + /// Maximum number of batches to scan ahead while waiting for blocks. +/// +/// Raising this does not buy more throughput while a batch is stalled on a block. +/// Every scanned batch queues its matched blocks into one FIFO download window, +/// so a deeper lookahead fills that window with blocks belonging to batches that +/// cannot commit for a long time, ahead of the blocks the committing batch is +/// actually waiting on. Block downloads would have to be ordered by height +/// before this could go higher. const MAX_LOOKAHEAD_BATCHES: usize = 3; /// Filters manager for downloading and matching compact block filters. @@ -71,6 +91,21 @@ pub struct FiltersManager< /// `BlockProcessed` and the per-wallet record of which wallets already /// have a given processed block applied. pub(super) tracker: BlockMatchTracker, + /// Scripts already forward-rescanned but still awaiting the combined + /// backward sweep over the committed range (#846). Held at the manager + /// level and carried across batch commits: intermediate commits only + /// accumulate here, and `try_commit_batches` runs the sweep when the + /// forward pipeline drains — the committing batch is the last one active + /// and no lookahead can extend past it — so script-carrying commits share + /// one walk of the stored history instead of walking it once per commit. + /// Deliberately survives `reset_for_rescan`: the restarted scan covers + /// heights above its entry point, while these scripts still owe a pass + /// over the committed prefix below it. + backward_scripts: HashMap>, + /// Number of committed-range sweeps that reached the chunk walk in + /// `rescan_committed_range`. Diagnostic counter; the sweep-coalescing + /// regression test asserts on it. + pub(super) committed_range_sweeps: u64, } impl @@ -115,11 +150,17 @@ impl bool { + /// + /// `pub(super)` so the resume decision in `start_sync` can be taken on the + /// same predicate `start_download` asserts. Checking a subset there is how + /// a disconnect/reconnect cycle used to reach that assert. + pub(super) fn is_idle(&self) -> bool { self.active_batches.is_empty() && self.tracker.is_empty() && self.pending_batches.is_empty() @@ -136,6 +177,11 @@ impl SyncResult> { debug_assert!(self.is_idle(), "manager should have no in-flight state on start"); + // Coverage certified before this process is certified maturity too, so + // lift each wallet's clock to its own checkpoint. A boot that is already + // at the tip commits no batch, and nothing else would move it + // (dashpay/rust-dashcore#995). + { + let mut wallet = self.wallet.write().await; + // Every wallet the manager holds: none can sit at `u32::MAX`. + for wallet_id in wallet.wallets_behind(u32::MAX) { + let synced = wallet.wallet_synced_height(&wallet_id); + if synced > 0 { + wallet.update_wallet_last_processed_height(&wallet_id, synced); + } + } + } + // Use synced_height for restart recovery instead of // last_processed_height, which advances per-block and may exceed committed scan progress. let (wallet_birth_height, wallet_committed_height) = { @@ -175,8 +236,17 @@ impl