Add opt-in public pool snapshots for Pali privacy preparation

.env.example

@@ -93,3 +93,11 @@ SYSCOIN_BLOCKBOOK_URL=
 # Requires Core's protx_capabilities and extended protx_operator_key_info RPCs.
 # Blank disables management. No operator keys belong in this file or service.
 SYSCOIN_SENTRY_GENESIS_HASH=
+
+# Optional public Tornado Classic ETH pool index. Disabled by default.
+# Set BOTH values to opt in. The RPC must support Ethereum mainnet finalized
+# headers, historical state and eth_getLogs from the 2019 deployment blocks.
+# HTTPS only; no wallet keys or caller-supplied endpoints. No RPC runs until a
+# public snapshot request arrives. See docs/privacy-index.md for lifecycle.
+PRIVACY_INDEX_ENABLED=false
+PRIVACY_INDEX_RPC_URL=

.gitignore

@@ -5,5 +5,6 @@ coverage/
 *.sqlite
 *.sqlite-journal
 data/sysnode.db*
+data/privacy-index/
 .DS_Store
 npm-debug.log*

README.md

@@ -21,6 +21,8 @@ should use lowercase.
 - `GET  /mnlist` — fresh `masternode_list` RPC passthrough
 - `POST /mnsearch` — paginated/searched view over the in-memory tracker snapshot
 - `POST /govlist` — active + historical governance proposals (Syscoin Core `gobject list`)
+- `GET /api/privacy/v1/pools/:poolId/snapshot` — opt-in anonymous Classic ETH
+  pool snapshots for local wallet verification; see [configuration and protocol](docs/privacy-index.md).
 
 Authenticated routes (cookie + CSRF, same-site):
 

deploy/nginx/sysnode.conf.example

@@ -69,6 +69,13 @@ server {
     location /recovery/ { proxy_pass http://127.0.0.1:3001; include /etc/nginx/snippets/sysnode-proxy.conf; }
     location /api/recovery/ { proxy_pass http://127.0.0.1:3001; include /etc/nginx/snippets/sysnode-proxy.conf; }
     location /api/sentry/v1/ { proxy_pass http://127.0.0.1:3001; include /etc/nginx/snippets/sysnode-proxy.conf; }
+    # Anonymous pool-wide snapshots. Preserve the path and avoid access logs
+    # linking a requesting IP to a pool. No note/leaf parameters are supported.
+    location /api/privacy/v1/ {
+        access_log off;
+        proxy_pass http://127.0.0.1:3001;
+        include /etc/nginx/snippets/sysnode-proxy.conf;
+    }
 
     # Public anonymous data routes. Canonical casing is lowercase
     # (matches the historical `https://syscoin.dev/mnstats` URL still

docs/privacy-index.md

@@ -0,0 +1,110 @@
+# Public Classic ETH pool snapshots
+
+`GET /api/privacy/v1/pools/:poolId/snapshot` supplies public deposit data for
+Pali's local Tornado Classic Merkle-tree reconstruction. It never receives a
+note, commitment, nullifier, leaf position, account address, signing key, RPC
+URL or arbitrary contract address. Every query string or request body is
+rejected. Only the following Ethereum mainnet pools are accepted:
+
+| ID | Pool address | Deployment block |
+| --- | --- | --- |
+| `eth-0.1` | `0x12d66f87a04a9e220743712ce6d9bb1b5616b8fc` | 9116966 |
+| `eth-1` | `0x47ce0c6ed5b0ce3d3a51fdb1c52dc66a7c3c2936` | 9117609 |
+| `eth-10` | `0x910cbd523d972eb0a6f4cae4618ad62622b39dbf` | 9117720 |
+| `eth-100` | `0xa160cdab225685da1d56aa342ad8841c3b53f291` | 9161895 |
+
+These are pinned from the original
+[CLI registry](https://github.com/tornadocash/tornado-cli/blob/378ddf8b8b92a4924037d7b64a94dbfd5a7dd6e8/config.js).
+
+## Configuration and deployment
+
+The feature is disabled unless `PRIVACY_INDEX_ENABLED=true` and
+`PRIVACY_INDEX_RPC_URL` supplies an operator-controlled HTTPS Ethereum mainnet
+RPC. URLs containing credentials or fragments are rejected. A provider API
+token may be in its path/query; it never enters responses or application logs.
+The RPC must support `finalized` headers, historical `eth_call`/`eth_getCode`,
+and `eth_getLogs` from the deployment blocks. Scans start with 5,000-block
+ranges and shrink on provider range/response-size failures. There is no
+fallback to unfinalized data or to a different provider. A provider which cannot
+serve even one block of historical logs needs a suitable endpoint before enabling.
+
+No RPC is sent at startup. The first request starts a job, bounded to 16 chunks
+and a 45-second scheduling budget (an already-started RPC can take another
+10 seconds). A global queue runs only one job at a time; requests for the same
+pool share it. Further requests resume incomplete work. Successful snapshots
+are checked for refresh at most once a minute; failures back off for 30 seconds.
+Per-process IP limits allow 60 requests/minute and apply IPv6 subnet bucketing.
+Run one writer process for this file store; multiple workers require explicit
+shared locking and shared request limits before deployment.
+
+The backend process needs write access to `data/privacy-index/`, resolved from
+its working directory. Checkpoints and complete snapshots live in separate
+files with checksums and are published by same-directory atomic rename. They
+contain only public data, can be rebuilt, and never change `sysnode.db` or its
+schema. The existing deployment preserves `data/`. The feature needs no SQL
+migration, but the opt-in environment values and nginx route must be configured
+separately; automatic deployment does not modify the host environment or nginx.
+See the `/api/privacy/v1/` location in the nginx example. It disables access
+logging for this path; any upstream CDN/load balancer also needs an equivalent
+policy if operators want to avoid recording client IP/pool access metadata.
+The endpoint is anonymous with wildcard CORS and does not parse sessions or
+set cookies. This does not conceal a client's IP from the serving operator.
+
+## Response contract
+
+An initial or incomplete index returns HTTP 202:
+
+```json
+{"phase":"syncing","scannedBlock":9116965,"startBlock":9116966,"targetBlock":9116965}
+```
+
+Before the first RPC completes, `targetBlock` equals `startBlock - 1`; it is
+then the pinned finalized target for that scan. `scannedBlock` is the last
+fully accepted block, so initially it also equals `startBlock - 1`.
+
+HTTP 200 contains only:
+
+```text
+{ version: 1, chainId: 1, poolAddress, asOfBlockNumber, asOfBlockHash,
+  leafCount, root, leaves: ["0x<64 lowercase hex digits>", ...] }
+```
+
+Deposits are ordered by canonical event position and must have contiguous
+indices starting at zero. The server validates field ranges, the pool's
+20-level configuration and denomination, and matches total count and root
+metadata at the pinned finalized block. It rejects removed/malformed logs,
+missing indices, count mismatches and changed block anchors. JSON-RPC responses
+have a 10-second timeout, 4-MiB bound, no redirects and exact response-ID checks.
+
+The server does **not** independently reconstruct the MiMC root or authenticate
+the configured RPC cryptographically. Pali must reconstruct the tree and check
+the returned root against its own trusted chain observation before use. This
+service is a public-data accelerator, not a replacement for wallet verification.
+
+Complete snapshots remain immutable while a refresh is incomplete or fails.
+`X-Privacy-Index-State` reports `ready`, `refreshing`, or `stale`; the block/hash
+in the response remain the observation anchor, not the current HTTP time.
+Strong ETags support `If-None-Match`/304 with the same freshness header.
+`Cache-Control: public, max-age=0, must-revalidate` requires revalidation.
+202/503 responses are not cached and carry `Retry-After`.
+
+A persisted snapshot is withheld after restart until its block/hash,
+on-chain count and root are checked again. Known checkpoint/snapshot reorgs
+discard the affected pool's files and restart from deployment. A gap discards
+unfinished work and retries from the last complete snapshot or deployment. An RPC
+outage does not destroy a previously validated in-memory snapshot; cold starts
+during an outage return 503. Invalid/corrupt files are never served.
+
+## Validation
+
+```sh
+npm test -- --runInBand tests/privacyIndex.test.js tests/privacyRpc.test.js tests/privacy.routes.test.js
+npm test -- --runInBand --ci
+npm run test:sentry-codec
+npm run audit:prod
+```
+
+Tests use synthetic RPCs and temporary directories and never make mainnet
+writes. They cover restart/resume, gaps, reorgs, unavailable sources,
+deduplication, atomic snapshot availability, malformed inputs and anonymous
+HTTP/ETag behavior. No deployment or live pool claim is part of these tests.

lib/appFactory.js

@@ -29,6 +29,8 @@ const { CREDENTIALED_PREFIXES, SENTRY_PREFIXES, RECOVERY_PREFIXES } = require('.
 const { createRecoveryRepository } = require('./recoveryRepository');
 const { createRecoveryMonitor } = require('./recoveryMonitor');
 const { createRecoveryRouter } = require('../routes/recovery');
+const { configuredPrivacyIndex } = require('./privacyIndex');
+const { PRIVACY_PREFIX, createPrivacyRouter } = require('../routes/privacy');
 
 function parseBooleanEnv(value, name) {
   if (value === undefined || value === '') return undefined;
@@ -363,6 +365,7 @@ function createApp({
   proposalRpc = null,
   sentryAdapter = null,
   recoveryChains = [],
+  privacyIndex = configuredPrivacyIndex(),
   // If true and a proposalRpc is wired, start the background
   // proposal dispatcher interval. Production: true. Tests: false
   // (they prefer to advance the dispatcher manually by calling
@@ -393,6 +396,8 @@ function createApp({
   const app = express();
   const recoveryMonitor = createRecoveryMonitor({repository:services.recoveryRepository,users:services.users,chains:recoveryChains,mailer,now});
   app.use(helmet());
+  // Public pool snapshots never enter the session or credentialed-CORS path.
+  app.use(PRIVACY_PREFIX, createPrivacyRouter({ index: privacyIndex }));
   app.use(cors({ origin: effectiveCorsOrigin, credentials: true }));
   app.use(express.json({ limit: '256kb' }));
   app.use(cookieParser());

lib/privacyIndex.js

@@ -0,0 +1,186 @@
+'use strict';
+
+const { createHash } = require('crypto');
+const { performance } = require('perf_hooks');
+const { getPool, MAX_LEAVES } = require('./privacyRegistry');
+const { createPrivacyRpc } = require('./privacyRpc');
+const { createPrivacyStore } = require('./privacyStore');
+
+// One process-wide queue. Only a request can enqueue work; no boot-time RPC,
+// timers, wallet records or account-specific queries are involved.
+function createPrivacyIndex({ rpc, store = createPrivacyStore(), now = Date.now,
+  chunkBlocks = 5000, maxChunks = 16, jobBudgetMs = 45000,
+  refreshMs = 60000, retryMs = 30000, log = () => {} } = {}) {
+  if (!rpc) throw new Error('privacy_rpc_required');
+  const states = new Map();
+  const queue = [];
+  let running = null;
+  async function stateFor(pool) {
+    if (!states.has(pool.id)) {
+      const state = { snapshot: null, encoded: null, etag: null, checkpoint: null,
+        candidate: null, syncing: false, retryAt: 0, error: false };
+      state.loaded = Promise.all([store.load(pool.id, 'snapshot'), store.load(pool.id, 'checkpoint')])
+        .then(([snapshot, checkpoint]) => { state.candidate = snapshot; state.checkpoint = checkpoint; });
+      states.set(pool.id, state);
+    }
+    const state = states.get(pool.id);
+    await state.loaded;
+    return state;
+  }
+  function publish(state, snapshot) {
+    state.snapshot = snapshot;
+    state.encoded = JSON.stringify(snapshot);
+    state.etag = `"${createHash('sha256').update(state.encoded).digest('hex')}"`;
+    state.candidate = null;
+  }
+  async function reset(pool, state) {
+    state.snapshot = state.candidate = state.checkpoint = state.encoded = state.etag = null;
+    await store.remove(pool.id, 'snapshot');
+    await store.remove(pool.id, 'checkpoint');
+  }
+  async function verifySaved(pool, snapshot, head) {
+    if (snapshot.asOfBlockNumber > head.number) throw new Error('privacy_reorg');
+    const header = await rpc.block(snapshot.asOfBlockNumber);
+    if (header.hash !== snapshot.asOfBlockHash) throw new Error('privacy_reorg');
+    const observed = await rpc.state(pool, header);
+    if (observed.nextIndex !== snapshot.leafCount || observed.root !== snapshot.root) throw new Error('privacy_reorg');
+  }
+  async function sync(pool, state) {
+    const deadline = performance.now() + jobBudgetMs;
+    try {
+      const head = await rpc.head();
+      if (head.number < pool.deploymentBlock) throw new Error('privacy_pool_not_deployed');
+      // In-memory snapshots are also checked before extending. On restart the
+      // saved snapshot is withheld until this independent canonical read passes.
+      const saved = state.snapshot || state.candidate;
+      if (saved) {
+        await verifySaved(pool, saved, head);
+        if (!state.snapshot) publish(state, saved);
+      }
+      let checkpoint = state.checkpoint;
+      if (checkpoint && state.snapshot && checkpoint.targetBlock < state.snapshot.asOfBlockNumber) checkpoint = null;
+      if (checkpoint) {
+        if (checkpoint.targetBlock > head.number ||
+            (await rpc.block(checkpoint.targetBlock)).hash !== checkpoint.targetHash ||
+            (checkpoint.scannedBlock >= pool.deploymentBlock &&
+             (await rpc.block(checkpoint.scannedBlock)).hash !== checkpoint.scannedHash)) throw new Error('privacy_reorg');
+      } else if (state.snapshot && state.snapshot.asOfBlockNumber === head.number) {
+        state.error = false;
+        state.retryAt = now() + refreshMs;
+        return;
+      } else {
+        checkpoint = {
+          version: 1, poolAddress: pool.address,
+          scannedBlock: state.snapshot?.asOfBlockNumber ?? pool.deploymentBlock - 1,
+          scannedHash: state.snapshot?.asOfBlockHash ?? null,
+          targetBlock: head.number, targetHash: head.hash,
+          leaves: state.snapshot ? state.snapshot.leaves.slice() : [],
+        };
+      }
+      state.checkpoint = checkpoint;
+      const target = await rpc.block(checkpoint.targetBlock);
+      if (target.hash !== checkpoint.targetHash) throw new Error('privacy_reorg');
+      const expected = await rpc.state(pool, target);
+      if (checkpoint.leaves.length > expected.nextIndex) throw new Error('privacy_index_gap');
+      let blockSpan = Math.min(checkpoint.blockSpan || chunkBlocks, chunkBlocks);
+      for (let chunk = 0; checkpoint.scannedBlock < target.number && chunk < maxChunks && performance.now() < deadline; chunk++) {
+        const from = checkpoint.scannedBlock + 1;
+        let to, boundary, rows;
+        for (;;) {
+          to = Math.min(target.number, from + blockSpan - 1);
+          boundary = await rpc.block(to);
+          try { rows = await rpc.deposits(pool, from, to); break; }
+          catch (error) {
+            // Read-only provider size/range limits can be retried on a smaller
+            // public range. The job deadline and a one-block floor bound this.
+            if (!['privacy_rpc_unavailable', 'privacy_rpc_rejected', 'privacy_logs_too_large'].includes(error.message) ||
+                to === from || performance.now() >= deadline) throw error;
+            blockSpan = Math.max(1, Math.floor((to - from + 1) / 2));
+          }
+        }
+        // Validate the complete chunk before mutating its checkpoint.
+        let next = checkpoint.leaves.length;
+        let previous = null;
+        for (const row of rows) {
+          if (row.leafIndex !== next || next >= expected.nextIndex || next >= MAX_LEAVES ||
+              (previous && row.blockNumber === previous.blockNumber && row.logIndex <= previous.logIndex) ||
+              (row.blockNumber === boundary.number && row.blockHash !== boundary.hash)) throw new Error('privacy_index_gap');
+          previous = row; next++;
+        }
+        await rpc.assertBlock(boundary);
+        checkpoint.leaves.push(...rows.map((row) => row.leaf));
+        checkpoint.scannedBlock = to;
+        checkpoint.scannedHash = boundary.hash;
+        checkpoint.blockSpan = blockSpan;
+      }
+      // A reorg during any read cannot publish a mixed-fork view.
+      await rpc.assertBlock(target);
+      if (checkpoint.scannedBlock === target.number) {
+        if (checkpoint.leaves.length !== expected.nextIndex) throw new Error('privacy_index_gap');
+        const snapshot = { version: 1, chainId: 1, poolAddress: pool.address,
+          asOfBlockNumber: target.number, asOfBlockHash: target.hash,
+          leafCount: checkpoint.leaves.length, root: expected.root, leaves: checkpoint.leaves };
+        // The root comes from the pool at this exact block. Wallets must rebuild
+        // MiMC independently before trusting these public RPC-provided leaves.
+        await store.save(pool.id, 'snapshot', snapshot);
+        publish(state, snapshot);
+        state.checkpoint = null;
+        await store.remove(pool.id, 'checkpoint');
+        state.retryAt = now() + refreshMs;
+      } else {
+        await store.save(pool.id, 'checkpoint', checkpoint);
+        state.retryAt = 0; // the next explicit poll may advance another bounded job
+      }
+      state.error = false;
+    } catch (error) {
+      if (error.message === 'privacy_reorg') await reset(pool, state);
+      if (error.message === 'privacy_index_gap') {
+        // Missing new logs invalidate only unfinished work. A previously
+        // checked complete snapshot remains a useful, explicitly stale view.
+        state.checkpoint = null;
+        await store.remove(pool.id, 'checkpoint');
+      }
+      state.error = true;
+      state.retryAt = now() + retryMs;
+      log('privacy_sync_failed', { poolId: pool.id, code: /^privacy_[a-z_]+$/.test(error.message) ? error.message : 'privacy_index_unavailable' });
+    }
+  }
+  function enqueue(pool, state) {
+    if (state.syncing || now() < state.retryAt) return;
+    state.syncing = true;
+    queue.push({ pool, state });
+    if (running) return;
+    running = (async () => {
+      while (queue.length) {
+        const next = queue.shift();
+        try { await sync(next.pool, next.state); }
+        catch (_) { next.state.error = true; next.state.retryAt = now() + retryMs; }
+        finally { next.state.syncing = false; }
+      }
+    })().finally(() => { running = null; });
+  }
+  async function read(id) {
+    const pool = getPool(id);
+    if (!pool) return { status: 404, body: { error: 'unknown_privacy_pool' } };
+    let state;
+    try { state = await stateFor(pool); }
+    catch (_) { return { status: 503, body: { error: 'privacy_index_unavailable' } }; }
+    enqueue(pool, state);
+    if (state.snapshot) return { status: 200, body: state.snapshot, encoded: state.encoded, etag: state.etag,
+      phase: state.error ? 'stale' : state.syncing ? 'refreshing' : 'ready' };
+    if (state.error) return { status: 503, body: { error: 'privacy_index_unavailable' } };
+    return { status: 202, body: { phase: 'syncing', scannedBlock: state.checkpoint?.scannedBlock ?? pool.deploymentBlock - 1,
+      startBlock: pool.deploymentBlock, targetBlock: state.checkpoint?.targetBlock ?? pool.deploymentBlock - 1 } };
+  }
+  return { read, whenIdle: () => running || Promise.resolve() };
+}
+
+function configuredPrivacyIndex({ enabled = process.env.PRIVACY_INDEX_ENABLED,
+  rpcUrl = process.env.PRIVACY_INDEX_RPC_URL, ...options } = {}) {
+  if (enabled !== undefined && enabled !== '' && enabled !== 'true' && enabled !== 'false') throw new Error('PRIVACY_INDEX_ENABLED_invalid');
+  if (enabled !== 'true') return null;
+  if (!rpcUrl) throw new Error('PRIVACY_INDEX_RPC_URL_required');
+  return createPrivacyIndex({ rpc: createPrivacyRpc(rpcUrl), ...options });
+}
+
+module.exports = { createPrivacyIndex, configuredPrivacyIndex };