From 2ee472e893f5b02c86b4f026e6f8f6cef2ec37cc Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 22:57:49 -0700 Subject: [PATCH 01/13] feat(config): add paymentSplits flag, default off Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WxkKbrg2Xohf3mQj7BLrs6 --- dev/config/base-config.yaml | 7 +++++++ src/config/schema.ts | 24 ++++++++++++++++++++++++ src/config/schema.types.d.ts | 8 ++++++++ src/config/yaml.ts | 7 +++++++ 4 files changed, 46 insertions(+) diff --git a/dev/config/base-config.yaml b/dev/config/base-config.yaml index c3827d0b8..5d065c6d3 100644 --- a/dev/config/base-config.yaml +++ b/dev/config/base-config.yaml @@ -56,6 +56,13 @@ fygaro: lookbackDays: 7 maxPerSweep: 20 +# ENG-653: Flash-native payment splits for incoming payments. +paymentSplits: + enabled: false # flags-off baseline; enable per environment via config overrides + executorBatchSize: 50 + maxAttempts: 5 + minShareCents: 1 + bridge: enabled: false # flags-off baseline; enable per environment via config overrides apiKey: "" # real key injected via config overrides; never commit a real key diff --git a/src/config/schema.ts b/src/config/schema.ts index 6ab853029..0a08a4c59 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -1121,6 +1121,30 @@ export const configSchema = { // Default OFF baseline (mirrors topup); enable per environment via overrides. default: { enabled: false }, }, + // ENG-653: Flash-native payment splits for incoming payments. With this OFF + // the webhook middleware records nothing, the cron tasks no-op and the + // walletSplitsSet / walletSplitDelete mutations reject. + paymentSplits: { + type: "object", + properties: { + enabled: { type: "boolean", default: false }, + // Split executions the cron executor claims and pays per run. + executorBatchSize: { type: "integer", minimum: 1, default: 50 }, + // Sends attempted per execution before a retryable error becomes a + // terminal `failed`. Backoff between attempts is 1, 2, 4, 8... minutes. + maxAttempts: { type: "integer", minimum: 1, default: 5 }, + // A share below this many US cents is recorded as `skipped_min` and + // never sent. + minShareCents: { type: "integer", minimum: 1, default: 1 }, + }, + additionalProperties: false, + default: { + enabled: false, + executorBatchSize: 50, + maxAttempts: 5, + minShareCents: 1, + }, + }, frappe: { type: "object", properties: { diff --git a/src/config/schema.types.d.ts b/src/config/schema.types.d.ts index 6b76f6639..5f2643ea8 100644 --- a/src/config/schema.types.d.ts +++ b/src/config/schema.types.d.ts @@ -128,6 +128,13 @@ type FygaroConfig = { } } +type PaymentSplitsConfig = { + enabled: boolean + executorBatchSize: number + maxAttempts: number + minShareCents: number +} + type CashoutEmail = { to: string from: string @@ -308,6 +315,7 @@ type YamlSchema = { enabled: boolean } fygaro: FygaroConfig + paymentSplits?: PaymentSplitsConfig sendgrid: SendGridConfig frappe: FrappeConfig fcmTopics: { diff --git a/src/config/yaml.ts b/src/config/yaml.ts index 6f5dbd34d..43d8a8b88 100644 --- a/src/config/yaml.ts +++ b/src/config/yaml.ts @@ -645,4 +645,11 @@ export const BridgeConfig = yamlConfig.bridge as BridgeConfig export const FygaroConfig = yamlConfig.fygaro as FygaroConfig +export const PaymentSplitsConfig = (yamlConfig.paymentSplits ?? { + enabled: false, + executorBatchSize: 50, + maxAttempts: 5, + minShareCents: 1, +}) as PaymentSplitsConfig + export const FrappeConfig = yamlConfig.frappe as FrappeConfig From 27d383db1d9381da5485d7b4413f288a24128b31 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 22:57:49 -0700 Subject: [PATCH 02/13] feat(payment-splits): split-set validation, share math and errors Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WxkKbrg2Xohf3mQj7BLrs6 --- src/app/errors.ts | 2 + src/domain/payment-splits/compute-share.ts | 63 ++++++++++++++ src/domain/payment-splits/errors.ts | 30 +++++++ src/domain/payment-splits/index.ts | 28 ++++++ src/domain/payment-splits/validate.ts | 80 +++++++++++++++++ src/graphql/error-map.ts | 23 +++++ .../payment-splits/compute-share.spec.ts | 71 +++++++++++++++ .../domain/payment-splits/validate.spec.ts | 86 +++++++++++++++++++ 8 files changed, 383 insertions(+) create mode 100644 src/domain/payment-splits/compute-share.ts create mode 100644 src/domain/payment-splits/errors.ts create mode 100644 src/domain/payment-splits/index.ts create mode 100644 src/domain/payment-splits/validate.ts create mode 100644 test/flash/unit/domain/payment-splits/compute-share.spec.ts create mode 100644 test/flash/unit/domain/payment-splits/validate.spec.ts diff --git a/src/app/errors.ts b/src/app/errors.ts index 1fabd9921..1b68b60b6 100644 --- a/src/app/errors.ts +++ b/src/app/errors.ts @@ -32,6 +32,7 @@ import * as ErpNextErrors from "@services/frappe/errors" import * as BridgeErrors from "@services/bridge/errors" import * as FygaroErrors from "@services/fygaro/errors" import * as ApiKeyErrors from "@domain/api-keys/errors" +import * as PaymentSplitErrors from "@domain/payment-splits/errors" export const ApplicationErrors = { ...SharedErrors, @@ -69,4 +70,5 @@ export const ApplicationErrors = { ...BridgeErrors, ...FygaroErrors, ...ApiKeyErrors, + ...PaymentSplitErrors, } as const diff --git a/src/domain/payment-splits/compute-share.ts b/src/domain/payment-splits/compute-share.ts new file mode 100644 index 000000000..7ddd6059c --- /dev/null +++ b/src/domain/payment-splits/compute-share.ts @@ -0,0 +1,63 @@ +import { WalletCurrency } from "@domain/shared" + +const MINOR_UNITS_PER_MAJOR: Record = { + [WalletCurrency.Usd]: 100n, // cents + [WalletCurrency.Usdt]: 1_000_000n, // micros +} + +// Minor units per US cent: the send API (usdWalletAmountFromInput) takes cents. +const MINOR_UNITS_PER_CENT: Record = { + [WalletCurrency.Usd]: 1n, + [WalletCurrency.Usdt]: 10_000n, +} + +export type SplitShare = + | { status: "ok"; triggerAmountMinor: string; shareCents: number } + | { status: "skipped_min"; triggerAmountMinor: string; shareCents: number } + +export const isSplitCurrency = (currency: string): boolean => + currency === WalletCurrency.Usd || currency === WalletCurrency.Usdt + +/** + * The share of an incoming payment one split rule pays out, in whole US cents. + * + * `amount` is the receive amount in major units as IBEX reports it (dollars for + * USD, USDT for USDT). It is converted to minor units the same way the receive + * notification does (on-receive.ts toPaymentAmount), the percentage is floored + * in minor units, and the result is floored again to whole cents. Every + * rounding step rounds the share DOWN, so the splits of one receive can never + * add up to more than the receive itself. + */ +export const computeShare = ({ + amount, + currency, + percent, + minShareCents, +}: { + amount: number + currency: string + percent: number + minShareCents: number +}): SplitShare | Error => { + const perMajor = MINOR_UNITS_PER_MAJOR[currency] + const perCent = MINOR_UNITS_PER_CENT[currency] + if (perMajor === undefined || perCent === undefined) { + return new Error(`Unsupported split currency: ${currency}`) + } + if (!Number.isFinite(amount) || amount < 0) { + return new Error(`Invalid split trigger amount: ${amount}`) + } + if (!Number.isInteger(percent) || percent < 1 || percent > 100) { + return new Error(`Invalid split percent: ${percent}`) + } + + const triggerMinor = BigInt(Math.round(amount * Number(perMajor))) + const shareMinor = (triggerMinor * BigInt(percent)) / 100n + const shareCents = Number(shareMinor / perCent) + const triggerAmountMinor = triggerMinor.toString() + + if (shareCents < minShareCents) { + return { status: "skipped_min", triggerAmountMinor, shareCents } + } + return { status: "ok", triggerAmountMinor, shareCents } +} diff --git a/src/domain/payment-splits/errors.ts b/src/domain/payment-splits/errors.ts new file mode 100644 index 000000000..88994ca59 --- /dev/null +++ b/src/domain/payment-splits/errors.ts @@ -0,0 +1,30 @@ +import { DomainError, ErrorLevel, ValidationError } from "@domain/shared" + +// ENG-653 payment splits. Every class exported here is an ApplicationError key +// and therefore needs a case in src/graphql/error-map.ts. + +// The percentages of a wallet's split set must sum to more than 0 and at most +// 100 (an empty set sums to 0). +export class SplitPercentExceededError extends ValidationError {} +// A single percent is not an integer in [1, 100]. +export class InvalidSplitPercentError extends ValidationError {} +// More rules than MAX_SPLIT_RULES_PER_WALLET in one set. +export class SplitRuleLimitExceededError extends ValidationError {} +// The destination username or walletId does not resolve to a wallet in the +// source wallet's currency. +export class SplitDestinationNotFoundError extends ValidationError {} +export class SelfSplitNotAllowedError extends ValidationError {} +// The destination wallet's currency differs from the source wallet's, or the +// source wallet is not a USD/USDT wallet. send-intraledger rejects +// cross-currency sends, so such a rule could never pay out. +export class SplitCurrencyMismatchError extends ValidationError {} +export class SplitDuplicateDestinationError extends ValidationError {} +export class SplitRuleNotFoundError extends ValidationError {} +export class PaymentSplitsDisabledError extends ValidationError {} + +// The internal-transfer marker could not be persisted before paying an +// intraledger invoice. The send is refused (no money moved) so that the +// recipient's split rules can never fire on a system credit. +export class InternalTransferMarkerError extends DomainError { + level = ErrorLevel.Critical +} diff --git a/src/domain/payment-splits/index.ts b/src/domain/payment-splits/index.ts new file mode 100644 index 000000000..34be9051d --- /dev/null +++ b/src/domain/payment-splits/index.ts @@ -0,0 +1,28 @@ +export * from "./errors" +export * from "./compute-share" +export * from "./validate" + +export const PaymentSplitExecutionStatus = { + Pending: "pending", + Sending: "sending", + Sent: "sent", + IbexPending: "ibex_pending", + Failed: "failed", + SkippedMin: "skipped_min", + Unknown: "unknown", +} as const + +export type PaymentSplitExecutionStatus = + (typeof PaymentSplitExecutionStatus)[keyof typeof PaymentSplitExecutionStatus] + +export const InternalTransferOrigin = { + Intraledger: "intraledger", + Cutover: "cutover", +} as const + +export type InternalTransferOrigin = + (typeof InternalTransferOrigin)[keyof typeof InternalTransferOrigin] + +// Backoff before attempt n+1 after the nth failed attempt: 1, 2, 4, 8, 16 min. +export const splitRetryBackoffMs = (failedAttempts: number): number => + 60_000 * 2 ** Math.max(0, failedAttempts - 1) diff --git a/src/domain/payment-splits/validate.ts b/src/domain/payment-splits/validate.ts new file mode 100644 index 000000000..d9280f156 --- /dev/null +++ b/src/domain/payment-splits/validate.ts @@ -0,0 +1,80 @@ +import { ValidationError } from "@domain/shared" + +import { + InvalidSplitPercentError, + SelfSplitNotAllowedError, + SplitCurrencyMismatchError, + SplitDuplicateDestinationError, + SplitPercentExceededError, + SplitRuleLimitExceededError, +} from "./errors" +import { isSplitCurrency } from "./compute-share" + +export const MAX_SPLIT_RULES_PER_WALLET = 10 + +export type SplitSetEntry = { + destinationWalletId: string + destinationCurrency: string + percent: number +} + +/** + * Validates a complete split set for one source wallet. The set replaces any + * existing rules, so the percent sum is checked over this set alone. + */ +export const validateSplitSet = ({ + sourceWalletId, + sourceCurrency, + entries, + maxRules = MAX_SPLIT_RULES_PER_WALLET, +}: { + sourceWalletId: string + sourceCurrency: string + entries: SplitSetEntry[] + maxRules?: number +}): true | ValidationError => { + if (!isSplitCurrency(sourceCurrency)) { + return new SplitCurrencyMismatchError("Splits are only supported on USD wallets") + } + + if (entries.length > maxRules) { + return new SplitRuleLimitExceededError( + `A wallet can have at most ${maxRules} payment splits`, + ) + } + + for (const { percent } of entries) { + if (!Number.isInteger(percent) || percent < 1 || percent > 100) { + return new InvalidSplitPercentError( + "Each split percent must be a whole number from 1 to 100", + ) + } + } + + const sum = entries.reduce((total, { percent }) => total + percent, 0) + if (sum <= 0 || sum > 100) { + return new SplitPercentExceededError( + "Split percentages must add up to more than 0 and at most 100", + ) + } + + const seen = new Set() + for (const { destinationWalletId, destinationCurrency } of entries) { + if (destinationWalletId === sourceWalletId) { + return new SelfSplitNotAllowedError("A wallet cannot split payments to itself") + } + if (seen.has(destinationWalletId)) { + return new SplitDuplicateDestinationError( + "Each split destination can appear only once", + ) + } + seen.add(destinationWalletId) + if (destinationCurrency !== sourceCurrency) { + return new SplitCurrencyMismatchError( + "Split destination wallet must be in the same currency as the source wallet", + ) + } + } + + return true +} diff --git a/src/graphql/error-map.ts b/src/graphql/error-map.ts index 51e1f189c..9d0b6cef4 100644 --- a/src/graphql/error-map.ts +++ b/src/graphql/error-map.ts @@ -88,6 +88,29 @@ export const mapError = (error: ApplicationError): CustomApolloError => { message = "Sending is temporarily unavailable, please try again later." return new TransactionRestrictedError({ message, logger: baseLogger }) + // ENG-653 payment splits: validation messages are written for the user. + case "SplitPercentExceededError": + case "InvalidSplitPercentError": + case "SplitRuleLimitExceededError": + case "SplitDestinationNotFoundError": + case "SelfSplitNotAllowedError": + case "SplitCurrencyMismatchError": + case "SplitDuplicateDestinationError": + message = error.message + return new ValidationInternalError({ message, logger: baseLogger }) + + case "SplitRuleNotFoundError": + message = "Payment split not found" + return new NotFoundError({ message, logger: baseLogger }) + + case "PaymentSplitsDisabledError": + message = "Payment splits are not available" + return new ValidationInternalError({ message, logger: baseLogger }) + + case "InternalTransferMarkerError": + message = "Sending is temporarily unavailable, please try again later." + return new UnexpectedClientError({ message, logger: baseLogger }) + case "CompletedInvoice": case "AlreadyPaidError": message = "Invoice is already paid" diff --git a/test/flash/unit/domain/payment-splits/compute-share.spec.ts b/test/flash/unit/domain/payment-splits/compute-share.spec.ts new file mode 100644 index 000000000..25f330c0d --- /dev/null +++ b/test/flash/unit/domain/payment-splits/compute-share.spec.ts @@ -0,0 +1,71 @@ +import { computeShare, splitRetryBackoffMs } from "@domain/payment-splits" +import { WalletCurrency } from "@domain/shared" + +const share = (amount: number, currency: string, percent: number, minShareCents = 1) => + computeShare({ amount, currency, percent, minShareCents }) + +describe("computeShare", () => { + it("floors a USD share to whole cents: 10% of $0.99 is 9 cents", () => { + expect(share(0.99, WalletCurrency.Usd, 10)).toEqual({ + status: "ok", + triggerAmountMinor: "99", + shareCents: 9, + }) + }) + + it("floors a USDT share in micros, then to cents: 10% of 1.234567 USDT is 12 cents", () => { + expect(share(1.234567, WalletCurrency.Usdt, 10)).toEqual({ + status: "ok", + triggerAmountMinor: "1234567", + shareCents: 12, + }) + }) + + it("pays 100% of a whole-cent USD receive exactly", () => { + expect(share(19.46, WalletCurrency.Usd, 100)).toMatchObject({ shareCents: 1946 }) + }) + + it("never rounds a share up across a set", () => { + // 3 x 33% of $1.00 = 99 cents, never more than the receive. + const shares = [33, 33, 33].map((p) => share(1, WalletCurrency.Usd, p)) + const total = shares.reduce( + (sum, s) => sum + (s instanceof Error ? 0 : s.shareCents), + 0, + ) + expect(total).toBeLessThanOrEqual(100) + }) + + it("marks a sub-1-cent share skipped_min", () => { + expect(share(0.05, WalletCurrency.Usd, 10)).toEqual({ + status: "skipped_min", + triggerAmountMinor: "5", + shareCents: 0, + }) + expect(share(0.000099, WalletCurrency.Usdt, 100)).toMatchObject({ + status: "skipped_min", + shareCents: 0, + }) + }) + + it("honours a higher minShareCents", () => { + expect(share(1, WalletCurrency.Usd, 10, 25)).toMatchObject({ + status: "skipped_min", + shareCents: 10, + }) + }) + + it("rejects BTC, negative amounts and invalid percents", () => { + expect(share(1, WalletCurrency.Btc, 10)).toBeInstanceOf(Error) + expect(share(-1, WalletCurrency.Usd, 10)).toBeInstanceOf(Error) + expect(share(1, WalletCurrency.Usd, 0)).toBeInstanceOf(Error) + expect(share(1, WalletCurrency.Usd, 10.5)).toBeInstanceOf(Error) + }) +}) + +describe("splitRetryBackoffMs", () => { + it("backs off 1, 2, 4, 8, 16 minutes", () => { + expect([1, 2, 3, 4, 5].map(splitRetryBackoffMs)).toEqual( + [1, 2, 4, 8, 16].map((m) => m * 60_000), + ) + }) +}) diff --git a/test/flash/unit/domain/payment-splits/validate.spec.ts b/test/flash/unit/domain/payment-splits/validate.spec.ts new file mode 100644 index 000000000..5b6000259 --- /dev/null +++ b/test/flash/unit/domain/payment-splits/validate.spec.ts @@ -0,0 +1,86 @@ +import { + InvalidSplitPercentError, + MAX_SPLIT_RULES_PER_WALLET, + SelfSplitNotAllowedError, + SplitCurrencyMismatchError, + SplitDuplicateDestinationError, + SplitPercentExceededError, + SplitRuleLimitExceededError, + validateSplitSet, +} from "@domain/payment-splits" +import { WalletCurrency } from "@domain/shared" + +const SOURCE = "source-wallet" + +const entry = (n: number, percent: number, currency: string = WalletCurrency.Usdt) => ({ + destinationWalletId: `dest-${n}`, + destinationCurrency: currency, + percent, +}) + +const validate = (entries: ReturnType[], sourceCurrency = "USDT") => + validateSplitSet({ sourceWalletId: SOURCE, sourceCurrency, entries }) + +describe("validateSplitSet", () => { + it("accepts a set that sums to exactly 100", () => { + expect(validate([entry(1, 60), entry(2, 40)])).toBe(true) + }) + + it("accepts a set below 100 (the remainder stays with the owner)", () => { + expect(validate([entry(1, 10)])).toBe(true) + }) + + it("rejects a set that sums to 101", () => { + expect(validate([entry(1, 60), entry(2, 41)])).toBeInstanceOf( + SplitPercentExceededError, + ) + }) + + it("rejects a 0 sum (the empty set)", () => { + expect(validate([])).toBeInstanceOf(SplitPercentExceededError) + }) + + it("rejects non-integer and out-of-range percents", () => { + expect(validate([entry(1, 12.5)])).toBeInstanceOf(InvalidSplitPercentError) + expect(validate([entry(1, 0)])).toBeInstanceOf(InvalidSplitPercentError) + expect(validate([entry(1, -5)])).toBeInstanceOf(InvalidSplitPercentError) + expect(validate([entry(1, 101)])).toBeInstanceOf(InvalidSplitPercentError) + }) + + it("rejects a split to the source wallet itself", () => { + expect( + validateSplitSet({ + sourceWalletId: SOURCE, + sourceCurrency: "USDT", + entries: [ + { destinationWalletId: SOURCE, destinationCurrency: "USDT", percent: 5 }, + ], + }), + ).toBeInstanceOf(SelfSplitNotAllowedError) + }) + + it("rejects duplicate destinations", () => { + expect(validate([entry(1, 10), entry(1, 20)])).toBeInstanceOf( + SplitDuplicateDestinationError, + ) + }) + + it("rejects a destination in another currency", () => { + expect(validate([entry(1, 10, WalletCurrency.Usd)])).toBeInstanceOf( + SplitCurrencyMismatchError, + ) + }) + + it("rejects a BTC source wallet", () => { + expect( + validate([entry(1, 10, WalletCurrency.Btc)], WalletCurrency.Btc), + ).toBeInstanceOf(SplitCurrencyMismatchError) + }) + + it("accepts the maximum number of rules and rejects one more", () => { + const max = Array.from({ length: MAX_SPLIT_RULES_PER_WALLET }, (_, i) => entry(i, 1)) + expect(MAX_SPLIT_RULES_PER_WALLET).toBe(10) + expect(validate(max)).toBe(true) + expect(validate([...max, entry(99, 1)])).toBeInstanceOf(SplitRuleLimitExceededError) + }) +}) From 4d01e1f5c4b9bc10f6b5f461443d725e3558e4c4 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 22:57:49 -0700 Subject: [PATCH 03/13] feat(mongoose): split rule, execution and internal-transfer marker stores Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WxkKbrg2Xohf3mQj7BLrs6 --- .../mongoose/internal-transfer-invoices.ts | 52 ++++++ .../mongoose/payment-split-executions.ts | 156 ++++++++++++++++++ src/services/mongoose/payment-split-rules.ts | 118 +++++++++++++ src/services/mongoose/schema.ts | 79 +++++++++ .../payment-splits-repositories.spec.ts | 123 ++++++++++++++ 5 files changed, 528 insertions(+) create mode 100644 src/services/mongoose/internal-transfer-invoices.ts create mode 100644 src/services/mongoose/payment-split-executions.ts create mode 100644 src/services/mongoose/payment-split-rules.ts create mode 100644 test/flash/integration/payment-splits/payment-splits-repositories.spec.ts diff --git a/src/services/mongoose/internal-transfer-invoices.ts b/src/services/mongoose/internal-transfer-invoices.ts new file mode 100644 index 000000000..97735a0bd --- /dev/null +++ b/src/services/mongoose/internal-transfer-invoices.ts @@ -0,0 +1,52 @@ +import { parseRepositoryError } from "./utils" +import { InternalTransferInvoice } from "./schema" + +const isDuplicateKeyError = (error: unknown): boolean => + (error as { code?: number } | null)?.code === 11000 + +// Typed RepositoryErrors, so every failure maps cleanly through error-map. +const toError = (error: unknown): RepositoryError => parseRepositoryError(error) + +/** + * Marks a payment hash as an invoice Flash minted for one of its own transfers + * (ENG-653), so a receive on it never triggers the recipient's payment splits. + * Recording the same hash twice is a no-op. + */ +export const recordInternalTransferInvoice = async ({ + paymentHash, + senderWalletId, + recipientWalletId, + origin, +}: { + paymentHash: string + senderWalletId?: string + recipientWalletId: string + origin: "intraledger" | "cutover" +}): Promise => { + try { + await InternalTransferInvoice.create({ + paymentHash: paymentHash.toLowerCase(), + senderWalletId, + recipientWalletId, + origin, + createdAt: new Date(), + }) + return true + } catch (error) { + if (isDuplicateKeyError(error)) return true + return toError(error) + } +} + +export const isInternalTransferInvoice = async ( + paymentHash: string, +): Promise => { + try { + const found = await InternalTransferInvoice.exists({ + paymentHash: { $eq: paymentHash.toLowerCase() }, + }) + return found !== null + } catch (error) { + return toError(error) + } +} diff --git a/src/services/mongoose/payment-split-executions.ts b/src/services/mongoose/payment-split-executions.ts new file mode 100644 index 000000000..7de8a2ad8 --- /dev/null +++ b/src/services/mongoose/payment-split-executions.ts @@ -0,0 +1,156 @@ +import { parseRepositoryError } from "./utils" +import { PaymentSplitExecution as PaymentSplitExecutionModel } from "./schema" + +export type PaymentSplitExecutionStatusValue = + | "pending" + | "sending" + | "sent" + | "ibex_pending" + | "failed" + | "skipped_min" + | "unknown" + +export type PaymentSplitExecutionRecord = { + id: string + triggerKey: string + ruleId: string + walletId: string + destinationWalletId: string + currency: string + triggerAmountMinor: string + shareCents: number + status: PaymentSplitExecutionStatusValue + attempts: number + nextAttemptAt: Date + lastError?: string + createdAt: Date + updatedAt: Date +} + +type RawExecution = Omit & { + _id: { toString(): string } +} + +const isDuplicateKeyError = (error: unknown): boolean => + (error as { code?: number } | null)?.code === 11000 + +// Typed RepositoryErrors, so every failure maps cleanly through error-map. +const toError = (error: unknown): RepositoryError => parseRepositoryError(error) + +const toExecution = (raw: RawExecution): PaymentSplitExecutionRecord => ({ + id: raw._id.toString(), + triggerKey: raw.triggerKey, + ruleId: raw.ruleId, + walletId: raw.walletId, + destinationWalletId: raw.destinationWalletId, + currency: raw.currency, + triggerAmountMinor: raw.triggerAmountMinor, + shareCents: raw.shareCents, + status: raw.status, + attempts: raw.attempts, + nextAttemptAt: raw.nextAttemptAt, + lastError: raw.lastError, + createdAt: raw.createdAt, + updatedAt: raw.updatedAt, +}) + +export const PaymentSplitExecutionsRepository = () => { + /** + * Inserts one execution. A duplicate (triggerKey, ruleId) — a webhook replay, + * or the reconcile cron finding a receive the webhook already recorded — is a + * typed no-op: `{ inserted: false }`. + */ + const insert = async (data: { + triggerKey: string + ruleId: string + walletId: string + destinationWalletId: string + currency: string + triggerAmountMinor: string + shareCents: number + status: "pending" | "skipped_min" + now?: Date + }): Promise<{ inserted: true; id: string } | { inserted: false } | RepositoryError> => { + const { now = new Date(), ...fields } = data + try { + const doc = await PaymentSplitExecutionModel.create({ + ...fields, + attempts: 0, + nextAttemptAt: now, + createdAt: now, + updatedAt: now, + }) + return { inserted: true, id: doc._id.toString() } + } catch (error) { + if (isDuplicateKeyError(error)) return { inserted: false } + return toError(error) + } + } + + /** + * Atomically moves one due `pending` row to `sending`. Two concurrent + * claimers can never both get the same row: findOneAndUpdate matches on the + * status it is changing. + */ + const claimNextDue = async ( + now: Date, + ): Promise => { + try { + const doc = await PaymentSplitExecutionModel.findOneAndUpdate( + { status: "pending", nextAttemptAt: { $lte: now } }, + { $set: { status: "sending", updatedAt: now } }, + { sort: { nextAttemptAt: 1 }, new: true }, + ) + .lean() + .exec() + return doc ? toExecution(doc) : null + } catch (error) { + return toError(error) + } + } + + // Records the outcome of a claimed row. Matches on `sending` so a row the + // stale sweep already moved to `unknown` is never overwritten. + const finishClaimed = async ( + id: string, + update: { + status: "pending" | "sent" | "ibex_pending" | "failed" + attempts: number + nextAttemptAt?: Date + lastError?: string + now?: Date + }, + ): Promise => { + const { now = new Date(), ...fields } = update + try { + const result = await PaymentSplitExecutionModel.updateOne( + { _id: id, status: "sending" }, + { $set: { ...fields, updatedAt: now } }, + ) + return result.modifiedCount > 0 + } catch (error) { + return toError(error) + } + } + + // Moves one row stuck in `sending` since before `cutoff` to `unknown`. + const markNextStaleUnknown = async ( + cutoff: Date, + now: Date, + ): Promise => { + try { + const doc = await PaymentSplitExecutionModel.findOneAndUpdate( + { status: "sending", updatedAt: { $lt: cutoff } }, + { $set: { status: "unknown", updatedAt: now } }, + { new: true }, + ) + .lean() + .exec() + return doc ? toExecution(doc) : null + } catch (error) { + return toError(error) + } + } + + return { insert, claimNextDue, finishClaimed, markNextStaleUnknown } +} diff --git a/src/services/mongoose/payment-split-rules.ts b/src/services/mongoose/payment-split-rules.ts new file mode 100644 index 000000000..b56f57d85 --- /dev/null +++ b/src/services/mongoose/payment-split-rules.ts @@ -0,0 +1,118 @@ +import { parseRepositoryError } from "./utils" +import { PaymentSplitRule as PaymentSplitRuleModel } from "./schema" + +export type PaymentSplitRuleRecord = { + id: string + walletId: string + destinationWalletId: string + percent: number + enabled: boolean + createdAt: Date + updatedAt: Date +} + +type RawRule = { + _id: { toString(): string } + walletId: string + destinationWalletId: string + percent: number + enabled: boolean + createdAt: Date + updatedAt: Date +} + +// Typed RepositoryErrors, so every failure maps cleanly through error-map. +const toError = (error: unknown): RepositoryError => parseRepositoryError(error) + +const toRule = (raw: RawRule): PaymentSplitRuleRecord => ({ + id: raw._id.toString(), + walletId: raw.walletId, + destinationWalletId: raw.destinationWalletId, + percent: raw.percent, + enabled: raw.enabled, + createdAt: raw.createdAt, + updatedAt: raw.updatedAt, +}) + +export const PaymentSplitRulesRepository = () => { + const listByWalletId = async ( + walletId: string, + { enabledOnly = false }: { enabledOnly?: boolean } = {}, + ): Promise => { + try { + const filter: Record = { walletId: { $eq: walletId } } + if (enabledOnly) filter.enabled = true + const docs = await PaymentSplitRuleModel.find(filter) + .sort({ createdAt: 1 }) + .lean() + .exec() + return docs.map(toRule) + } catch (error) { + return toError(error) + } + } + + const findById = async ( + id: string, + ): Promise => { + try { + if (!/^[0-9a-f]{24}$/i.test(id)) return null + const doc = await PaymentSplitRuleModel.findById(id).lean().exec() + return doc ? toRule(doc) : null + } catch (error) { + return toError(error) + } + } + + // Replace semantics: the new set becomes the wallet's whole rule set. Callers + // hold the wallet lock so two concurrent replaces cannot interleave. + const replaceForWallet = async ( + walletId: string, + rules: { destinationWalletId: string; percent: number }[], + ): Promise => { + try { + await PaymentSplitRuleModel.deleteMany({ walletId: { $eq: walletId } }) + if (rules.length === 0) return [] + const now = new Date() + const docs = await PaymentSplitRuleModel.insertMany( + rules.map(({ destinationWalletId, percent }) => ({ + walletId, + destinationWalletId, + percent, + enabled: true, + createdAt: now, + updatedAt: now, + })), + ) + return docs.map((doc) => toRule(doc.toObject() as unknown as RawRule)) + } catch (error) { + return toError(error) + } + } + + const deleteById = async (id: string): Promise => { + try { + const result = await PaymentSplitRuleModel.deleteOne({ _id: id }) + return result.deletedCount > 0 + } catch (error) { + return toError(error) + } + } + + const listWalletIdsWithEnabledRules = async (): Promise => { + try { + const ids = await PaymentSplitRuleModel.distinct("walletId", { enabled: true }) + return ids.map(String) + } catch (error) { + return toError(error) + } + } + + return { + listByWalletId, + findById, + replaceForWallet, + deleteById, + listWalletIdsWithEnabledRules, + } +} diff --git a/src/services/mongoose/schema.ts b/src/services/mongoose/schema.ts index 766c729e3..70a06714f 100644 --- a/src/services/mongoose/schema.ts +++ b/src/services/mongoose/schema.ts @@ -896,6 +896,85 @@ export const BridgeReconciliationOrphan = mongoose.model( BridgeReconciliationOrphanSchema, ) +// ENG-653: Flash-native payment splits. One rule = "pay `percent` of every +// trigger-eligible receive on `walletId` to `destinationWalletId`". +const PaymentSplitRuleSchema = new Schema({ + walletId: { type: String, required: true }, + destinationWalletId: { type: String, required: true }, + percent: { type: Number, required: true, min: 1, max: 100 }, + enabled: { type: Boolean, required: true, default: true }, + createdAt: { type: Date, required: true, default: Date.now }, + updatedAt: { type: Date, required: true, default: Date.now }, +}) + +PaymentSplitRuleSchema.index({ walletId: 1, enabled: 1 }) + +export const PaymentSplitRule = mongoose.model("PaymentSplitRule", PaymentSplitRuleSchema) + +// One row per (receive, rule). The unique (triggerKey, ruleId) index is the +// dedupe between the webhook and the reconcile cron, and between webhook +// replays: a second insert for the same receive is a duplicate-key no-op. +const PaymentSplitExecutionSchema = new Schema({ + triggerKey: { type: String, required: true }, + ruleId: { type: String, required: true }, + walletId: { type: String, required: true }, + destinationWalletId: { type: String, required: true }, + currency: { type: String, required: true }, + // The receive amount in the source wallet's minor units (cents / micros). + triggerAmountMinor: { type: String, required: true }, + shareCents: { type: Number, required: true }, + status: { + type: String, + enum: [ + "pending", + "sending", + "sent", + "ibex_pending", + "failed", + "skipped_min", + "unknown", + ], + required: true, + }, + attempts: { type: Number, required: true, default: 0 }, + nextAttemptAt: { type: Date, required: true, default: Date.now }, + lastError: { type: String }, + createdAt: { type: Date, required: true, default: Date.now }, + updatedAt: { type: Date, required: true, default: Date.now }, +}) + +PaymentSplitExecutionSchema.index({ triggerKey: 1, ruleId: 1 }, { unique: true }) +PaymentSplitExecutionSchema.index({ status: 1, nextAttemptAt: 1 }) + +export const PaymentSplitExecution = mongoose.model( + "PaymentSplitExecution", + PaymentSplitExecutionSchema, +) + +// Payment hashes of invoices Flash minted for itself: the recipient side of an +// intraledger send (P2P, Fygaro top-up credits, referral payouts, split +// payouts) and of cash-wallet cutover moves. A receive whose hash is here never +// triggers payment splits. TTL must stay above the split reconcile lookback. +export const INTERNAL_TRANSFER_INVOICE_TTL_SECONDS = 30 * 24 * 60 * 60 + +const InternalTransferInvoiceSchema = new Schema({ + paymentHash: { type: String, required: true, unique: true }, + senderWalletId: { type: String }, + recipientWalletId: { type: String, required: true }, + origin: { type: String, enum: ["intraledger", "cutover"], required: true }, + createdAt: { type: Date, required: true, default: Date.now }, +}) + +InternalTransferInvoiceSchema.index( + { createdAt: 1 }, + { expireAfterSeconds: INTERNAL_TRANSFER_INVOICE_TTL_SECONDS }, +) + +export const InternalTransferInvoice = mongoose.model( + "InternalTransferInvoice", + InternalTransferInvoiceSchema, +) + const BridgeReplaySchema = new Schema({ eventId: { type: String, required: true }, eventType: { type: String, required: true }, diff --git a/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts b/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts new file mode 100644 index 000000000..325cf4ece --- /dev/null +++ b/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts @@ -0,0 +1,123 @@ +import { + INTERNAL_TRANSFER_INVOICE_TTL_SECONDS, + InternalTransferInvoice, + PaymentSplitExecution, + PaymentSplitRule, +} from "@services/mongoose/schema" +import { + isInternalTransferInvoice, + recordInternalTransferInvoice, +} from "@services/mongoose/internal-transfer-invoices" +import { PaymentSplitExecutionsRepository } from "@services/mongoose/payment-split-executions" +import { PaymentSplitRulesRepository } from "@services/mongoose/payment-split-rules" + +const executionFields = (triggerKey: string, ruleId: string) => ({ + triggerKey, + ruleId, + walletId: "source-wallet", + destinationWalletId: "dest-wallet", + currency: "USDT", + triggerAmountMinor: "1000000", + shareCents: 10, + status: "pending" as const, +}) + +describe("payment split repositories (ENG-653)", () => { + beforeAll(async () => { + await PaymentSplitExecution.syncIndexes() + await InternalTransferInvoice.syncIndexes() + }) + + beforeEach(async () => { + await PaymentSplitExecution.deleteMany({}) + await InternalTransferInvoice.deleteMany({}) + await PaymentSplitRule.deleteMany({}) + }) + + it("replaces a wallet's whole rule set and lists wallets with enabled rules", async () => { + const rules = PaymentSplitRulesRepository() + + const first = await rules.replaceForWallet("wallet-a", [ + { destinationWalletId: "dest-1", percent: 10 }, + { destinationWalletId: "dest-2", percent: 20 }, + ]) + if (first instanceof Error) throw first + expect(first).toHaveLength(2) + + const second = await rules.replaceForWallet("wallet-a", [ + { destinationWalletId: "dest-3", percent: 50 }, + ]) + if (second instanceof Error) throw second + + const listed = await rules.listByWalletId("wallet-a", { enabledOnly: true }) + expect(listed).toEqual([ + expect.objectContaining({ + destinationWalletId: "dest-3", + percent: 50, + enabled: true, + }), + ]) + expect(await rules.findById(second[0].id)).toMatchObject({ walletId: "wallet-a" }) + expect(await rules.findById("not-an-object-id")).toBeNull() + expect(await rules.listWalletIdsWithEnabledRules()).toEqual(["wallet-a"]) + + expect(await rules.deleteById(second[0].id)).toBe(true) + expect(await rules.deleteById(second[0].id)).toBe(false) + expect(await rules.listWalletIdsWithEnabledRules()).toEqual([]) + }) + + it("rejects a duplicate (triggerKey, ruleId) as a typed no-op", async () => { + const repo = PaymentSplitExecutionsRepository() + const triggerKey = "d".repeat(64) + + const first = await repo.insert(executionFields(triggerKey, "rule-1")) + const second = await repo.insert(executionFields(triggerKey, "rule-1")) + const otherRule = await repo.insert(executionFields(triggerKey, "rule-2")) + + expect(first).toMatchObject({ inserted: true }) + expect(second).toEqual({ inserted: false }) + expect(otherRule).toMatchObject({ inserted: true }) + expect(await PaymentSplitExecution.countDocuments({ triggerKey })).toBe(2) + }) + + it("lets exactly one of several concurrent claimers claim a pending row", async () => { + const repo = PaymentSplitExecutionsRepository() + await repo.insert(executionFields("e".repeat(64), "rule-1")) + + const now = new Date(Date.now() + 1000) + const claims = await Promise.all( + Array.from({ length: 5 }, () => repo.claimNextDue(now)), + ) + + const claimed = claims.filter((c) => c !== null && !(c instanceof Error)) + expect(claimed).toHaveLength(1) + expect(claimed[0]).toMatchObject({ status: "sending" }) + }) + + it("has a TTL index on internal-transfer markers that outlives the reconcile lookback", async () => { + const indexes = await InternalTransferInvoice.collection.indexes() + const ttl = indexes.find((index) => index.expireAfterSeconds !== undefined) + + expect(ttl).toMatchObject({ + key: { createdAt: 1 }, + expireAfterSeconds: INTERNAL_TRANSFER_INVOICE_TTL_SECONDS, + }) + expect(INTERNAL_TRANSFER_INVOICE_TTL_SECONDS).toBe(30 * 24 * 60 * 60) + }) + + it("records a marker idempotently and finds it case-insensitively", async () => { + const hash = "AB".repeat(32) + const args = { + paymentHash: hash, + senderWalletId: "sender", + recipientWalletId: "recipient", + origin: "intraledger" as const, + } + + expect(await recordInternalTransferInvoice(args)).toBe(true) + expect(await recordInternalTransferInvoice(args)).toBe(true) + expect(await isInternalTransferInvoice(hash.toLowerCase())).toBe(true) + expect(await isInternalTransferInvoice("cd".repeat(32))).toBe(false) + expect(await InternalTransferInvoice.countDocuments({})).toBe(1) + }) +}) From 7c30e4efd232c90468b27d9525114c20bc1ba35c Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 22:57:49 -0700 Subject: [PATCH 04/13] feat(payments): mark internally minted invoices before paying them Every IBEX addInvoice is bound to the on-receive webhook, so system credits (Fygaro top-ups, referral payouts) and split payouts would otherwise trigger the recipient's payment splits. The marker write fails closed. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WxkKbrg2Xohf3mQj7BLrs6 --- .../cash-wallet-cutover/runtime-services.ts | 30 +++- src/app/payments/send-intraledger.ts | 49 ++++++- .../runtime-services.spec.ts | 49 +++++++ .../add-invoice-call-sites.spec.ts | 41 ++++++ .../app/payments/send-intraledger.spec.ts | 136 +++++++++++++++++- 5 files changed, 298 insertions(+), 7 deletions(-) create mode 100644 test/flash/unit/app/payment-splits/add-invoice-call-sites.spec.ts diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index be1e3cd59..da3a57534 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -7,6 +7,8 @@ import { AccountsRepository, WalletsRepository } from "@services/mongoose" import Ibex from "@services/ibex/client" import { UnexpectedIbexResponse } from "@services/ibex/errors" import { getFunderWalletId } from "@services/ledger/caching" +import { recordInternalTransferInvoice } from "@services/mongoose/internal-transfer-invoices" +import { InternalTransferMarkerError } from "@domain/payment-splits/errors" import { CashWalletMigrationFailedError, @@ -33,6 +35,7 @@ type RuntimeServiceDependencies = { walletsRepo?: Pick, "findById" | "listByAccountId"> getFunderWalletId?: typeof getFunderWalletId getTreasuryWalletId?: () => Promise + recordInternalTransferInvoice?: typeof recordInternalTransferInvoice maxRateLimitAttempts?: number rateLimitRetryDelayMs?: number sleep?: SleepFn @@ -181,6 +184,25 @@ export const createCashWalletMigrationRuntimeServices = ( const invoiceForRecipient = deps.createInvoice ?? Ibex.addInvoice const noAmountInvoiceForRecipient = deps.createNoAmountInvoice ?? Ibex.addInvoice const payInvoice = deps.payInvoice ?? Ibex.payInvoice + const recordMarker = deps.recordInternalTransferInvoice ?? recordInternalTransferInvoice + // ENG-653: every invoice the cutover mints is paid by Flash itself, so its + // receive must never trigger the recipient's payment splits. The marker is + // written before the invoice is handed back to be paid; if it cannot be + // written the invoice is not returned and nothing is paid. + const markCutoverInvoice = + (recipientWalletId: WalletId) => + async ( + invoice: LnInvoice | ApplicationError, + ): Promise => { + if (invoice instanceof Error) return invoice + const recorded = await recordMarker({ + paymentHash: invoice.paymentHash, + recipientWalletId, + origin: "cutover", + }) + if (recorded instanceof Error) return new InternalTransferMarkerError(recorded) + return invoice + } const accountsRepo = deps.accountsRepo ?? AccountsRepository() const walletsRepo = deps.walletsRepo ?? WalletsRepository() const funderWalletId = deps.getFunderWalletId ?? getFunderWalletId @@ -292,7 +314,9 @@ export const createCashWalletMigrationRuntimeServices = ( memo, expiration: CUTOVER_IBEX_INVOICE_EXPIRATION_SECONDS as Seconds, }), - }).then(ibexInvoiceToDomainInvoice) + }) + .then(ibexInvoiceToDomainInvoice) + .then(markCutoverInvoice(recipientWalletId)) }, createNoAmountInvoice: ({ recipientWalletId, @@ -310,7 +334,9 @@ export const createCashWalletMigrationRuntimeServices = ( memo, expiration: CUTOVER_IBEX_INVOICE_EXPIRATION_SECONDS as Seconds, }), - }).then(ibexInvoiceToDomainInvoice), + }) + .then(ibexInvoiceToDomainInvoice) + .then(markCutoverInvoice(recipientWalletId)), }, paymentService: { payInvoice: async ({ diff --git a/src/app/payments/send-intraledger.ts b/src/app/payments/send-intraledger.ts index 1b5ee3a7a..1008dab0c 100644 --- a/src/app/payments/send-intraledger.ts +++ b/src/app/payments/send-intraledger.ts @@ -2,13 +2,15 @@ import { validateIsBtcWallet, validateIsUsdWallet } from "@app/wallets" import { usdWalletAmountFromInput } from "@app/wallets/usd-wallet-amount" import { AccountValidator } from "@domain/accounts" -import { PaymentSendStatus } from "@domain/bitcoin/lightning" +import { decodeInvoice, PaymentSendStatus } from "@domain/bitcoin/lightning" +import { InternalTransferMarkerError } from "@domain/payment-splits/errors" import { checkedToWalletId } from "@domain/wallets" import { MismatchedCurrencyForWalletError } from "@domain/errors" import { notifyOpsEvent } from "@services/alerts/ops-events" import { addAttributesToCurrentSpan } from "@services/tracing" import { AccountsRepository, WalletsRepository } from "@services/mongoose" +import { recordInternalTransferInvoice } from "@services/mongoose/internal-transfer-invoices" import Ibex from "@services/ibex/client" import { UnexpectedIbexResponse } from "@services/ibex/errors" @@ -92,6 +94,20 @@ const intraledgerPaymentSendWalletId = async ({ if (invoiceResp.invoice?.bolt11 === undefined) return new UnexpectedIbexResponse("Bolt11 field not found.") + // ENG-653: mark the recipient invoice as internal BEFORE paying it. IBEX + // binds every addInvoice to the on-receive webhook, so without this marker a + // system credit (Fygaro top-up, referral payout) or a split payout itself + // would trigger the recipient's payment splits. Fails closed: if the marker + // cannot be written nothing is paid, and the error return is left uncached + // by withPaymentIdempotency, so the same key can be retried. + const marked = await markInternalTransferInvoice({ + bolt11: invoiceResp.invoice.bolt11, + ibexHash: invoiceResp.invoice.hash, + senderWalletId: senderWallet.id, + recipientWalletId, + }) + if (marked instanceof Error) return marked + const payResp = await Ibex.payInvoice({ accountId: uncheckedSenderWalletId, invoice: invoiceResp.invoice.bolt11 as Bolt11, @@ -128,6 +144,37 @@ const intraledgerPaymentSendWalletId = async ({ return paymentSendStatus } +const PaymentHashRegex = /^[0-9a-f]{64}$/i + +const markInternalTransferInvoice = async ({ + bolt11, + ibexHash, + senderWalletId, + recipientWalletId, +}: { + bolt11: string + ibexHash: string | undefined + senderWalletId: WalletId + recipientWalletId: WalletId +}): Promise => { + // The hash of the bolt11 we are about to pay is authoritative; IBEX's own + // `invoice.hash` is the fallback when the bolt11 cannot be decoded locally. + const decoded = decodeInvoice(bolt11) + const paymentHash = decoded instanceof Error ? ibexHash : decoded.paymentHash + if (typeof paymentHash !== "string" || !PaymentHashRegex.test(paymentHash)) { + return new InternalTransferMarkerError("Could not read recipient invoice hash") + } + + const recorded = await recordInternalTransferInvoice({ + paymentHash, + senderWalletId, + recipientWalletId, + origin: "intraledger", + }) + if (recorded instanceof Error) return new InternalTransferMarkerError(recorded) + return true +} + export const intraledgerPaymentSendWalletIdForBtcWallet = async ( args: IntraLedgerPaymentSendWalletIdArgs, ): Promise => diff --git a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts index 4de21b46b..9fc4d5652 100644 --- a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts @@ -1,4 +1,5 @@ import { createCashWalletMigrationRuntimeServices } from "@app/cash-wallet-cutover/runtime-services" +import { InternalTransferMarkerError } from "@domain/payment-splits/errors" const migration = { legacyUsdWalletId: "legacy-wallet-id" as WalletId, @@ -215,3 +216,51 @@ describe("cutover runtime services — IBEX rate-limit retry (ENG-483)", () => { expect(getRawAccountDetails).toHaveBeenCalledTimes(1) }) }) + +describe("cutover runtime services — internal-transfer marker (ENG-653)", () => { + // Every cutover invoice is paid by Flash itself; its receive must never + // trigger the recipient's payment splits. + const bolt11 = + "lnbc140n1p3k6yzupp53p305l6de6s9xw2j0qaa59pl7lahys4f2uavwncll9z2vq0syvvsdqqcqzpgxqzuysp5mdgsaa734eg7srwx92rsn3hyc4xzt5tphfpadl5c6fanhppwaz4s9qyyssqm6yhnnhl8jltwjtclzk4g7nxr99ycsp4sqd6vksevqh06h8l3gm5fdhtl59t6g3fsalv26sj5zvwhxwlghc9wcfgkrjrtuh4873ejnspc5xksy" + const paymentHash = "8862fa7f4dcea0533952783bda143ff7fb7242a9573ac74f1ff944a601f02319" + const recipientWalletId = "destination-wallet-id" as WalletId + + it.each([ + ["createInvoice", { recipientWalletId, amount: "1000000", memo: "cutover" }], + ["createNoAmountInvoice", { recipientWalletId, memo: "cutover" }], + ] as const)("%s records the invoice hash with origin cutover", async (method, args) => { + const recordInternalTransferInvoice = jest.fn().mockResolvedValue(true) + const addInvoice = jest.fn().mockResolvedValue({ invoice: { bolt11 } }) + const services = createCashWalletMigrationRuntimeServices({ + createInvoice: addInvoice, + createNoAmountInvoice: addInvoice, + recordInternalTransferInvoice, + }) + + const invoice = await ( + services.invoiceService[method] as (a: typeof args) => Promise + )(args) + + expect(invoice).not.toBeInstanceOf(Error) + expect((invoice as LnInvoice).paymentHash).toBe(paymentHash) + expect(recordInternalTransferInvoice).toHaveBeenCalledWith({ + paymentHash, + recipientWalletId, + origin: "cutover", + }) + }) + + it("does not hand back an invoice to pay when the marker write fails", async () => { + const services = createCashWalletMigrationRuntimeServices({ + createNoAmountInvoice: jest.fn().mockResolvedValue({ invoice: { bolt11 } }), + recordInternalTransferInvoice: jest.fn().mockResolvedValue(new Error("mongo down")), + }) + + const invoice = await services.invoiceService.createNoAmountInvoice({ + recipientWalletId, + memo: "cutover", + }) + + expect(invoice).toBeInstanceOf(InternalTransferMarkerError) + }) +}) diff --git a/test/flash/unit/app/payment-splits/add-invoice-call-sites.spec.ts b/test/flash/unit/app/payment-splits/add-invoice-call-sites.spec.ts new file mode 100644 index 000000000..a74717dae --- /dev/null +++ b/test/flash/unit/app/payment-splits/add-invoice-call-sites.spec.ts @@ -0,0 +1,41 @@ +import { readdirSync, readFileSync, statSync } from "fs" +import { join, relative } from "path" + +// ENG-653 risk guard. The internal-transfer marker is a denylist: IBEX binds +// EVERY addInvoice to the on-receive webhook, so a new code path that mints an +// invoice Flash pays itself — without writing the marker — would fire the +// recipient's payment splits on a system credit. Adding a call site makes this +// test fail until it is classified below. +const CLASSIFIED_CALL_SITES: Record = { + // Marked internal (origin "intraledger") before paying. + "src/app/payments/send-intraledger.ts": "marked", + // Marked internal (origin "cutover") before the invoice is handed back. + "src/app/cash-wallet-cutover/runtime-services.ts": "marked", + // User-facing receive invoices: an external payer pays them; they SHOULD + // trigger splits. + "src/app/wallets/add-invoice-for-wallet.ts": "user-receive", + "src/graphql/public/root/mutation/ln-noamount-invoice-create.ts": "user-receive", + // Cashout: the user pays an invoice on Flash's own cashout wallet. Flash's + // wallets carry no split rules, so this receive has no rules to trigger. + "src/app/offers/CashoutManager.ts": "flash-owned-recipient", +} + +const listTs = (dir: string): string[] => + readdirSync(dir).flatMap((name) => { + const path = join(dir, name) + if (statSync(path).isDirectory()) return listTs(path) + return path.endsWith(".ts") && !path.endsWith(".d.ts") ? [path] : [] + }) + +describe("Ibex.addInvoice call sites", () => { + it("are all classified for the payment-splits marker", () => { + const root = join(__dirname, "../../../../..") + const callers = listTs(join(root, "src")) + .filter((path) => !path.endsWith(join("services", "ibex", "client.ts"))) + .filter((path) => /\bIbex\.addInvoice\b/.test(readFileSync(path, "utf8"))) + .map((path) => relative(root, path).split("\\").join("/")) + .sort() + + expect(callers).toEqual(Object.keys(CLASSIFIED_CALL_SITES).sort()) + }) +}) diff --git a/test/flash/unit/app/payments/send-intraledger.spec.ts b/test/flash/unit/app/payments/send-intraledger.spec.ts index fff2c8fe0..f8b27da9d 100644 --- a/test/flash/unit/app/payments/send-intraledger.spec.ts +++ b/test/flash/unit/app/payments/send-intraledger.spec.ts @@ -2,6 +2,7 @@ const mockAddInvoice = jest.fn() const mockPayInvoice = jest.fn() const mockFindWalletById = jest.fn() const mockFindAccountById = jest.fn() +const mockRecordInternalTransferInvoice = jest.fn() // In-memory backing stores for the ENG-530 idempotency helper (see the // "intraledger idempotency" describe block below). Keyed on the scoped cache/lock @@ -70,6 +71,12 @@ jest.mock("@services/mongoose", () => ({ UsersRepository: jest.fn(), })) +// ENG-653: the internal-transfer marker written before every intraledger pay. +jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ + recordInternalTransferInvoice: (...args: unknown[]) => + mockRecordInternalTransferInvoice(...args), +})) + jest.mock("@services/dealer-price", () => ({ DealerPriceService: jest.fn(() => ({})), })) @@ -135,6 +142,7 @@ import { IntraledgerLimitsExceededError, MismatchedCurrencyForWalletError, } from "@domain/errors" +import { InternalTransferMarkerError } from "@domain/payment-splits/errors" import { USDAmount, USDTAmount, WalletCurrency } from "@domain/shared" import { notifyOpsEvent } from "@services/alerts/ops-events" import { addEventToCurrentSpan, recordExceptionInCurrentSpan } from "@services/tracing" @@ -150,6 +158,17 @@ const senderUsdtWalletId = "22222222-2222-4222-8222-222222222222" as WalletId const recipientUsdWalletId = "33333333-3333-4333-8333-333333333333" as WalletId const recipientUsdtWalletId = "44444444-4444-4444-8444-444444444444" as WalletId +// IBEX returns the recipient invoice's payment hash alongside the bolt11. The +// fixture bolt11 is not decodable, so the marker falls back to this hash. +const RECIPIENT_HASH = "ab".repeat(32) +const ibexAddInvoiceResponse = { + invoice: { bolt11: "lnbc1recipient", hash: RECIPIENT_HASH }, +} + +beforeEach(() => { + mockRecordInternalTransferInvoice.mockResolvedValue(true) +}) + const activeAccount = (id: string) => ({ id, @@ -196,7 +215,7 @@ describe("intraledgerPaymentSendWalletIdForUsdWallet", () => { mockFindAccountById.mockImplementation(async (accountId: AccountId) => activeAccount(accountId as string), ) - mockAddInvoice.mockResolvedValue({ invoice: { bolt11: "lnbc1recipient" } }) + mockAddInvoice.mockResolvedValue(ibexAddInvoiceResponse) mockPayInvoice.mockResolvedValue(ibexPayResponse(2)) }) @@ -345,7 +364,7 @@ describe("intraledger IBEX status reading", () => { mockFindAccountById.mockImplementation(async (accountId: AccountId) => activeAccount(accountId as string), ) - mockAddInvoice.mockResolvedValue({ invoice: { bolt11: "lnbc1recipient" } }) + mockAddInvoice.mockResolvedValue(ibexAddInvoiceResponse) mockFindWalletById.mockImplementation(async (walletId: WalletId) => wallet({ id: walletId, @@ -480,7 +499,7 @@ describe("intraledger send ops events", () => { mockFindAccountById.mockImplementation(async (accountId: AccountId) => activeAccount(accountId as string), ) - mockAddInvoice.mockResolvedValue({ invoice: { bolt11: "lnbc1recipient" } }) + mockAddInvoice.mockResolvedValue(ibexAddInvoiceResponse) mockPayInvoice.mockResolvedValue(ibexPayResponse(2)) mockFindWalletById.mockImplementation(async (walletId: WalletId) => wallet({ @@ -620,7 +639,7 @@ describe("intraledger idempotency (ENG-530)", () => { mockFindAccountById.mockImplementation(async (accountId: AccountId) => activeAccount(accountId as string), ) - mockAddInvoice.mockResolvedValue({ invoice: { bolt11: "lnbc1recipient" } }) + mockAddInvoice.mockResolvedValue(ibexAddInvoiceResponse) mockPayInvoice.mockResolvedValue(ibexPayResponse(2)) mockFindWalletById.mockImplementation(async (walletId: WalletId) => { const isUsdt = walletId === senderUsdtWalletId || walletId === recipientUsdtWalletId @@ -810,3 +829,112 @@ describe("intraledger idempotency (ENG-530)", () => { expect(mockPayInvoice).toHaveBeenCalledTimes(1) }) }) + +describe("internal-transfer marker (ENG-653)", () => { + // IBEX binds every addInvoice to the on-receive webhook, so the recipient of + // a system credit (Fygaro top-up, referral payout) or of a split payout would + // otherwise trigger their own payment splits. The marker is what tells the + // split pipeline the receive is internal. + const sendArgs = { + senderWalletId: senderUsdWalletId, + recipientWalletId: recipientUsdWalletId, + amount: 250, + memo: "marker test", + authorize: allowSend, + } + + beforeEach(() => { + jest.clearAllMocks() + mockCacheStore.clear() + mockHeldLocks.clear() + mockRecordInternalTransferInvoice.mockResolvedValue(true) + mockFindAccountById.mockImplementation(async (accountId: AccountId) => + activeAccount(accountId as string), + ) + mockAddInvoice.mockResolvedValue(ibexAddInvoiceResponse) + mockPayInvoice.mockResolvedValue(ibexPayResponse(2)) + mockFindWalletById.mockImplementation(async (walletId: WalletId) => + wallet({ + id: walletId, + accountId: + walletId === senderUsdWalletId ? "sender-account" : "recipient-account", + currency: WalletCurrency.Usd, + }), + ) + }) + + it("writes the marker for the recipient invoice before paying it", async () => { + const order: string[] = [] + mockRecordInternalTransferInvoice.mockImplementation(async () => { + order.push("marker") + return true + }) + mockPayInvoice.mockImplementation(async () => { + order.push("pay") + return ibexPayResponse(2) + }) + + const result = await intraledgerPaymentSendWalletIdForUsdWallet(sendArgs) + + expect(result).toEqual({ value: "success" }) + expect(order).toEqual(["marker", "pay"]) + expect(mockRecordInternalTransferInvoice).toHaveBeenCalledWith({ + paymentHash: RECIPIENT_HASH, + senderWalletId: senderUsdWalletId, + recipientWalletId: recipientUsdWalletId, + origin: "intraledger", + }) + }) + + it("returns an error and never pays when the marker write fails", async () => { + mockRecordInternalTransferInvoice.mockResolvedValue(new Error("mongo down")) + + const result = await intraledgerPaymentSendWalletIdForUsdWallet(sendArgs) + + expect(result).toBeInstanceOf(InternalTransferMarkerError) + expect(mockPayInvoice).not.toHaveBeenCalled() + }) + + it("leaves the idempotency key retryable after a marker failure", async () => { + mockRecordInternalTransferInvoice + .mockResolvedValueOnce(new Error("mongo down")) + .mockResolvedValue(true) + + const first = await intraledgerPaymentSendWalletIdForUsdWallet({ + ...sendArgs, + idempotencyKey: "marker-retry", + }) + const second = await intraledgerPaymentSendWalletIdForUsdWallet({ + ...sendArgs, + idempotencyKey: "marker-retry", + }) + + expect(first).toBeInstanceOf(InternalTransferMarkerError) + expect(second).toEqual({ value: "success" }) + expect(mockPayInvoice).toHaveBeenCalledTimes(1) + }) + + it("marks the hash decoded from the bolt11 it pays, over IBEX's reported hash", async () => { + const bolt11 = + "lnbc140n1p3k6yzupp53p305l6de6s9xw2j0qaa59pl7lahys4f2uavwncll9z2vq0syvvsdqqcqzpgxqzuysp5mdgsaa734eg7srwx92rsn3hyc4xzt5tphfpadl5c6fanhppwaz4s9qyyssqm6yhnnhl8jltwjtclzk4g7nxr99ycsp4sqd6vksevqh06h8l3gm5fdhtl59t6g3fsalv26sj5zvwhxwlghc9wcfgkrjrtuh4873ejnspc5xksy" + mockAddInvoice.mockResolvedValue({ invoice: { bolt11, hash: RECIPIENT_HASH } }) + + await intraledgerPaymentSendWalletIdForUsdWallet(sendArgs) + + expect(mockRecordInternalTransferInvoice).toHaveBeenCalledWith( + expect.objectContaining({ + paymentHash: "8862fa7f4dcea0533952783bda143ff7fb7242a9573ac74f1ff944a601f02319", + }), + ) + }) + + it("refuses to pay when no payment hash can be read for the invoice", async () => { + mockAddInvoice.mockResolvedValue({ invoice: { bolt11: "lnbc1recipient" } }) + + const result = await intraledgerPaymentSendWalletIdForUsdWallet(sendArgs) + + expect(result).toBeInstanceOf(InternalTransferMarkerError) + expect(mockRecordInternalTransferInvoice).not.toHaveBeenCalled() + expect(mockPayInvoice).not.toHaveBeenCalled() + }) +}) From 2469ae9d45e08b10c2e1a3b7a5da7044ec350ee4 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 22:57:49 -0700 Subject: [PATCH 05/13] feat(payment-splits): record, execute, sweep and reconcile split payouts Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WxkKbrg2Xohf3mQj7BLrs6 --- src/app/payment-splits/delete-split.ts | 44 +++ .../payment-splits/execute-pending-splits.ts | 189 +++++++++++ src/app/payment-splits/get-splits.ts | 9 + src/app/payment-splits/index.ts | 7 + src/app/payment-splits/reconcile-splits.ts | 115 +++++++ .../payment-splits/record-split-executions.ts | 92 +++++ src/app/payment-splits/set-splits.ts | 137 ++++++++ src/app/payment-splits/sweep-stale-splits.ts | 53 +++ src/config/locales/en.json | 10 + src/config/locales/es.json | 10 + .../execute-pending-splits.spec.ts | 313 ++++++++++++++++++ .../payment-splits/fake-executions-store.ts | 90 +++++ .../payment-splits/reconcile-splits.spec.ts | 209 ++++++++++++ .../record-split-executions.spec.ts | 148 +++++++++ .../app/payment-splits/set-splits.spec.ts | 288 ++++++++++++++++ .../payment-splits/sweep-stale-splits.spec.ts | 91 +++++ 16 files changed, 1805 insertions(+) create mode 100644 src/app/payment-splits/delete-split.ts create mode 100644 src/app/payment-splits/execute-pending-splits.ts create mode 100644 src/app/payment-splits/get-splits.ts create mode 100644 src/app/payment-splits/index.ts create mode 100644 src/app/payment-splits/reconcile-splits.ts create mode 100644 src/app/payment-splits/record-split-executions.ts create mode 100644 src/app/payment-splits/set-splits.ts create mode 100644 src/app/payment-splits/sweep-stale-splits.ts create mode 100644 test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts create mode 100644 test/flash/unit/app/payment-splits/fake-executions-store.ts create mode 100644 test/flash/unit/app/payment-splits/reconcile-splits.spec.ts create mode 100644 test/flash/unit/app/payment-splits/record-split-executions.spec.ts create mode 100644 test/flash/unit/app/payment-splits/set-splits.spec.ts create mode 100644 test/flash/unit/app/payment-splits/sweep-stale-splits.spec.ts diff --git a/src/app/payment-splits/delete-split.ts b/src/app/payment-splits/delete-split.ts new file mode 100644 index 000000000..017048b75 --- /dev/null +++ b/src/app/payment-splits/delete-split.ts @@ -0,0 +1,44 @@ +import { PaymentSplitsConfig } from "@config" + +import { + PaymentSplitsDisabledError, + SplitRuleNotFoundError, +} from "@domain/payment-splits" + +import { LockService } from "@services/lock" +import { PaymentSplitRulesRepository } from "@services/mongoose/payment-split-rules" + +import { resolveOwnedWallet } from "./set-splits" + +/** + * Deletes one split rule. Executions already recorded for it are left alone + * and still pay out: they belong to receives that arrived while it existed. + */ +export const deletePaymentSplit = async ({ + account, + walletId, + splitId, +}: { + account: Account + walletId: string + splitId: string +}): Promise => { + if (!PaymentSplitsConfig.enabled) return new PaymentSplitsDisabledError() + + const wallet = await resolveOwnedWallet({ account, walletId }) + if (wallet instanceof Error) return wallet + + return LockService().lockWalletId(wallet.id, async () => { + const rules = PaymentSplitRulesRepository() + const rule = await rules.findById(splitId) + if (rule instanceof Error) return rule + if (rule === null || rule.walletId !== wallet.id) { + return new SplitRuleNotFoundError(splitId) + } + + const deleted = await rules.deleteById(rule.id) + if (deleted instanceof Error) return deleted + if (!deleted) return new SplitRuleNotFoundError(splitId) + return true as const + }) +} diff --git a/src/app/payment-splits/execute-pending-splits.ts b/src/app/payment-splits/execute-pending-splits.ts new file mode 100644 index 000000000..c12a0d175 --- /dev/null +++ b/src/app/payment-splits/execute-pending-splits.ts @@ -0,0 +1,189 @@ +import { PaymentSplitsConfig } from "@config" + +import { sendOutcomeNotificationBestEffort } from "@app/notifications/send-outcome-notification" +import { authorizeSend } from "@app/payments/authorize-send" +import { intraledgerPaymentSendWalletIdForUsdWallet } from "@app/payments/send-intraledger" + +import { PaymentSendStatus } from "@domain/bitcoin/lightning" +import { LockServiceError } from "@domain/lock" +import { InternalTransferMarkerError, splitRetryBackoffMs } from "@domain/payment-splits" +import { SendLimitsUnavailableError } from "@domain/payments/errors" +import { RateLimitServiceError } from "@domain/rate-limit/errors" + +import { IbexError } from "@services/ibex/errors" +import { baseLogger } from "@services/logger" +import { AccountsRepository, WalletsRepository } from "@services/mongoose" +import { + PaymentSplitExecutionsRepository, + type PaymentSplitExecutionRecord, +} from "@services/mongoose/payment-split-executions" + +const logger = baseLogger.child({ module: "payment-splits" }) + +export const PAYMENT_SPLIT_MEMO = "Payment split" + +export const splitIdempotencyKey = (executionId: string): IdempotencyKey => + `split:${executionId}` as IdempotencyKey + +/** + * Errors worth another attempt: raised before IBEX paid anything and expected + * to clear on their own (rate limits, a send guard that could not reach Redis + * or a price, a busy idempotency lock, the internal-transfer marker write, an + * IBEX 429). Everything else — limits, KYC level, insufficient balance, an + * inactive account, an unknown IBEX error — is terminal on the first attempt, + * so an error whose payment outcome is unknown is never re-sent. + */ +export const isRetryableSplitError = (error: Error): boolean => + error instanceof RateLimitServiceError || + error instanceof SendLimitsUnavailableError || + error instanceof LockServiceError || + error instanceof InternalTransferMarkerError || + (error instanceof IbexError && error.httpCode === 429) + +const formatCents = (cents: number): string => `US$${(cents / 100).toFixed(2)}` + +const notifyOwner = async ( + accountId: string, + outcome: "sent" | "failed", + execution: PaymentSplitExecutionRecord, +): Promise => + sendOutcomeNotificationBestEffort({ + accountId, + phraseBase: `notification.paymentSplit.${outcome}`, + dataType: `payment_split_${outcome}`, + amountArg: formatCents(execution.shareCents), + extraData: { + amount: (execution.shareCents / 100).toFixed(2), + currency: "USD", + walletId: execution.walletId, + }, + logMessage: `Failed to send payment split ${outcome} push notification`, + logContext: { executionId: execution.id, walletId: execution.walletId }, + }) + +type ExecuteOutcome = "sent" | "ibex_pending" | "failed" | "retry" + +const executeOne = async ( + execution: PaymentSplitExecutionRecord, + now: Date, +): Promise => { + const executions = PaymentSplitExecutionsRepository() + const attempts = execution.attempts + 1 + + const fail = async (error: Error | string, accountId?: string) => { + const lastError = typeof error === "string" ? error : error.name || error.message + await executions.finishClaimed(execution.id, { + status: "failed", + attempts, + lastError, + now, + }) + if (accountId) await notifyOwner(accountId, "failed", execution) + return "failed" as const + } + + const wallet = await WalletsRepository().findById(execution.walletId as WalletId) + if (wallet instanceof Error) return fail(wallet) + const account = await AccountsRepository().findById(wallet.accountId) + if (account instanceof Error) return fail(account) + + const result = await intraledgerPaymentSendWalletIdForUsdWallet({ + senderWalletId: wallet.id, + recipientWalletId: execution.destinationWalletId as WalletId, + amount: execution.shareCents, + memo: PAYMENT_SPLIT_MEMO, + senderAccount: account, + // Stable across retries of the same execution, so a retry after an + // ambiguous failure replays the cached outcome instead of paying again. + idempotencyKey: splitIdempotencyKey(execution.id), + // A split is a send out of the user's own wallet: it goes through the same + // ENG-573 guard (attempt budget, amount sanity, daily limit) as a manual + // intraledger send. + authorize: () => + authorizeSend({ + senderAccount: account, + senderWalletId: wallet.id, + amount: { currency: "USD", cents: execution.shareCents }, + kind: "intraledger", + }), + }) + + if (result instanceof Error) { + if (isRetryableSplitError(result) && attempts < PaymentSplitsConfig.maxAttempts) { + await executions.finishClaimed(execution.id, { + status: "pending", + attempts, + nextAttemptAt: new Date(now.getTime() + splitRetryBackoffMs(attempts)), + lastError: result.name, + now, + }) + return "retry" + } + logger.warn( + { executionId: execution.id, walletId: execution.walletId, error: result }, + "payment split failed", + ) + return fail(result, account.id) + } + + if (result === PaymentSendStatus.Success) { + await executions.finishClaimed(execution.id, { status: "sent", attempts, now }) + await notifyOwner(account.id, "sent", execution) + return "sent" + } + + if (result === PaymentSendStatus.Pending) { + // Money has probably left the wallet: never re-pay (same stance as the + // Fygaro credit). Ops resolves the final state from IBEX. + await executions.finishClaimed(execution.id, { + status: "ibex_pending", + attempts, + now, + }) + return "ibex_pending" + } + + return fail(`unexpected payment status: ${String(result.value)}`, account.id) +} + +export type ExecutePendingSplitsSummary = { + claimed: number + sent: number + ibexPending: number + failed: number + retried: number +} + +/** + * Cron executor: claims due `pending` executions one at a time (atomically, so + * concurrent runs never claim the same row) and pays each share as an + * intraledger send from the receiving wallet to the split destination. + */ +export const executePendingSplits = async ({ + now = new Date(), +}: { now?: Date } = {}): Promise => { + const summary: ExecutePendingSplitsSummary = { + claimed: 0, + sent: 0, + ibexPending: 0, + failed: 0, + retried: 0, + } + if (!PaymentSplitsConfig.enabled) return summary + + const executions = PaymentSplitExecutionsRepository() + for (let i = 0; i < PaymentSplitsConfig.executorBatchSize; i++) { + const execution = await executions.claimNextDue(now) + if (execution instanceof Error) return execution + if (execution === null) break + + summary.claimed += 1 + const outcome = await executeOne(execution, now) + if (outcome === "sent") summary.sent += 1 + else if (outcome === "ibex_pending") summary.ibexPending += 1 + else if (outcome === "failed") summary.failed += 1 + else summary.retried += 1 + } + + return summary +} diff --git a/src/app/payment-splits/get-splits.ts b/src/app/payment-splits/get-splits.ts new file mode 100644 index 000000000..37db54e91 --- /dev/null +++ b/src/app/payment-splits/get-splits.ts @@ -0,0 +1,9 @@ +import { + PaymentSplitRulesRepository, + type PaymentSplitRuleRecord, +} from "@services/mongoose/payment-split-rules" + +export const getPaymentSplits = async ( + walletId: string, +): Promise => + PaymentSplitRulesRepository().listByWalletId(walletId) diff --git a/src/app/payment-splits/index.ts b/src/app/payment-splits/index.ts new file mode 100644 index 000000000..3c5dcb813 --- /dev/null +++ b/src/app/payment-splits/index.ts @@ -0,0 +1,7 @@ +export * from "./set-splits" +export * from "./delete-split" +export * from "./get-splits" +export * from "./record-split-executions" +export * from "./execute-pending-splits" +export * from "./sweep-stale-splits" +export * from "./reconcile-splits" diff --git a/src/app/payment-splits/reconcile-splits.ts b/src/app/payment-splits/reconcile-splits.ts new file mode 100644 index 000000000..386c4257d --- /dev/null +++ b/src/app/payment-splits/reconcile-splits.ts @@ -0,0 +1,115 @@ +import { PaymentSplitsConfig } from "@config" + +import Ibex from "@services/ibex/client" +import { baseLogger } from "@services/logger" +import { WalletsRepository } from "@services/mongoose" +import { PaymentSplitRulesRepository } from "@services/mongoose/payment-split-rules" + +import { recordSplitExecutions } from "./record-split-executions" + +const logger = baseLogger.child({ module: "payment-splits-reconcile" }) + +// IBEX transaction type ids (GET /v2/transaction-types/all). +const IBEX_LIGHTNING_RECEIVE = 1 +// Type 9 (crypto receive: USDT on Ethereum) deliberately does NOT trigger in +// v1. Bridge virtual-account deposits arrive as the same type on the same +// address, so a split could not tell them apart (ENG-653 decision 2). + +// Transactions read per wallet per run. The window is short, so a page this +// size covers any plausible receive rate. +const TRANSACTIONS_PAGE_SIZE = 100 + +const PaymentHashRegex = /^[0-9a-f]{64}$/i + +export type ReconcileSplitsSummary = { + wallets: number + receivesSeen: number + recorded: number + errors: number +} + +/** + * Safety net for missed or failed webhooks: re-reads recent Lightning receives + * on every wallet with enabled split rules and records any execution the + * webhook did not. The trigger key is the payment hash — the same key the + * webhook uses — so a receive the webhook already recorded is a duplicate-key + * no-op. + */ +export const reconcileSplits = async ({ + windowMs, + now = new Date(), +}: { + windowMs: number + now?: Date +}): Promise => { + const summary: ReconcileSplitsSummary = { + wallets: 0, + receivesSeen: 0, + recorded: 0, + errors: 0, + } + if (!PaymentSplitsConfig.enabled) return summary + + const walletIds = await PaymentSplitRulesRepository().listWalletIdsWithEnabledRules() + if (walletIds instanceof Error) return walletIds + + const since = now.getTime() - windowMs + + for (const walletId of walletIds) { + summary.wallets += 1 + + const wallet = await WalletsRepository().findById(walletId as WalletId) + if (wallet instanceof Error) { + summary.errors += 1 + logger.warn({ walletId, error: wallet }, "split reconcile: wallet lookup failed") + continue + } + + const transactions = await Ibex.getAccountTransactions({ + account_id: walletId, + limit: TRANSACTIONS_PAGE_SIZE, + }) + if (transactions instanceof Error) { + summary.errors += 1 + logger.warn({ walletId, error: transactions }, "split reconcile: IBEX read failed") + continue + } + + for (const tx of transactions) { + if (tx.transactionTypeId !== IBEX_LIGHTNING_RECEIVE) continue + if (!tx.id || !tx.createdAt) continue + const receivedAt = new Date(tx.createdAt) + if (Number.isNaN(receivedAt.getTime()) || receivedAt.getTime() < since) continue + + summary.receivesSeen += 1 + + // The list endpoint carries no payment hash; the details endpoint does. + const details = await Ibex.getTransactionDetails(tx.id as IbexTransactionId) + if (details instanceof Error) { + summary.errors += 1 + logger.warn({ walletId, txId: tx.id, error: details }, "split reconcile: details") + continue + } + const paymentHash = details.invoice?.hash + if (typeof paymentHash !== "string" || !PaymentHashRegex.test(paymentHash)) continue + + const recorded = await recordSplitExecutions({ + walletId, + triggerKey: paymentHash, + paymentHash, + amount: Number(details.amount ?? tx.amount ?? 0), + currency: wallet.currency, + receivedAt, + now, + }) + if (recorded instanceof Error) { + summary.errors += 1 + logger.warn({ walletId, txId: tx.id, error: recorded }, "split reconcile: record") + continue + } + summary.recorded += recorded.recorded + } + } + + return summary +} diff --git a/src/app/payment-splits/record-split-executions.ts b/src/app/payment-splits/record-split-executions.ts new file mode 100644 index 000000000..a74ae208e --- /dev/null +++ b/src/app/payment-splits/record-split-executions.ts @@ -0,0 +1,92 @@ +import { PaymentSplitsConfig } from "@config" + +import { computeShare, isSplitCurrency } from "@domain/payment-splits" + +import { isInternalTransferInvoice } from "@services/mongoose/internal-transfer-invoices" +import { PaymentSplitExecutionsRepository } from "@services/mongoose/payment-split-executions" +import { PaymentSplitRulesRepository } from "@services/mongoose/payment-split-rules" + +export type RecordSplitExecutionsResult = { + recorded: number + duplicates: number + skippedMin: number +} + +const NOTHING: RecordSplitExecutionsResult = { recorded: 0, duplicates: 0, skippedMin: 0 } + +/** + * Records one pending split execution per enabled rule on `walletId` for one + * receive. Shared by the on-receive webhook and the reconcile cron, which both + * key the receive by its payment hash so the unique (triggerKey, ruleId) index + * dedupes them against each other and against webhook replays. + * + * Never sends: payouts happen only in the cron executor. + * + * `receivedAt`, when known (the cron reads it from IBEX), keeps a rule from + * firing on a receive that settled before the rule was created. + */ +export const recordSplitExecutions = async ({ + walletId, + triggerKey, + paymentHash, + amount, + currency, + receivedAt, + now = new Date(), +}: { + walletId: string + triggerKey: string + paymentHash?: string + amount: number + currency: string + receivedAt?: Date + now?: Date +}): Promise => { + if (!PaymentSplitsConfig.enabled) return NOTHING + if (!isSplitCurrency(currency)) return NOTHING + + const rules = await PaymentSplitRulesRepository().listByWalletId(walletId, { + enabledOnly: true, + }) + if (rules instanceof Error) return rules + if (rules.length === 0) return NOTHING + + // Invoices Flash minted for its own transfers (P2P, Fygaro top-ups, referral + // payouts, split payouts, cutover moves) never trigger splits. + if (paymentHash) { + const internal = await isInternalTransferInvoice(paymentHash) + if (internal instanceof Error) return internal + if (internal) return NOTHING + } + + const executions = PaymentSplitExecutionsRepository() + const result = { ...NOTHING } + for (const rule of rules) { + if (receivedAt && receivedAt.getTime() < rule.createdAt.getTime()) continue + + const share = computeShare({ + amount, + currency, + percent: rule.percent, + minShareCents: PaymentSplitsConfig.minShareCents, + }) + if (share instanceof Error) return share + + const inserted = await executions.insert({ + triggerKey, + ruleId: rule.id, + walletId, + destinationWalletId: rule.destinationWalletId, + currency, + triggerAmountMinor: share.triggerAmountMinor, + shareCents: share.shareCents, + status: share.status === "ok" ? "pending" : "skipped_min", + now, + }) + if (inserted instanceof Error) return inserted + if (!inserted.inserted) result.duplicates += 1 + else if (share.status === "skipped_min") result.skippedMin += 1 + else result.recorded += 1 + } + return result +} diff --git a/src/app/payment-splits/set-splits.ts b/src/app/payment-splits/set-splits.ts new file mode 100644 index 000000000..58b418021 --- /dev/null +++ b/src/app/payment-splits/set-splits.ts @@ -0,0 +1,137 @@ +import { PaymentSplitsConfig } from "@config" + +import { checkedToUsername } from "@domain/accounts" +import { InvalidWalletForAccountError } from "@domain/accounts/errors" +import { + PaymentSplitsDisabledError, + SplitDestinationNotFoundError, + validateSplitSet, + type SplitSetEntry, +} from "@domain/payment-splits" +import { checkedToWalletId } from "@domain/wallets" + +import { LockService } from "@services/lock" +import { AccountsRepository, WalletsRepository } from "@services/mongoose" +import { + PaymentSplitRulesRepository, + type PaymentSplitRuleRecord, +} from "@services/mongoose/payment-split-rules" + +export type SplitInput = { destination: string; percent: number } + +/** + * Resolves the source wallet of a split mutation and re-checks that it belongs + * to the calling account. The wallet-id middleware already enforces this for + * every atWalletLevel mutation; this is defense in depth. + */ +export const resolveOwnedWallet = async ({ + account, + walletId: uncheckedWalletId, +}: { + account: Account + walletId: string +}): Promise => { + const walletId = checkedToWalletId(uncheckedWalletId) + if (walletId instanceof Error) return walletId + + const wallet = await WalletsRepository().findById(walletId) + if (wallet instanceof Error) return wallet + if (wallet.accountId !== account.id) return new InvalidWalletForAccountError() + return wallet +} + +/** + * A destination is either a walletId or a Flash username. A username resolves + * to that account's wallet in the SOURCE wallet's currency — not to its + * default wallet, which may be BTC or the other cash currency. + */ +const resolveDestination = async ({ + destination, + sourceCurrency, +}: { + destination: string + sourceCurrency: WalletCurrency +}): Promise => { + const trimmed = destination.trim() + + const walletId = checkedToWalletId(trimmed) + if (!(walletId instanceof Error)) { + const wallet = await WalletsRepository().findById(walletId) + if (wallet instanceof Error) { + return new SplitDestinationNotFoundError(`No wallet found for ${trimmed}`) + } + return wallet + } + + const username = checkedToUsername(trimmed.replace(/^@/, "")) + if (username instanceof Error) { + return new SplitDestinationNotFoundError(`No Flash user found for ${trimmed}`) + } + + const account = await AccountsRepository().findByUsername(username) + if (account instanceof Error) { + return new SplitDestinationNotFoundError(`No Flash user found for ${trimmed}`) + } + + const wallets = await WalletsRepository().listByAccountId(account.id) + if (wallets instanceof Error) return wallets + + const match = wallets.find((wallet) => wallet.currency === sourceCurrency) + if (!match) { + return new SplitDestinationNotFoundError( + `${trimmed} has no ${sourceCurrency} wallet to receive splits`, + ) + } + return match +} + +/** + * Replaces the wallet's split rules with `splits`. Runs under the wallet lock + * so two concurrent sets cannot interleave their delete/insert and leave the + * wallet with more than 100% allocated. + */ +export const setPaymentSplits = async ({ + account, + walletId, + splits, +}: { + account: Account + walletId: string + splits: SplitInput[] +}): Promise => { + if (!PaymentSplitsConfig.enabled) return new PaymentSplitsDisabledError() + + const sourceWallet = await resolveOwnedWallet({ account, walletId }) + if (sourceWallet instanceof Error) return sourceWallet + + const entries: SplitSetEntry[] = [] + for (const { destination, percent } of splits) { + const destinationWallet = await resolveDestination({ + destination, + sourceCurrency: sourceWallet.currency, + }) + if (destinationWallet instanceof Error) return destinationWallet + entries.push({ + destinationWalletId: destinationWallet.id, + destinationCurrency: destinationWallet.currency, + percent, + }) + } + + const valid = validateSplitSet({ + sourceWalletId: sourceWallet.id, + sourceCurrency: sourceWallet.currency, + entries, + }) + if (valid instanceof Error) return valid + + return LockService().lockWalletId(sourceWallet.id, async () => + PaymentSplitRulesRepository().replaceForWallet( + sourceWallet.id, + entries.map(({ destinationWalletId, percent }) => ({ + destinationWalletId, + percent, + })), + ), + ) +} diff --git a/src/app/payment-splits/sweep-stale-splits.ts b/src/app/payment-splits/sweep-stale-splits.ts new file mode 100644 index 000000000..6e45b09b5 --- /dev/null +++ b/src/app/payment-splits/sweep-stale-splits.ts @@ -0,0 +1,53 @@ +import { PaymentSplitsConfig } from "@config" + +import { notifyOpsEvent } from "@services/alerts/ops-events" +import { baseLogger } from "@services/logger" +import { PaymentSplitExecutionsRepository } from "@services/mongoose/payment-split-executions" + +export const STALE_SENDING_MS = 10 * 60 * 1000 + +// Bound on rows moved per run, so a pathological backlog cannot hold the cron. +const MAX_SWEEP = 100 + +/** + * A row still `sending` after STALE_SENDING_MS means the executor died between + * claiming it and recording the outcome — the send may or may not have gone + * out. It becomes `unknown` and raises an ops alert. It is NEVER resent + * automatically: the send's idempotency cache lives only 24h and a crash before + * the cache write leaves nothing to replay, so a resend could pay twice. + */ +export const sweepStaleSplits = async ({ + now = new Date(), +}: { now?: Date } = {}): Promise<{ markedUnknown: number } | Error> => { + if (!PaymentSplitsConfig.enabled) return { markedUnknown: 0 } + + const executions = PaymentSplitExecutionsRepository() + const cutoff = new Date(now.getTime() - STALE_SENDING_MS) + let markedUnknown = 0 + + for (let i = 0; i < MAX_SWEEP; i++) { + const stale = await executions.markNextStaleUnknown(cutoff, now) + if (stale instanceof Error) return stale + if (stale === null) break + + markedUnknown += 1 + baseLogger.error( + { executionId: stale.id, walletId: stale.walletId }, + "payment split stuck in sending; marked unknown for manual resolution", + ) + notifyOpsEvent({ + flow: "transfer", + phase: "split-unknown", + status: "failed", + amount: { value: (stale.shareCents / 100).toFixed(2), currency: "USD" }, + meta: { + executionId: stale.id, + senderWalletId: stale.walletId, + recipientWalletId: stale.destinationWalletId, + reason: "stuck-in-sending", + }, + }) + } + + return { markedUnknown } +} diff --git a/src/config/locales/en.json b/src/config/locales/en.json index f625da861..e0e5ab783 100644 --- a/src/config/locales/en.json +++ b/src/config/locales/en.json @@ -134,6 +134,16 @@ "title": "Top-up on its way", "body": "We're crediting {{amount}} to your wallet now. It should appear shortly." } + }, + "paymentSplit": { + "sent": { + "title": "Payment split sent", + "body": "{{amount}} from a payment you received was sent to your split recipient." + }, + "failed": { + "title": "Payment split failed", + "body": "We could not send a {{amount}} payment split. The money is still in your wallet." + } } } } diff --git a/src/config/locales/es.json b/src/config/locales/es.json index 8253f7ccc..6a7177e91 100644 --- a/src/config/locales/es.json +++ b/src/config/locales/es.json @@ -130,6 +130,16 @@ "title": "Tu recarga está en camino", "body": "Estamos acreditando {{amount}} a tu billetera. Debería aparecer en breve." } + }, + "paymentSplit": { + "sent": { + "title": "División de pago enviada", + "body": "Se enviaron {{amount}} de un pago que recibiste al destinatario de tu división." + }, + "failed": { + "title": "La división de pago falló", + "body": "No pudimos enviar una división de pago de {{amount}}. El dinero sigue en tu billetera." + } } } } diff --git a/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts new file mode 100644 index 000000000..cb45f2ca7 --- /dev/null +++ b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts @@ -0,0 +1,313 @@ +const mockSend = jest.fn() +const mockAuthorizeSend = jest.fn() +const mockNotify = jest.fn() +const mockFindWalletById = jest.fn() +const mockFindAccountById = jest.fn() + +jest.mock("@app/payments/send-intraledger", () => ({ + intraledgerPaymentSendWalletIdForUsdWallet: (...args: unknown[]) => mockSend(...args), +})) + +jest.mock("@app/payments/authorize-send", () => ({ + authorizeSend: (...args: unknown[]) => mockAuthorizeSend(...args), +})) + +jest.mock("@app/notifications/send-outcome-notification", () => ({ + sendOutcomeNotificationBestEffort: (...args: unknown[]) => mockNotify(...args), +})) + +jest.mock("@services/mongoose", () => ({ + WalletsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindWalletById(...args), + })), + AccountsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindAccountById(...args), + })), +})) + +jest.mock("@services/mongoose/payment-split-executions", () => ({ + PaymentSplitExecutionsRepository: jest.fn(() => mockStore.repo), +})) + +import { PaymentSplitsConfig } from "@config" +import { + executePendingSplits, + splitIdempotencyKey, +} from "@app/payment-splits/execute-pending-splits" +import { PaymentSendStatus } from "@domain/bitcoin/lightning" +import { InactiveAccountError, IntraledgerLimitsExceededError } from "@domain/errors" +import { InternalTransferMarkerError } from "@domain/payment-splits" +import { SendLimitsUnavailableError } from "@domain/payments/errors" +import { PaymentSendRateLimiterExceededError } from "@domain/rate-limit/errors" +import { WalletCurrency } from "@domain/shared" +import { InsufficientIbexBalance } from "@services/ibex/errors" + +import { createFakeExecutionsStore } from "./fake-executions-store" + +// Read lazily by the repository mock above, so defining it after the imports +// is safe: the mock factory only captures the reference. +const mockStore = createFakeExecutionsStore() + +const SOURCE = "source-wallet" +const DEST = "dest-wallet" +const OWNER = { id: "owner-account", level: 1 } +const T0 = new Date("2026-10-11T12:00:00Z") +const minutes = (m: number) => new Date(T0.getTime() + m * 60_000) + +const seed = async (shareCents = 250): Promise<{ id: string }> => { + const inserted = await mockStore.repo.insert({ + triggerKey: "a".repeat(64), + ruleId: "rule-1", + walletId: SOURCE, + destinationWalletId: DEST, + currency: WalletCurrency.Usdt, + triggerAmountMinor: "10000000", + shareCents, + status: "pending", + now: T0, + }) + if (!inserted.inserted) throw new Error("seed collided") + return { id: inserted.id } +} + +const onlyRow = () => [...mockStore.rows.values()][0] + +const original = { ...PaymentSplitsConfig } + +beforeEach(() => { + jest.clearAllMocks() + mockStore.rows.clear() + Object.assign(PaymentSplitsConfig, { + enabled: true, + maxAttempts: 5, + executorBatchSize: 50, + }) + mockFindWalletById.mockResolvedValue({ + id: SOURCE, + accountId: OWNER.id, + currency: WalletCurrency.Usdt, + }) + mockFindAccountById.mockResolvedValue(OWNER) + mockAuthorizeSend.mockResolvedValue(true) + mockNotify.mockResolvedValue(undefined) +}) + +afterAll(() => { + Object.assign(PaymentSplitsConfig, original) +}) + +describe("executePendingSplits", () => { + it("pays the share from the receiving wallet through the guarded intraledger send", async () => { + const { id } = await seed(250) + mockSend.mockResolvedValue(PaymentSendStatus.Success) + + const summary = await executePendingSplits({ now: T0 }) + + expect(summary).toEqual({ + claimed: 1, + sent: 1, + ibexPending: 0, + failed: 0, + retried: 0, + }) + expect(mockSend).toHaveBeenCalledWith( + expect.objectContaining({ + senderWalletId: SOURCE, + recipientWalletId: DEST, + amount: 250, + memo: "Payment split", + senderAccount: OWNER, + idempotencyKey: `split:${id}`, + }), + ) + + // The authorize hook is the ENG-573 guard for an intraledger send. + await mockSend.mock.calls[0][0].authorize() + expect(mockAuthorizeSend).toHaveBeenCalledWith({ + senderAccount: OWNER, + senderWalletId: SOURCE, + amount: { currency: "USD", cents: 250 }, + kind: "intraledger", + }) + + expect(onlyRow()).toMatchObject({ status: "sent", attempts: 1 }) + expect(mockNotify).toHaveBeenCalledTimes(1) + expect(mockNotify).toHaveBeenCalledWith( + expect.objectContaining({ + accountId: OWNER.id, + phraseBase: "notification.paymentSplit.sent", + amountArg: "US$2.50", + }), + ) + }) + + it("marks an IBEX pending send ibex_pending and never retries it", async () => { + await seed() + mockSend.mockResolvedValue(PaymentSendStatus.Pending) + + await executePendingSplits({ now: T0 }) + expect(onlyRow()).toMatchObject({ status: "ibex_pending", attempts: 1 }) + + await executePendingSplits({ now: minutes(60) }) + expect(mockSend).toHaveBeenCalledTimes(1) + expect(mockNotify).not.toHaveBeenCalled() + }) + + it.each([ + ["an ENG-573 limit rejection", new IntraledgerLimitsExceededError("over limit")], + ["insufficient balance", new InsufficientIbexBalance(new Error("insufficient"))], + ["an inactive account", new InactiveAccountError()], + ])("fails on the first attempt with %s and pushes the owner", async (_label, error) => { + await seed() + mockSend.mockResolvedValue(error) + + const summary = await executePendingSplits({ now: T0 }) + + expect(summary).toMatchObject({ claimed: 1, failed: 1, retried: 0 }) + expect(onlyRow()).toMatchObject({ + status: "failed", + attempts: 1, + lastError: error.name, + }) + expect(mockNotify).toHaveBeenCalledWith( + expect.objectContaining({ + accountId: OWNER.id, + phraseBase: "notification.paymentSplit.failed", + }), + ) + }) + + it("fails an IBEX Failure status and pushes the owner", async () => { + await seed() + mockSend.mockResolvedValue(PaymentSendStatus.Failure) + + await executePendingSplits({ now: T0 }) + + expect(onlyRow()).toMatchObject({ status: "failed" }) + expect(mockNotify).toHaveBeenCalledTimes(1) + }) + + it("retries a retryable error with 1/2/4/8 minute backoff, then fails at the attempt budget", async () => { + await seed() + mockSend.mockResolvedValue(new PaymentSendRateLimiterExceededError()) + + let now = T0 + const expectedBackoffs = [1, 2, 4, 8] + for (const [i, backoff] of expectedBackoffs.entries()) { + await executePendingSplits({ now }) + expect(onlyRow()).toMatchObject({ status: "pending", attempts: i + 1 }) + const next = new Date(now.getTime() + backoff * 60_000) + expect(onlyRow().nextAttemptAt).toEqual(next) + + // Not due yet: nothing is claimed a second early. + await executePendingSplits({ now: new Date(next.getTime() - 1000) }) + expect(mockSend).toHaveBeenCalledTimes(i + 1) + now = next + } + + await executePendingSplits({ now }) + expect(mockSend).toHaveBeenCalledTimes(5) + expect(onlyRow()).toMatchObject({ status: "failed", attempts: 5 }) + expect(mockNotify).toHaveBeenCalledTimes(1) + }) + + it.each([ + ["send guard unavailable", new SendLimitsUnavailableError("redis")], + ["marker write failure", new InternalTransferMarkerError("mongo")], + ])("treats %s as retryable", async (_label, error) => { + await seed() + mockSend.mockResolvedValue(error) + + await executePendingSplits({ now: T0 }) + + expect(onlyRow()).toMatchObject({ status: "pending", attempts: 1 }) + expect(mockNotify).not.toHaveBeenCalled() + }) + + it("uses the same idempotency key on every retry of one execution", async () => { + const { id } = await seed() + mockSend + .mockResolvedValueOnce(new PaymentSendRateLimiterExceededError()) + .mockResolvedValueOnce(PaymentSendStatus.Success) + + await executePendingSplits({ now: T0 }) + await executePendingSplits({ now: minutes(1) }) + + const keys = mockSend.mock.calls.map((call) => call[0].idempotencyKey) + expect(keys).toEqual([splitIdempotencyKey(id), splitIdempotencyKey(id)]) + expect(onlyRow()).toMatchObject({ status: "sent", attempts: 2 }) + }) + + it("lets only one of two concurrent executors claim a row", async () => { + await seed() + let release: (v: unknown) => void = () => undefined + const sendReached = new Promise((reached) => { + mockSend.mockImplementation( + () => + new Promise((resolve) => { + release = resolve + reached() + }), + ) + }) + + const first = executePendingSplits({ now: T0 }) + const second = executePendingSplits({ now: T0 }) + const secondSummary = await second + await sendReached + release(PaymentSendStatus.Success) + const firstSummary = await first + + expect(mockSend).toHaveBeenCalledTimes(1) + expect( + [firstSummary, secondSummary].map((s) => (s instanceof Error ? -1 : s.claimed)), + ).toEqual([1, 0]) + }) + + it("claims at most executorBatchSize rows per run", async () => { + PaymentSplitsConfig.executorBatchSize = 2 + for (let i = 0; i < 3; i++) { + await mockStore.repo.insert({ + triggerKey: `${i}`.repeat(64), + ruleId: "rule-1", + walletId: SOURCE, + destinationWalletId: DEST, + currency: WalletCurrency.Usdt, + triggerAmountMinor: "1000000", + shareCents: 10, + status: "pending", + now: T0, + }) + } + mockSend.mockResolvedValue(PaymentSendStatus.Success) + + expect(await executePendingSplits({ now: T0 })).toMatchObject({ claimed: 2 }) + expect(await executePendingSplits({ now: T0 })).toMatchObject({ claimed: 1 }) + }) + + it("never sends a skipped_min row", async () => { + await mockStore.repo.insert({ + triggerKey: "b".repeat(64), + ruleId: "rule-1", + walletId: SOURCE, + destinationWalletId: DEST, + currency: WalletCurrency.Usd, + triggerAmountMinor: "3", + shareCents: 0, + status: "skipped_min", + now: T0, + }) + + expect(await executePendingSplits({ now: T0 })).toMatchObject({ claimed: 0 }) + expect(mockSend).not.toHaveBeenCalled() + }) + + it("does nothing while the feature flag is off", async () => { + await seed() + PaymentSplitsConfig.enabled = false + + expect(await executePendingSplits({ now: T0 })).toMatchObject({ claimed: 0 }) + expect(onlyRow()).toMatchObject({ status: "pending" }) + expect(mockSend).not.toHaveBeenCalled() + }) +}) diff --git a/test/flash/unit/app/payment-splits/fake-executions-store.ts b/test/flash/unit/app/payment-splits/fake-executions-store.ts new file mode 100644 index 000000000..8852a2aed --- /dev/null +++ b/test/flash/unit/app/payment-splits/fake-executions-store.ts @@ -0,0 +1,90 @@ +// In-memory stand-in for PaymentSplitExecutionsRepository that models the two +// properties the split pipeline relies on: the unique (triggerKey, ruleId) +// index and the atomic pending -> sending claim. +type Row = { + id: string + triggerKey: string + ruleId: string + walletId: string + destinationWalletId: string + currency: string + triggerAmountMinor: string + shareCents: number + status: string + attempts: number + nextAttemptAt: Date + lastError?: string + createdAt: Date + updatedAt: Date +} + +export const createFakeExecutionsStore = () => { + const rows = new Map() + let seq = 0 + + const repo = { + insert: async (data: { + triggerKey: string + ruleId: string + walletId: string + destinationWalletId: string + currency: string + triggerAmountMinor: string + shareCents: number + status: string + now?: Date + }) => { + const { now = new Date(), ...fields } = data + const dup = [...rows.values()].find( + (r) => r.triggerKey === fields.triggerKey && r.ruleId === fields.ruleId, + ) + if (dup) return { inserted: false as const } + const id = `exec-${++seq}` + rows.set(id, { + ...fields, + id, + attempts: 0, + nextAttemptAt: now, + createdAt: now, + updatedAt: now, + }) + return { inserted: true as const, id } + }, + claimNextDue: async (now: Date) => { + const due = [...rows.values()] + .filter((r) => r.status === "pending" && r.nextAttemptAt <= now) + .sort((a, b) => a.nextAttemptAt.getTime() - b.nextAttemptAt.getTime())[0] + if (!due) return null + due.status = "sending" + due.updatedAt = now + return { ...due } + }, + finishClaimed: async ( + id: string, + update: { + status: string + attempts: number + nextAttemptAt?: Date + lastError?: string + now?: Date + }, + ) => { + const row = rows.get(id) + if (!row || row.status !== "sending") return false + const { now = new Date(), ...fields } = update + Object.assign(row, fields, { updatedAt: now }) + return true + }, + markNextStaleUnknown: async (cutoff: Date, now: Date) => { + const stale = [...rows.values()].find( + (r) => r.status === "sending" && r.updatedAt < cutoff, + ) + if (!stale) return null + stale.status = "unknown" + stale.updatedAt = now + return { ...stale } + }, + } + + return { rows, repo } +} diff --git a/test/flash/unit/app/payment-splits/reconcile-splits.spec.ts b/test/flash/unit/app/payment-splits/reconcile-splits.spec.ts new file mode 100644 index 000000000..70123157f --- /dev/null +++ b/test/flash/unit/app/payment-splits/reconcile-splits.spec.ts @@ -0,0 +1,209 @@ +const mockGetAccountTransactions = jest.fn() +const mockGetTransactionDetails = jest.fn() +const mockListRules = jest.fn() +const mockListWalletIds = jest.fn() +const mockIsInternal = jest.fn() +const mockFindWalletById = jest.fn() + +jest.mock("@services/ibex/client", () => ({ + __esModule: true, + default: { + getAccountTransactions: (...args: unknown[]) => mockGetAccountTransactions(...args), + getTransactionDetails: (...args: unknown[]) => mockGetTransactionDetails(...args), + }, +})) + +jest.mock("@services/mongoose", () => ({ + WalletsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindWalletById(...args), + })), +})) + +jest.mock("@services/mongoose/payment-split-rules", () => ({ + PaymentSplitRulesRepository: jest.fn(() => ({ + listByWalletId: (...args: unknown[]) => mockListRules(...args), + listWalletIdsWithEnabledRules: (...args: unknown[]) => mockListWalletIds(...args), + })), +})) + +jest.mock("@services/mongoose/payment-split-executions", () => ({ + PaymentSplitExecutionsRepository: jest.fn(() => mockStore.repo), +})) + +jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ + isInternalTransferInvoice: (...args: unknown[]) => mockIsInternal(...args), +})) + +import { PaymentSplitsConfig } from "@config" +import { recordSplitExecutions } from "@app/payment-splits/record-split-executions" +import { reconcileSplits } from "@app/payment-splits/reconcile-splits" +import { WalletCurrency } from "@domain/shared" + +import { createFakeExecutionsStore } from "./fake-executions-store" + +// Read lazily by the repository mock above, so defining it after the imports +// is safe: the mock factory only captures the reference. +const mockStore = createFakeExecutionsStore() + +const WALLET = "source-wallet" +const NOW = new Date("2026-10-11T12:00:00Z") +const WINDOW_MS = 15 * 60 * 1000 +const RULE_CREATED = new Date("2026-10-11T11:00:00Z") +const HASH_1 = "1".repeat(64) +const HASH_2 = "2".repeat(64) + +const minutesAgo = (m: number) => new Date(NOW.getTime() - m * 60_000).toISOString() + +const tx = (id: string, transactionTypeId: number, createdAt: string, amount = 10) => ({ + id, + transactionTypeId, + createdAt, + amount, + accountId: WALLET, +}) + +const hashes: Record = { "tx-1": HASH_1, "tx-2": HASH_2 } + +const originalEnabled = PaymentSplitsConfig.enabled + +beforeEach(() => { + jest.clearAllMocks() + mockStore.rows.clear() + PaymentSplitsConfig.enabled = true + mockListWalletIds.mockResolvedValue([WALLET]) + mockFindWalletById.mockResolvedValue({ id: WALLET, currency: WalletCurrency.Usdt }) + mockListRules.mockResolvedValue([ + { + id: "rule-1", + walletId: WALLET, + destinationWalletId: "dest", + percent: 10, + enabled: true, + createdAt: RULE_CREATED, + }, + ]) + mockIsInternal.mockResolvedValue(false) + mockGetTransactionDetails.mockImplementation(async (id: string) => ({ + id, + amount: 10, + invoice: { hash: hashes[id] }, + })) +}) + +afterAll(() => { + PaymentSplitsConfig.enabled = originalEnabled +}) + +describe("reconcileSplits", () => { + it("records a missed Lightning receive keyed by its payment hash", async () => { + mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, minutesAgo(5))]) + + const summary = await reconcileSplits({ windowMs: WINDOW_MS, now: NOW }) + + expect(summary).toEqual({ wallets: 1, receivesSeen: 1, recorded: 1, errors: 0 }) + expect(mockGetAccountTransactions).toHaveBeenCalledWith( + expect.objectContaining({ account_id: WALLET }), + ) + expect(mockGetTransactionDetails).toHaveBeenCalledWith("tx-1") + expect([...mockStore.rows.values()]).toEqual([ + expect.objectContaining({ triggerKey: HASH_1, ruleId: "rule-1", shareCents: 100 }), + ]) + }) + + it("does not duplicate a receive the webhook already recorded", async () => { + // The webhook path keys the same receive by the same hash. + await recordSplitExecutions({ + walletId: WALLET, + triggerKey: HASH_1, + paymentHash: HASH_1, + amount: 10, + currency: WalletCurrency.Usdt, + now: NOW, + }) + mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, minutesAgo(5))]) + + const summary = await reconcileSplits({ windowMs: WINDOW_MS, now: NOW }) + + expect(summary).toMatchObject({ receivesSeen: 1, recorded: 0 }) + expect(mockStore.rows.size).toBe(1) + }) + + it("ignores a receive that settled before the rule was created", async () => { + mockListRules.mockResolvedValue([ + { + id: "rule-1", + walletId: WALLET, + destinationWalletId: "dest", + percent: 10, + enabled: true, + createdAt: new Date(NOW.getTime() - 60_000), + }, + ]) + mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, minutesAgo(5))]) + + expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + recorded: 0, + }) + expect(mockStore.rows.size).toBe(0) + }) + + it("ignores type 9 (ETH USDT crypto receives) and every non-receive type", async () => { + mockGetAccountTransactions.mockResolvedValue([ + tx("tx-9", 9, minutesAgo(1)), + tx("tx-2", 2, minutesAgo(1)), + tx("tx-3", 3, minutesAgo(1)), + ]) + + expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + receivesSeen: 0, + recorded: 0, + }) + expect(mockGetTransactionDetails).not.toHaveBeenCalled() + expect(mockStore.rows.size).toBe(0) + }) + + it("ignores receives older than the lookback window", async () => { + mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, minutesAgo(16))]) + + expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + receivesSeen: 0, + }) + expect(mockGetTransactionDetails).not.toHaveBeenCalled() + }) + + it("skips a receive whose hash marks it an internal transfer", async () => { + mockIsInternal.mockImplementation(async (hash: string) => hash === HASH_2) + mockGetAccountTransactions.mockResolvedValue([ + tx("tx-1", 1, minutesAgo(2)), + tx("tx-2", 1, minutesAgo(3)), + ]) + + expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + receivesSeen: 2, + recorded: 1, + }) + expect([...mockStore.rows.values()].map((r) => r.triggerKey)).toEqual([HASH_1]) + }) + + it("counts an IBEX failure for one wallet and carries on", async () => { + mockListWalletIds.mockResolvedValue([WALLET, "other"]) + mockGetAccountTransactions + .mockResolvedValueOnce(new Error("ibex down")) + .mockResolvedValueOnce([tx("tx-1", 1, minutesAgo(2))]) + + expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + wallets: 2, + errors: 1, + recorded: 1, + }) + }) + + it("does nothing while the feature flag is off", async () => { + PaymentSplitsConfig.enabled = false + + expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + wallets: 0, + }) + expect(mockListWalletIds).not.toHaveBeenCalled() + }) +}) diff --git a/test/flash/unit/app/payment-splits/record-split-executions.spec.ts b/test/flash/unit/app/payment-splits/record-split-executions.spec.ts new file mode 100644 index 000000000..08a99a37a --- /dev/null +++ b/test/flash/unit/app/payment-splits/record-split-executions.spec.ts @@ -0,0 +1,148 @@ +const mockListRules = jest.fn() +const mockIsInternal = jest.fn() + +jest.mock("@services/mongoose/payment-split-rules", () => ({ + PaymentSplitRulesRepository: jest.fn(() => ({ + listByWalletId: (...args: unknown[]) => mockListRules(...args), + })), +})) + +jest.mock("@services/mongoose/payment-split-executions", () => ({ + PaymentSplitExecutionsRepository: jest.fn(() => mockStore.repo), +})) + +jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ + isInternalTransferInvoice: (...args: unknown[]) => mockIsInternal(...args), +})) + +import { PaymentSplitsConfig } from "@config" +import { recordSplitExecutions } from "@app/payment-splits/record-split-executions" +import { WalletCurrency } from "@domain/shared" + +import { createFakeExecutionsStore } from "./fake-executions-store" + +// Read lazily by the repository mock above, so defining it after the imports +// is safe: the mock factory only captures the reference. +const mockStore = createFakeExecutionsStore() + +const WALLET = "source-wallet" +const HASH = "a".repeat(64) +const RULE_CREATED = new Date("2026-10-01T00:00:00Z") + +const rules = [ + { + id: "rule-1", + walletId: WALLET, + destinationWalletId: "dest-1", + percent: 10, + enabled: true, + createdAt: RULE_CREATED, + }, + { + id: "rule-2", + walletId: WALLET, + destinationWalletId: "dest-2", + percent: 25, + enabled: true, + createdAt: RULE_CREATED, + }, +] + +const record = (overrides: Record = {}) => + recordSplitExecutions({ + walletId: WALLET, + triggerKey: HASH, + paymentHash: HASH, + amount: 10, + currency: WalletCurrency.Usdt, + ...overrides, + }) + +const originalEnabled = PaymentSplitsConfig.enabled + +beforeEach(() => { + jest.clearAllMocks() + mockStore.rows.clear() + PaymentSplitsConfig.enabled = true + mockListRules.mockResolvedValue(rules) + mockIsInternal.mockResolvedValue(false) +}) + +afterAll(() => { + PaymentSplitsConfig.enabled = originalEnabled +}) + +describe("recordSplitExecutions", () => { + it("records one pending execution per enabled rule with the floored share", async () => { + const result = await record() + + expect(result).toEqual({ recorded: 2, duplicates: 0, skippedMin: 0 }) + expect(mockListRules).toHaveBeenCalledWith(WALLET, { enabledOnly: true }) + const rows = [...mockStore.rows.values()] + expect(rows).toEqual([ + expect.objectContaining({ + triggerKey: HASH, + ruleId: "rule-1", + destinationWalletId: "dest-1", + shareCents: 100, + triggerAmountMinor: "10000000", + status: "pending", + }), + expect.objectContaining({ ruleId: "rule-2", shareCents: 250, status: "pending" }), + ]) + }) + + it("records one row per rule however many times the same trigger is replayed", async () => { + for (let i = 0; i < 5; i++) await record() + + expect(mockStore.rows.size).toBe(2) + expect(await record()).toEqual({ recorded: 0, duplicates: 2, skippedMin: 0 }) + }) + + it("records nothing for a hash Flash minted for its own transfer", async () => { + mockIsInternal.mockResolvedValue(true) + + expect(await record()).toEqual({ recorded: 0, duplicates: 0, skippedMin: 0 }) + expect(mockIsInternal).toHaveBeenCalledWith(HASH) + expect(mockStore.rows.size).toBe(0) + }) + + it("records nothing while the feature flag is off", async () => { + PaymentSplitsConfig.enabled = false + + expect(await record()).toEqual({ recorded: 0, duplicates: 0, skippedMin: 0 }) + expect(mockListRules).not.toHaveBeenCalled() + expect(mockStore.rows.size).toBe(0) + }) + + it("records nothing for a BTC wallet", async () => { + expect(await record({ currency: WalletCurrency.Btc })).toEqual({ + recorded: 0, + duplicates: 0, + skippedMin: 0, + }) + expect(mockStore.rows.size).toBe(0) + }) + + it("records a below-minimum share as skipped_min so it is never sent", async () => { + const result = await record({ amount: 0.03, currency: WalletCurrency.Usd }) + + expect(result).toEqual({ recorded: 0, duplicates: 0, skippedMin: 2 }) + expect([...mockStore.rows.values()].map((r) => r.status)).toEqual([ + "skipped_min", + "skipped_min", + ]) + }) + + it("ignores a receive that settled before the rule existed", async () => { + const result = await record({ receivedAt: new Date("2026-09-30T23:59:59Z") }) + + expect(result).toEqual({ recorded: 0, duplicates: 0, skippedMin: 0 }) + }) + + it("returns repository errors instead of throwing", async () => { + const err = new Error("mongo down") + mockIsInternal.mockResolvedValue(err) + expect(await record()).toBe(err) + }) +}) diff --git a/test/flash/unit/app/payment-splits/set-splits.spec.ts b/test/flash/unit/app/payment-splits/set-splits.spec.ts new file mode 100644 index 000000000..2a3766633 --- /dev/null +++ b/test/flash/unit/app/payment-splits/set-splits.spec.ts @@ -0,0 +1,288 @@ +const mockFindWalletById = jest.fn() +const mockListByAccountId = jest.fn() +const mockFindByUsername = jest.fn() +const mockReplaceForWallet = jest.fn() +const mockFindRuleById = jest.fn() +const mockDeleteRuleById = jest.fn() +const mockLockWalletId = jest.fn() + +jest.mock("@services/mongoose", () => ({ + AccountsRepository: jest.fn(() => ({ + findByUsername: (...args: unknown[]) => mockFindByUsername(...args), + })), + WalletsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindWalletById(...args), + listByAccountId: (...args: unknown[]) => mockListByAccountId(...args), + })), +})) + +jest.mock("@services/mongoose/payment-split-rules", () => ({ + PaymentSplitRulesRepository: jest.fn(() => ({ + replaceForWallet: (...args: unknown[]) => mockReplaceForWallet(...args), + findById: (...args: unknown[]) => mockFindRuleById(...args), + deleteById: (...args: unknown[]) => mockDeleteRuleById(...args), + })), +})) + +jest.mock("@services/lock", () => ({ + LockService: jest.fn(() => ({ + lockWalletId: (...args: unknown[]) => mockLockWalletId(...args), + })), +})) + +import { PaymentSplitsConfig } from "@config" +import { deletePaymentSplit } from "@app/payment-splits/delete-split" +import { setPaymentSplits } from "@app/payment-splits/set-splits" +import { InvalidWalletForAccountError } from "@domain/accounts/errors" +import { + CouldNotFindAccountFromUsernameError, + CouldNotFindWalletFromIdError, +} from "@domain/errors" +import { + PaymentSplitsDisabledError, + SplitCurrencyMismatchError, + SplitDestinationNotFoundError, + SplitPercentExceededError, + SplitRuleNotFoundError, +} from "@domain/payment-splits" +import { WalletCurrency } from "@domain/shared" + +const OWNER = { id: "owner-account" } as Account +const SOURCE_ID = "11111111-1111-4111-8111-111111111111" +const DEST_ID = "22222222-2222-4222-8222-222222222222" +const OTHER_ID = "33333333-3333-4333-8333-333333333333" +const ALICE_USD = "44444444-4444-4444-8444-444444444444" +const ALICE_USDT = "55555555-5555-4555-8555-555555555555" +const ALICE_BTC = "66666666-6666-4666-8666-666666666666" + +const wallets: Record = { + [SOURCE_ID]: { id: SOURCE_ID, accountId: OWNER.id, currency: WalletCurrency.Usdt }, + [DEST_ID]: { id: DEST_ID, accountId: "dest-account", currency: WalletCurrency.Usdt }, + [OTHER_ID]: { id: OTHER_ID, accountId: "other-account", currency: WalletCurrency.Usdt }, +} as unknown as Record + +const aliceWallets = [ + // Alice's DEFAULT wallet is BTC; a username must not resolve to it blindly. + { id: ALICE_BTC, accountId: "alice", currency: WalletCurrency.Btc }, + { id: ALICE_USD, accountId: "alice", currency: WalletCurrency.Usd }, + { id: ALICE_USDT, accountId: "alice", currency: WalletCurrency.Usdt }, +] + +const originalEnabled = PaymentSplitsConfig.enabled + +beforeEach(() => { + jest.clearAllMocks() + PaymentSplitsConfig.enabled = true + mockFindWalletById.mockImplementation( + async (id: string) => wallets[id] ?? new CouldNotFindWalletFromIdError(), + ) + mockFindByUsername.mockImplementation(async (username: string) => + username.toLowerCase() === "alice" + ? { id: "alice", defaultWalletId: ALICE_BTC } + : new CouldNotFindAccountFromUsernameError(username), + ) + mockListByAccountId.mockResolvedValue(aliceWallets) + mockLockWalletId.mockImplementation(async (_id: string, fn: () => Promise) => + fn(), + ) + mockReplaceForWallet.mockImplementation( + async (walletId: string, rules: { destinationWalletId: string; percent: number }[]) => + rules.map((rule, i) => ({ id: `rule-${i}`, walletId, enabled: true, ...rule })), + ) +}) + +afterAll(() => { + PaymentSplitsConfig.enabled = originalEnabled +}) + +describe("setPaymentSplits", () => { + it("resolves a walletId destination and replaces the set under the wallet lock", async () => { + const result = await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [{ destination: DEST_ID, percent: 25 }], + }) + + expect(result).toEqual([ + expect.objectContaining({ destinationWalletId: DEST_ID, percent: 25 }), + ]) + expect(mockLockWalletId).toHaveBeenCalledWith(SOURCE_ID, expect.any(Function)) + expect(mockReplaceForWallet).toHaveBeenCalledWith(SOURCE_ID, [ + { destinationWalletId: DEST_ID, percent: 25 }, + ]) + }) + + it("only writes while the lock is held", async () => { + let lockHeld = false + mockLockWalletId.mockImplementation( + async (_id: string, fn: () => Promise) => { + lockHeld = true + try { + return await fn() + } finally { + lockHeld = false + } + }, + ) + mockReplaceForWallet.mockImplementation(async () => { + expect(lockHeld).toBe(true) + return [] + }) + + await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [{ destination: DEST_ID, percent: 25 }], + }) + expect(mockReplaceForWallet).toHaveBeenCalledTimes(1) + }) + + it("resolves a username to that account's wallet in the SOURCE currency, not its default", async () => { + await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [{ destination: "@Alice", percent: 10 }], + }) + + expect(mockListByAccountId).toHaveBeenCalledWith("alice") + expect(mockReplaceForWallet).toHaveBeenCalledWith(SOURCE_ID, [ + { destinationWalletId: ALICE_USDT, percent: 10 }, + ]) + }) + + it("rejects a username with no wallet in the source currency", async () => { + mockListByAccountId.mockResolvedValue([aliceWallets[0]]) + + const result = await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [{ destination: "alice", percent: 10 }], + }) + + expect(result).toBeInstanceOf(SplitDestinationNotFoundError) + expect(mockReplaceForWallet).not.toHaveBeenCalled() + }) + + it("rejects an unknown username or walletId", async () => { + expect( + await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [{ destination: "nobody", percent: 10 }], + }), + ).toBeInstanceOf(SplitDestinationNotFoundError) + expect( + await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [{ destination: "77777777-7777-4777-8777-777777777777", percent: 10 }], + }), + ).toBeInstanceOf(SplitDestinationNotFoundError) + }) + + it("rejects a walletId destination in another currency", async () => { + const result = await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [{ destination: ALICE_USD, percent: 10 }], + }) + // ALICE_USD is not in the walletId fixture map; give it one. + expect(result).toBeInstanceOf(SplitDestinationNotFoundError) + + wallets[ALICE_USD] = aliceWallets[1] as unknown as Wallet + expect( + await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [{ destination: ALICE_USD, percent: 10 }], + }), + ).toBeInstanceOf(SplitCurrencyMismatchError) + delete wallets[ALICE_USD] + }) + + it("re-checks that the source wallet belongs to the caller", async () => { + const result = await setPaymentSplits({ + account: OWNER, + walletId: OTHER_ID, + splits: [{ destination: DEST_ID, percent: 10 }], + }) + + expect(result).toBeInstanceOf(InvalidWalletForAccountError) + expect(mockLockWalletId).not.toHaveBeenCalled() + expect(mockReplaceForWallet).not.toHaveBeenCalled() + }) + + it("rejects a set over 100% without writing", async () => { + mockFindWalletById.mockImplementation(async (id: string) => + id === SOURCE_ID + ? wallets[SOURCE_ID] + : { id, accountId: "x", currency: WalletCurrency.Usdt }, + ) + const result = await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [ + { destination: DEST_ID, percent: 60 }, + { destination: OTHER_ID, percent: 41 }, + ], + }) + + expect(result).toBeInstanceOf(SplitPercentExceededError) + expect(mockReplaceForWallet).not.toHaveBeenCalled() + }) + + it("rejects everything while the feature flag is off", async () => { + PaymentSplitsConfig.enabled = false + const result = await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [{ destination: DEST_ID, percent: 25 }], + }) + expect(result).toBeInstanceOf(PaymentSplitsDisabledError) + expect(mockFindWalletById).not.toHaveBeenCalled() + }) +}) + +describe("deletePaymentSplit", () => { + it("deletes a rule on the caller's wallet", async () => { + mockFindRuleById.mockResolvedValue({ id: "rule-1", walletId: SOURCE_ID }) + mockDeleteRuleById.mockResolvedValue(true) + + const result = await deletePaymentSplit({ + account: OWNER, + walletId: SOURCE_ID, + splitId: "rule-1", + }) + + expect(result).toBe(true) + expect(mockDeleteRuleById).toHaveBeenCalledWith("rule-1") + expect(mockLockWalletId).toHaveBeenCalledWith(SOURCE_ID, expect.any(Function)) + }) + + it("refuses a rule that belongs to another wallet", async () => { + mockFindRuleById.mockResolvedValue({ id: "rule-1", walletId: OTHER_ID }) + + const result = await deletePaymentSplit({ + account: OWNER, + walletId: SOURCE_ID, + splitId: "rule-1", + }) + + expect(result).toBeInstanceOf(SplitRuleNotFoundError) + expect(mockDeleteRuleById).not.toHaveBeenCalled() + }) + + it("reports a missing rule", async () => { + mockFindRuleById.mockResolvedValue(null) + expect( + await deletePaymentSplit({ account: OWNER, walletId: SOURCE_ID, splitId: "nope" }), + ).toBeInstanceOf(SplitRuleNotFoundError) + }) + + it("re-checks wallet ownership", async () => { + expect( + await deletePaymentSplit({ account: OWNER, walletId: OTHER_ID, splitId: "rule-1" }), + ).toBeInstanceOf(InvalidWalletForAccountError) + expect(mockFindRuleById).not.toHaveBeenCalled() + }) +}) diff --git a/test/flash/unit/app/payment-splits/sweep-stale-splits.spec.ts b/test/flash/unit/app/payment-splits/sweep-stale-splits.spec.ts new file mode 100644 index 000000000..e1171fbbd --- /dev/null +++ b/test/flash/unit/app/payment-splits/sweep-stale-splits.spec.ts @@ -0,0 +1,91 @@ +const mockNotifyOpsEvent = jest.fn() + +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: (...args: unknown[]) => mockNotifyOpsEvent(...args), +})) + +jest.mock("@services/mongoose/payment-split-executions", () => ({ + PaymentSplitExecutionsRepository: jest.fn(() => mockStore.repo), +})) + +import { PaymentSplitsConfig } from "@config" +import { + STALE_SENDING_MS, + sweepStaleSplits, +} from "@app/payment-splits/sweep-stale-splits" + +import { createFakeExecutionsStore } from "./fake-executions-store" + +// Read lazily by the repository mock above, so defining it after the imports +// is safe: the mock factory only captures the reference. +const mockStore = createFakeExecutionsStore() + +const T0 = new Date("2026-10-11T12:00:00Z") + +const seedSending = async (claimedAt: Date) => { + await mockStore.repo.insert({ + triggerKey: "a".repeat(64), + ruleId: `rule-${mockStore.rows.size}`, + walletId: "source", + destinationWalletId: "dest", + currency: "USDT", + triggerAmountMinor: "1000000", + shareCents: 100, + status: "pending", + now: claimedAt, + }) + await mockStore.repo.claimNextDue(claimedAt) +} + +const originalEnabled = PaymentSplitsConfig.enabled + +beforeEach(() => { + jest.clearAllMocks() + mockStore.rows.clear() + PaymentSplitsConfig.enabled = true +}) + +afterAll(() => { + PaymentSplitsConfig.enabled = originalEnabled +}) + +describe("sweepStaleSplits", () => { + it("marks a row stuck in sending for over 10 minutes unknown and alerts ops, without sending", async () => { + await seedSending(T0) + const now = new Date(T0.getTime() + STALE_SENDING_MS + 1) + + expect(await sweepStaleSplits({ now })).toEqual({ markedUnknown: 1 }) + expect([...mockStore.rows.values()][0]).toMatchObject({ status: "unknown" }) + expect(mockNotifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "split-unknown", + status: "failed", + }), + ) + + // An unknown row is never claimable again. + expect(await mockStore.repo.claimNextDue(new Date(now.getTime() + 3_600_000))).toBe( + null, + ) + }) + + it("leaves a row that is still within the sending window alone", async () => { + await seedSending(T0) + + expect( + await sweepStaleSplits({ now: new Date(T0.getTime() + STALE_SENDING_MS - 1000) }), + ).toEqual({ markedUnknown: 0 }) + expect([...mockStore.rows.values()][0]).toMatchObject({ status: "sending" }) + expect(mockNotifyOpsEvent).not.toHaveBeenCalled() + }) + + it("does nothing while the feature flag is off", async () => { + await seedSending(T0) + PaymentSplitsConfig.enabled = false + + expect(await sweepStaleSplits({ now: new Date(T0.getTime() + 86_400_000) })).toEqual({ + markedUnknown: 0, + }) + }) +}) From bea80adca2dcf5b6f79bf6ad788eff7d6d643bbd Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 22:57:49 -0700 Subject: [PATCH 06/13] feat(ibex-webhook): record payment splits on Lightning and LNURL receives Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WxkKbrg2Xohf3mQj7BLrs6 --- .../ibex/webhook-server/routes/on-receive.ts | 44 ++++ .../routes/on-receive-splits.spec.ts | 243 ++++++++++++++++++ 2 files changed, 287 insertions(+) create mode 100644 test/flash/unit/services/ibex/webhook-server/routes/on-receive-splits.spec.ts diff --git a/src/services/ibex/webhook-server/routes/on-receive.ts b/src/services/ibex/webhook-server/routes/on-receive.ts index 9fd6c7d43..c6210331d 100644 --- a/src/services/ibex/webhook-server/routes/on-receive.ts +++ b/src/services/ibex/webhook-server/routes/on-receive.ts @@ -17,6 +17,7 @@ import { } from "@domain/notifications" import { getCurrentPriceAsDisplayPriceRatio } from "@app/prices" import { removeDeviceTokens } from "@app/users/remove-device-tokens" +import { recordSplitExecutions } from "@app/payment-splits/record-split-executions" import { ZapRequestModel } from "@services/mongoose/zap-request" import { ZapPublisher } from "@services/nostr/zapPublisher" import { ibexWebhookPaths } from "@services/ibex/webhook-config" @@ -70,6 +71,47 @@ const fetchPaymentContext = async ( next() } +// ENG-653: record (never send) the payment splits a Lightning/LNURL receive +// triggers. Failure here must not fail the webhook: IBEX would retry it and +// re-run the receive notification, and the reconcile cron records any receive +// this step missed. Duplicate deliveries are duplicate-key no-ops. +const recordPaymentSplits = async ( + req: PaymentRequest, + _resp: Response, + next: NextFunction, +) => { + if (!req.paymentContext) return next() + + try { + const { transaction } = req.body + const paymentHash = parsePaymentHash(transaction?.invoice?.hash) + const amount = Number(transaction?.amount) + if (paymentHash && Number.isFinite(amount)) { + const { receiverWallet } = req.paymentContext + const result = await recordSplitExecutions({ + walletId: receiverWallet.id, + triggerKey: paymentHash, + paymentHash, + amount, + currency: receiverWallet.currency, + }) + if (result instanceof Error) { + logger.error( + { err: result, walletId: receiverWallet.id, paymentHash }, + "Failed to record payment splits; the reconcile cron will retry", + ) + } + } + } catch (err) { + logger.error( + { err }, + "Failed to record payment splits; the reconcile cron will retry", + ) + } + + next() +} + const sendLightningNotification = async ( req: PaymentRequest, resp: Response, @@ -211,6 +253,7 @@ router.post( authenticate, logRequest, fetchPaymentContext, + recordPaymentSplits, sendLightningNotification, sendZapReceipt, (_req: Request, resp: Response) => resp.status(200).end(), @@ -223,6 +266,7 @@ router.post( authenticate, logRequest, fetchPaymentContext, + recordPaymentSplits, sendLightningNotification, sendZapReceipt, (_req: Request, resp: Response) => resp.status(200).end(), diff --git a/test/flash/unit/services/ibex/webhook-server/routes/on-receive-splits.spec.ts b/test/flash/unit/services/ibex/webhook-server/routes/on-receive-splits.spec.ts new file mode 100644 index 000000000..1fbac6031 --- /dev/null +++ b/test/flash/unit/services/ibex/webhook-server/routes/on-receive-splits.spec.ts @@ -0,0 +1,243 @@ +const mockFindWalletById = jest.fn() +const mockFindAccountById = jest.fn() +const mockFindUserById = jest.fn() +const mockLightningTxReceived = jest.fn() +const mockOnChainTxReceived = jest.fn() +const mockListRules = jest.fn() +const mockIsInternal = jest.fn() +const mockSend = jest.fn() + +jest.mock("@services/ibex/webhook-server/middleware", () => ({ + authenticate: jest.fn((_req, _res, next) => next()), + logRequest: jest.fn((_req, _res, next) => next()), + validateIbexIp: jest.fn((_req, _res, next) => next()), +})) + +jest.mock("@services/logger", () => ({ + baseLogger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})) + +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: jest.fn(), +})) + +jest.mock("@services/mongoose", () => ({ + WalletsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindWalletById(...args), + })), + AccountsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindAccountById(...args), + })), + UsersRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindUserById(...args), + })), +})) + +jest.mock("@services/notifications", () => ({ + NotificationsService: jest.fn(() => ({ + lightningTxReceived: (...args: unknown[]) => mockLightningTxReceived(...args), + onChainTxReceived: (...args: unknown[]) => mockOnChainTxReceived(...args), + })), +})) + +jest.mock("@app/prices", () => ({ + getCurrentPriceAsDisplayPriceRatio: jest.fn(async () => new Error("no price")), +})) + +jest.mock("@app/users/remove-device-tokens", () => ({ + removeDeviceTokens: jest.fn(), +})) + +jest.mock("@services/mongoose/zap-request", () => ({ + ZapRequestModel: { findOne: jest.fn(async () => null) }, +})) + +jest.mock("@services/nostr/zapPublisher", () => ({ + ZapPublisher: { publishFromWebhook: jest.fn() }, +})) + +jest.mock("@services/mongoose/payment-split-rules", () => ({ + PaymentSplitRulesRepository: jest.fn(() => ({ + listByWalletId: (...args: unknown[]) => mockListRules(...args), + })), +})) + +jest.mock("@services/mongoose/payment-split-executions", () => ({ + PaymentSplitExecutionsRepository: jest.fn(() => mockStore.repo), +})) + +jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ + isInternalTransferInvoice: (...args: unknown[]) => mockIsInternal(...args), +})) + +// The webhook must never pay a split: payouts happen only in the cron executor. +jest.mock("@app/payments/send-intraledger", () => ({ + intraledgerPaymentSendWalletIdForUsdWallet: (...args: unknown[]) => mockSend(...args), + intraledgerPaymentSendWalletIdForBtcWallet: (...args: unknown[]) => mockSend(...args), +})) + +import { PaymentSplitsConfig } from "@config" +import { WalletCurrency } from "@domain/shared" +import { paths, router } from "@services/ibex/webhook-server/routes/on-receive" + +import { createFakeExecutionsStore } from "test/flash/unit/app/payment-splits/fake-executions-store" + +// Read lazily by the repository mock above, so defining it after the imports +// is safe: the mock factory only captures the reference. +const mockStore = createFakeExecutionsStore() + +const WALLET = "receiver-wallet" +const HASH = "c".repeat(64) + +type Handle = (req: unknown, res: unknown, next: () => void) => unknown + +const handlesFor = (path: string): Handle[] => { + const layer = ( + router.stack as Array<{ + route?: { path: string; stack: Array<{ handle: Handle }> } + }> + ).find((l) => l.route?.path === path) + if (!layer?.route) throw new Error(`no route ${path}`) + // Skip the rate limiter (first handle); the IBEX guards are mocked through. + return layer.route.stack.slice(1).map((l) => l.handle) +} + +const post = async (path: string, body: Record) => { + const res = { + statusCode: 0, + status(code: number) { + this.statusCode = code + return this + }, + sendStatus(code: number) { + this.statusCode = code + return this + }, + end() { + return this + }, + } + const req = { body } as Record + for (const handle of handlesFor(path)) { + let advanced = false + await handle(req, res, () => { + advanced = true + }) + if (!advanced) break + } + return res +} + +const lightningBody = (hash = HASH) => ({ + transaction: { accountId: WALLET, amount: 10, invoice: { hash } }, + receivedMsat: 1000, +}) + +const originalEnabled = PaymentSplitsConfig.enabled + +beforeEach(() => { + jest.clearAllMocks() + mockStore.rows.clear() + PaymentSplitsConfig.enabled = true + mockFindWalletById.mockResolvedValue({ id: WALLET, currency: WalletCurrency.Usdt }) + mockFindAccountById.mockResolvedValue({ + id: "acct", + kratosUserId: "user", + displayCurrency: "USD", + }) + mockFindUserById.mockResolvedValue({ id: "user", deviceTokens: [] }) + mockLightningTxReceived.mockResolvedValue(true) + mockOnChainTxReceived.mockResolvedValue(true) + mockListRules.mockResolvedValue([ + { + id: "rule-1", + walletId: WALLET, + destinationWalletId: "dest", + percent: 20, + enabled: true, + createdAt: new Date("2026-01-01T00:00:00Z"), + }, + ]) + mockIsInternal.mockResolvedValue(false) +}) + +afterAll(() => { + PaymentSplitsConfig.enabled = originalEnabled +}) + +describe("on-receive webhook payment splits (ENG-653)", () => { + it.each([ + ["invoice", paths.invoice], + ["lnurl", paths.lnurl], + ])("the %s route records splits and never sends", async (_name, path) => { + const res = await post(path, lightningBody()) + + expect(res.statusCode).toBe(200) + expect([...mockStore.rows.values()]).toEqual([ + expect.objectContaining({ + triggerKey: HASH, + walletId: WALLET, + destinationWalletId: "dest", + shareCents: 200, + status: "pending", + }), + ]) + expect(mockSend).not.toHaveBeenCalled() + // The receive notification still runs after the split step. + expect(mockLightningTxReceived).toHaveBeenCalledTimes(1) + }) + + it("a replayed webhook records nothing new", async () => { + await post(paths.invoice, lightningBody()) + await post(paths.invoice, lightningBody()) + + expect(mockStore.rows.size).toBe(1) + }) + + it.each([ + ["onchain", paths.onchain], + ["zap", paths.zap], + ])("the %s route records nothing", async (_name, path) => { + const res = await post(path, { + transaction: { accountId: WALLET, amount: 10, invoice: { hash: HASH }, hash: "tx" }, + }) + + expect(res.statusCode).toBe(200) + expect(mockListRules).not.toHaveBeenCalled() + expect(mockStore.rows.size).toBe(0) + }) + + it("records nothing for a Fygaro top-up or referral payout (marked internal hash)", async () => { + mockIsInternal.mockImplementation(async (hash: string) => hash === HASH) + + const res = await post(paths.invoice, lightningBody()) + + expect(res.statusCode).toBe(200) + expect(mockIsInternal).toHaveBeenCalledWith(HASH) + expect(mockStore.rows.size).toBe(0) + expect(mockLightningTxReceived).toHaveBeenCalledTimes(1) + }) + + it("still returns 200 when recording fails", async () => { + mockListRules.mockResolvedValue(new Error("mongo down")) + + const res = await post(paths.invoice, lightningBody()) + + expect(res.statusCode).toBe(200) + expect(mockLightningTxReceived).toHaveBeenCalledTimes(1) + }) + + it("still returns 200 when recording throws", async () => { + mockListRules.mockRejectedValue(new Error("boom")) + + const res = await post(paths.invoice, lightningBody()) + + expect(res.statusCode).toBe(200) + }) + + it("skips a body with no valid payment hash", async () => { + await post(paths.invoice, lightningBody("not-a-hash")) + + expect(mockListRules).not.toHaveBeenCalled() + }) +}) From f1fb8946b5588dc841a45d3801d2aa3867ab3a0b Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 22:57:49 -0700 Subject: [PATCH 07/13] feat(cron): run split reconcile, executor and stale sweep Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WxkKbrg2Xohf3mQj7BLrs6 --- src/servers/cron.ts | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/servers/cron.ts b/src/servers/cron.ts index f544207c6..b1540912f 100644 --- a/src/servers/cron.ts +++ b/src/servers/cron.ts @@ -1,6 +1,11 @@ import { OnChain, Lightning, Wallets, Payments, Swap, Accounts } from "@app" -import { BridgeConfig, getCronConfig, TWO_MONTHS_IN_MS } from "@config" +import { + BridgeConfig, + getCronConfig, + PaymentSplitsConfig, + TWO_MONTHS_IN_MS, +} from "@config" import { ErrorLevel } from "@domain/shared" import { OperationInterruptedError } from "@domain/errors" @@ -25,6 +30,11 @@ import { reconcileBridgeAndIbexWithdrawals, } from "@services/bridge/reconciliation" import { checkFygaroTreasuryFloat } from "@services/fygaro/float-monitor" +import { + executePendingSplits, + reconcileSplits, + sweepStaleSplits, +} from "@app/payment-splits" import { elapsedSinceTimestamp, sleep } from "@utils" import { rebalancingInternalChannels } from "@services/lnd/rebalancing" @@ -89,6 +99,32 @@ const reconcileBridgeWithdrawalsJob = async () => { if (result instanceof Error) throw result } +// ENG-653 payment splits. Reconcile first so a receive whose webhook was +// missed is recorded before the executor runs; the sweep then flags any row a +// crashed run left in `sending`. All three self-guard on the config flag. +const reconcileSplitPaymentsJob = async () => { + if (!PaymentSplitsConfig.enabled) return + + const result = await reconcileSplits({ windowMs: RECONCILE_WINDOW_MS }) + if (result instanceof Error) throw result + logger.info({ result }, "payment split reconcile finished") +} + +const executeSplitPaymentsJob = async () => { + if (!PaymentSplitsConfig.enabled) return + + const result = await executePendingSplits() + if (result instanceof Error) throw result + logger.info({ result }, "payment split executor finished") +} + +const sweepStaleSplitsJob = async () => { + if (!PaymentSplitsConfig.enabled) return + + const result = await sweepStaleSplits() + if (result instanceof Error) throw result +} + // Bankowner treasury float check for Fygaro auto-credit. Runs every cron // invocation on whatever schedule the chart gives this Job; the fygaro-webhook // workload ALSO runs it every fygaro.float.checkIntervalMs (treasury-loop.ts) @@ -146,6 +182,9 @@ const main = async () => { ...(cronConfig.swapEnabled ? [swapOutJob] : []), reconcileBridgeDepositsJob, reconcileBridgeWithdrawalsJob, + reconcileSplitPaymentsJob, + executeSplitPaymentsJob, + sweepStaleSplitsJob, checkFygaroFloatJob, evidenceRetentionJob, deleteExpiredPaymentFlows, From 15b74b41cffffe6457a353ef8764053914fff2c9 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 22:57:49 -0700 Subject: [PATCH 08/13] feat(graphql): walletSplitsSet, walletSplitDelete and Wallet.splits Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WxkKbrg2Xohf3mQj7BLrs6 --- dev/apollo-federation/supergraph.graphql | 77 +++++++ src/domain/api-keys/scope-map.ts | 5 + src/graphql/admin/schema.graphql | 29 +++ src/graphql/public/mutations.ts | 5 + .../root/mutation/wallet-split-delete.ts | 49 ++++ .../public/root/mutation/wallet-splits-set.ts | 77 +++++++ src/graphql/public/schema.graphql | 67 ++++++ .../public/types/payload/wallet-splits-set.ts | 17 ++ src/graphql/shared/types/abstract/wallet.ts | 5 + src/graphql/shared/types/object/btc-wallet.ts | 6 + src/graphql/shared/types/object/usd-wallet.ts | 7 + .../shared/types/object/usdt-wallet.ts | 7 + .../shared/types/object/wallet-split.ts | 52 +++++ .../graphql/api-key-scope-enforcement.spec.ts | 2 + .../root/mutation/wallet-splits-set.spec.ts | 215 ++++++++++++++++++ .../unit/graphql/shared/wallet-split.spec.ts | 48 ++++ 16 files changed, 668 insertions(+) create mode 100644 src/graphql/public/root/mutation/wallet-split-delete.ts create mode 100644 src/graphql/public/root/mutation/wallet-splits-set.ts create mode 100644 src/graphql/public/types/payload/wallet-splits-set.ts create mode 100644 src/graphql/shared/types/object/wallet-split.ts create mode 100644 test/flash/unit/graphql/public/root/mutation/wallet-splits-set.spec.ts create mode 100644 test/flash/unit/graphql/shared/wallet-split.spec.ts diff --git a/dev/apollo-federation/supergraph.graphql b/dev/apollo-federation/supergraph.graphql index f5b3c0f71..155c032e5 100644 --- a/dev/apollo-federation/supergraph.graphql +++ b/dev/apollo-federation/supergraph.graphql @@ -840,6 +840,9 @@ type BTCWallet implements Wallet """An unconfirmed incoming onchain balance.""" pendingIncomingBalance: SignedAmount! + """Payment splits are not supported on BTC wallets; always empty.""" + splits: [WalletSplit!]! + """A list of BTC transactions associated with this wallet.""" transactions( """Returns the items in the list that come after the specified cursor.""" @@ -2259,6 +2262,18 @@ type Mutation userUpdateLanguage(input: UserUpdateLanguageInput!): UserUpdateLanguagePayload! userUpdateNpub(input: UserUpdateNpubInput!): UserUpdateNpubPayload! userUpdateUsername(input: UserUpdateUsernameInput!): UserUpdateUsernamePayload! @deprecated(reason: "Username will be moved to @Handle in Accounts. Also SetUsername naming should be used instead of UpdateUsername to reflect the idempotency of Handles") + + """ + Deletes one payment split from a wallet. Splits already triggered still pay out. + """ + walletSplitDelete(input: WalletSplitDeleteInput!): SuccessPayload! + + """ + Replaces the payment splits on a USD or USDT wallet. Each + external Lightning receive on the wallet then pays the given percentages on + to the destination wallets. Percentages must add up to at most 100. + """ + walletSplitsSet(input: WalletSplitsSetInput!): WalletSplitsSetPayload! } type MyReferralInvite @@ -3081,6 +3096,11 @@ type UsdtWallet implements Wallet """An unconfirmed incoming onchain balance.""" pendingIncomingBalance: SignedAmount! + + """ + Payment split rules on this wallet. Visible only to the wallet's owner. + """ + splits: [WalletSplit!]! transactions( """Returns the items in the list that come after the specified cursor.""" after: String @@ -3132,6 +3152,11 @@ type UsdWallet implements Wallet """An unconfirmed incoming onchain balance.""" pendingIncomingBalance: SignedAmount! + + """ + Payment split rules on this wallet. Visible only to the wallet's owner. + """ + splits: [WalletSplit!]! transactions( """Returns the items in the list that come after the specified cursor.""" after: String @@ -3468,6 +3493,9 @@ interface Wallet lnurlp: Lnurl pendingIncomingBalance: SignedAmount! + """Payment split rules on this wallet. Always empty for BTC wallets.""" + splits: [WalletSplit!]! + """ Transactions are ordered anti-chronologically, ie: the newest transaction will be first @@ -3520,3 +3548,52 @@ enum WalletCurrency """Unique identifier of a wallet""" scalar WalletId @join__type(graph: PUBLIC) + +""" +A payment split rule: this percentage of every external Lightning receive on the wallet is paid on to the destination wallet. +""" +type WalletSplit + @join__type(graph: PUBLIC) +{ + createdAt: Timestamp! + destinationWalletId: WalletId! + enabled: Boolean! + id: ID! + + """Whole percent of each receive, from 1 to 100.""" + percent: Int! +} + +input WalletSplitDeleteInput + @join__type(graph: PUBLIC) +{ + splitId: ID! + walletId: WalletId! +} + +input WalletSplitInput + @join__type(graph: PUBLIC) +{ + """A Flash username or a walletId in the same currency.""" + destination: String! + + """Whole percent of each receive, from 1 to 100.""" + percent: Int! +} + +input WalletSplitsSetInput + @join__type(graph: PUBLIC) +{ + """The complete split set. Replaces every existing split.""" + splits: [WalletSplitInput!]! + + """The wallet whose incoming payments are split.""" + walletId: WalletId! +} + +type WalletSplitsSetPayload + @join__type(graph: PUBLIC) +{ + errors: [Error!]! + splits: [WalletSplit!] +} diff --git a/src/domain/api-keys/scope-map.ts b/src/domain/api-keys/scope-map.ts index 016c40d3c..468e9748f 100644 --- a/src/domain/api-keys/scope-map.ts +++ b/src/domain/api-keys/scope-map.ts @@ -116,6 +116,8 @@ export const apiKeyScopeForField: Readonly> = onChainUsdPaymentSend: "write:wallet", onChainUsdPaymentSendAsBtcDenominated: "write:wallet", onChainPaymentSendAll: "write:wallet", + walletSplitsSet: "write:wallet", + walletSplitDelete: "write:wallet", }) // Nested-field guards: a root field like `me` only requires read:user, but the @@ -141,18 +143,21 @@ export const apiKeyNestedFieldScopes: Record< pendingIncomingBalance: "read:wallet", transactions: "read:transactions", transactionsByAddress: "read:transactions", + splits: "read:wallet", }, UsdWallet: { balance: "read:wallet", pendingIncomingBalance: "read:wallet", transactions: "read:transactions", transactionsByAddress: "read:transactions", + splits: "read:wallet", }, UsdtWallet: { balance: "read:wallet", pendingIncomingBalance: "read:wallet", transactions: "read:transactions", transactionsByAddress: "read:transactions", + splits: "read:wallet", }, }) diff --git a/src/graphql/admin/schema.graphql b/src/graphql/admin/schema.graphql index 434196eb8..a7f9db1d3 100644 --- a/src/graphql/admin/schema.graphql +++ b/src/graphql/admin/schema.graphql @@ -144,6 +144,9 @@ type BTCWallet implements Wallet { """An unconfirmed incoming onchain balance.""" pendingIncomingBalance: SignedAmount! + """Payment splits are not supported on BTC wallets; always empty.""" + splits: [WalletSplit!]! + """A list of BTC transactions associated with this wallet.""" transactions( """Returns the items in the list that come after the specified cursor.""" @@ -743,6 +746,11 @@ type UsdWallet implements Wallet { """An unconfirmed incoming onchain balance.""" pendingIncomingBalance: SignedAmount! + + """ + Payment split rules on this wallet. Visible only to the wallet's owner. + """ + splits: [WalletSplit!]! transactions( """Returns the items in the list that come after the specified cursor.""" after: String @@ -791,6 +799,11 @@ type UsdtWallet implements Wallet { """An unconfirmed incoming onchain balance.""" pendingIncomingBalance: SignedAmount! + + """ + Payment split rules on this wallet. Visible only to the wallet's owner. + """ + splits: [WalletSplit!]! transactions( """Returns the items in the list that come after the specified cursor.""" after: String @@ -863,6 +876,9 @@ interface Wallet { lnurlp: Lnurl pendingIncomingBalance: SignedAmount! + """Payment split rules on this wallet. Always empty for BTC wallets.""" + splits: [WalletSplit!]! + """ Transactions are ordered anti-chronologically, ie: the newest transaction will be first @@ -913,5 +929,18 @@ enum WalletCurrency { """Unique identifier of a wallet""" scalar WalletId +""" +A payment split rule: this percentage of every external Lightning receive on the wallet is paid on to the destination wallet. +""" +type WalletSplit { + createdAt: Timestamp! + destinationWalletId: WalletId! + enabled: Boolean! + id: ID! + + """Whole percent of each receive, from 1 to 100.""" + percent: Int! +} + """Nostr Identity public key""" scalar npub \ No newline at end of file diff --git a/src/graphql/public/mutations.ts b/src/graphql/public/mutations.ts index 1785976f1..9234a3391 100644 --- a/src/graphql/public/mutations.ts +++ b/src/graphql/public/mutations.ts @@ -38,6 +38,8 @@ import CaptchaRequestAuthCodeMutation from "@graphql/shared/root/mutation/captch import CaptchaCreateChallengeMutation from "@graphql/shared/root/mutation/captcha-create-challenge" import QuizCompletedMutation from "@graphql/public/root/mutation/quiz-completed" import AccountDeleteMutation from "@graphql/public/root/mutation/account-delete" +import WalletSplitsSetMutation from "@graphql/public/root/mutation/wallet-splits-set" +import WalletSplitDeleteMutation from "@graphql/public/root/mutation/wallet-split-delete" import UserLoginUpgradeMutation from "@graphql/public/root/mutation/user-login-upgrade" import UserEmailRegistrationInitiateMutation from "@graphql/public/root/mutation/user-email-registration-initiate" import UserEmailRegistrationValidateMutation from "@graphql/public/root/mutation/user-email-registration-validate" @@ -193,6 +195,9 @@ export const mutationFields = { requestCashout: RequestCashoutMutation, initiateCashout: InitiateCashoutMutation, + + walletSplitsSet: WalletSplitsSetMutation, + walletSplitDelete: WalletSplitDeleteMutation, }, }, } diff --git a/src/graphql/public/root/mutation/wallet-split-delete.ts b/src/graphql/public/root/mutation/wallet-split-delete.ts new file mode 100644 index 000000000..11364db4a --- /dev/null +++ b/src/graphql/public/root/mutation/wallet-split-delete.ts @@ -0,0 +1,49 @@ +import { deletePaymentSplit } from "@app/payment-splits/delete-split" + +import { mapAndParseErrorForGqlResponse } from "@graphql/error-map" +import { GT } from "@graphql/index" +import SuccessPayload from "@graphql/shared/types/payload/success-payload" +import WalletId from "@graphql/shared/types/scalar/wallet-id" + +const WalletSplitDeleteInput = GT.Input({ + name: "WalletSplitDeleteInput", + fields: () => ({ + walletId: { type: GT.NonNull(WalletId) }, + splitId: { type: GT.NonNullID }, + }), +}) + +const WalletSplitDeleteMutation = GT.Field< + null, + GraphQLPublicContextAuth, + { input: { walletId: WalletId | InputValidationError; splitId: string } } +>({ + extensions: { + complexity: 120, + }, + type: GT.NonNull(SuccessPayload), + description: + "Deletes one payment split from a wallet. Splits already triggered still pay out.", + args: { + input: { type: GT.NonNull(WalletSplitDeleteInput) }, + }, + resolve: async (_, args, { domainAccount }) => { + const { walletId, splitId } = args.input + if (walletId instanceof Error) { + return { errors: [{ message: walletId.message }], success: false } + } + + const result = await deletePaymentSplit({ + account: domainAccount, + walletId, + splitId, + }) + if (result instanceof Error) { + return { errors: [mapAndParseErrorForGqlResponse(result)], success: false } + } + + return { errors: [], success: true } + }, +}) + +export default WalletSplitDeleteMutation diff --git a/src/graphql/public/root/mutation/wallet-splits-set.ts b/src/graphql/public/root/mutation/wallet-splits-set.ts new file mode 100644 index 000000000..1834dc1b2 --- /dev/null +++ b/src/graphql/public/root/mutation/wallet-splits-set.ts @@ -0,0 +1,77 @@ +import dedent from "dedent" + +import { setPaymentSplits } from "@app/payment-splits/set-splits" + +import { mapAndParseErrorForGqlResponse } from "@graphql/error-map" +import { GT } from "@graphql/index" +import WalletSplitsSetPayload from "@graphql/public/types/payload/wallet-splits-set" +import WalletId from "@graphql/shared/types/scalar/wallet-id" + +const WalletSplitInput = GT.Input({ + name: "WalletSplitInput", + fields: () => ({ + destination: { + type: GT.NonNull(GT.String), + description: "A Flash username or a walletId in the same currency.", + }, + percent: { + type: GT.NonNull(GT.Int), + description: "Whole percent of each receive, from 1 to 100.", + }, + }), +}) + +const WalletSplitsSetInput = GT.Input({ + name: "WalletSplitsSetInput", + fields: () => ({ + walletId: { + type: GT.NonNull(WalletId), + description: "The wallet whose incoming payments are split.", + }, + splits: { + type: GT.NonNullList(WalletSplitInput), + description: "The complete split set. Replaces every existing split.", + }, + }), +}) + +const WalletSplitsSetMutation = GT.Field< + null, + GraphQLPublicContextAuth, + { + input: { + walletId: WalletId | InputValidationError + splits: { destination: string; percent: number }[] + } + } +>({ + extensions: { + complexity: 120, + }, + type: GT.NonNull(WalletSplitsSetPayload), + description: dedent`Replaces the payment splits on a USD or USDT wallet. Each + external Lightning receive on the wallet then pays the given percentages on + to the destination wallets. Percentages must add up to at most 100.`, + args: { + input: { type: GT.NonNull(WalletSplitsSetInput) }, + }, + resolve: async (_, args, { domainAccount }) => { + const { walletId, splits } = args.input + if (walletId instanceof Error) { + return { errors: [{ message: walletId.message }] } + } + + const result = await setPaymentSplits({ + account: domainAccount, + walletId, + splits, + }) + if (result instanceof Error) { + return { errors: [mapAndParseErrorForGqlResponse(result)] } + } + + return { errors: [], splits: result } + }, +}) + +export default WalletSplitsSetMutation diff --git a/src/graphql/public/schema.graphql b/src/graphql/public/schema.graphql index 9b8958f70..d4857fc87 100644 --- a/src/graphql/public/schema.graphql +++ b/src/graphql/public/schema.graphql @@ -385,6 +385,9 @@ type BTCWallet implements Wallet { """An unconfirmed incoming onchain balance.""" pendingIncomingBalance: SignedAmount! + """Payment splits are not supported on BTC wallets; always empty.""" + splits: [WalletSplit!]! + """A list of BTC transactions associated with this wallet.""" transactions( """Returns the items in the list that come after the specified cursor.""" @@ -1877,6 +1880,18 @@ type Mutation { userUpdateLanguage(input: UserUpdateLanguageInput!): UserUpdateLanguagePayload! userUpdateNpub(input: UserUpdateNpubInput!): UserUpdateNpubPayload! userUpdateUsername(input: UserUpdateUsernameInput!): UserUpdateUsernamePayload! @deprecated(reason: "Username will be moved to @Handle in Accounts. Also SetUsername naming should be used instead of UpdateUsername to reflect the idempotency of Handles") + + """ + Deletes one payment split from a wallet. Splits already triggered still pay out. + """ + walletSplitDelete(input: WalletSplitDeleteInput!): SuccessPayload! + + """ + Replaces the payment splits on a USD or USDT wallet. Each + external Lightning receive on the wallet then pays the given percentages on + to the destination wallets. Percentages must add up to at most 100. + """ + walletSplitsSet(input: WalletSplitsSetInput!): WalletSplitsSetPayload! } type MyReferralInvite { @@ -2525,6 +2540,11 @@ type UsdWallet implements Wallet { """An unconfirmed incoming onchain balance.""" pendingIncomingBalance: SignedAmount! + + """ + Payment split rules on this wallet. Visible only to the wallet's owner. + """ + splits: [WalletSplit!]! transactions( """Returns the items in the list that come after the specified cursor.""" after: String @@ -2573,6 +2593,11 @@ type UsdtWallet implements Wallet { """An unconfirmed incoming onchain balance.""" pendingIncomingBalance: SignedAmount! + + """ + Payment split rules on this wallet. Visible only to the wallet's owner. + """ + splits: [WalletSplit!]! transactions( """Returns the items in the list that come after the specified cursor.""" after: String @@ -2837,6 +2862,9 @@ interface Wallet { lnurlp: Lnurl pendingIncomingBalance: SignedAmount! + """Payment split rules on this wallet. Always empty for BTC wallets.""" + splits: [WalletSplit!]! + """ Transactions are ordered anti-chronologically, ie: the newest transaction will be first @@ -2887,6 +2915,45 @@ enum WalletCurrency { """Unique identifier of a wallet""" scalar WalletId +""" +A payment split rule: this percentage of every external Lightning receive on the wallet is paid on to the destination wallet. +""" +type WalletSplit { + createdAt: Timestamp! + destinationWalletId: WalletId! + enabled: Boolean! + id: ID! + + """Whole percent of each receive, from 1 to 100.""" + percent: Int! +} + +input WalletSplitDeleteInput { + splitId: ID! + walletId: WalletId! +} + +input WalletSplitInput { + """A Flash username or a walletId in the same currency.""" + destination: String! + + """Whole percent of each receive, from 1 to 100.""" + percent: Int! +} + +input WalletSplitsSetInput { + """The complete split set. Replaces every existing split.""" + splits: [WalletSplitInput!]! + + """The wallet whose incoming payments are split.""" + walletId: WalletId! +} + +type WalletSplitsSetPayload { + errors: [Error!]! + splits: [WalletSplit!] +} + """Nostr Identity public key""" scalar npub diff --git a/src/graphql/public/types/payload/wallet-splits-set.ts b/src/graphql/public/types/payload/wallet-splits-set.ts new file mode 100644 index 000000000..154809c0f --- /dev/null +++ b/src/graphql/public/types/payload/wallet-splits-set.ts @@ -0,0 +1,17 @@ +import { GT } from "@graphql/index" +import IError from "@graphql/shared/types/abstract/error" +import WalletSplit from "@graphql/shared/types/object/wallet-split" + +const WalletSplitsSetPayload = GT.Object({ + name: "WalletSplitsSetPayload", + fields: () => ({ + errors: { + type: GT.NonNullList(IError), + }, + splits: { + type: GT.List(GT.NonNull(WalletSplit)), + }, + }), +}) + +export default WalletSplitsSetPayload diff --git a/src/graphql/shared/types/abstract/wallet.ts b/src/graphql/shared/types/abstract/wallet.ts index f64afb810..7acc9c160 100644 --- a/src/graphql/shared/types/abstract/wallet.ts +++ b/src/graphql/shared/types/abstract/wallet.ts @@ -6,6 +6,7 @@ import { connectionArgs } from "@graphql/connections" import FractionalCentAmount from "@graphql/public/types/scalar/cent-amount-fraction" import { TransactionConnection } from "../object/transaction" +import WalletSplit from "../object/wallet-split" import WalletCurrency from "../scalar/wallet-currency" import SignedAmount from "../scalar/signed-amount" import OnChainAddress from "../scalar/on-chain-address" @@ -56,6 +57,10 @@ const IWallet = GT.Interface({ }, }, }, + splits: { + type: GT.NonNullList(WalletSplit), + description: "Payment split rules on this wallet. Always empty for BTC wallets.", + }, }), }) diff --git a/src/graphql/shared/types/object/btc-wallet.ts b/src/graphql/shared/types/object/btc-wallet.ts index 04345a8ab..036974caa 100644 --- a/src/graphql/shared/types/object/btc-wallet.ts +++ b/src/graphql/shared/types/object/btc-wallet.ts @@ -23,6 +23,7 @@ import OnChainAddress from "../scalar/on-chain-address" import Lnurl from "../scalar/lnurl" import { TransactionConnection } from "./transaction" +import WalletSplit from "./wallet-split" const BtcWallet = GT.Object({ name: "BTCWallet", @@ -141,6 +142,11 @@ const BtcWallet = GT.Object({ ) }, }, + splits: { + type: GT.NonNullList(WalletSplit), + description: "Payment splits are not supported on BTC wallets; always empty.", + resolve: () => [], + }, }), }) diff --git a/src/graphql/shared/types/object/usd-wallet.ts b/src/graphql/shared/types/object/usd-wallet.ts index 137bf3f0b..0621109ce 100644 --- a/src/graphql/shared/types/object/usd-wallet.ts +++ b/src/graphql/shared/types/object/usd-wallet.ts @@ -26,6 +26,7 @@ import Lnurl from "../scalar/lnurl" import { resolveCashWalletHistoryWalletsForWalletObject } from "./cash-wallet-history" import { TransactionConnection } from "./transaction" +import WalletSplit, { resolveWalletSplits } from "./wallet-split" const UsdWallet = GT.Object({ name: "UsdWallet", @@ -177,6 +178,12 @@ const UsdWallet = GT.Object({ ) }, }, + splits: { + type: GT.NonNullList(WalletSplit), + description: + "Payment split rules on this wallet. Visible only to the wallet's owner.", + resolve: resolveWalletSplits, + }, }), }) diff --git a/src/graphql/shared/types/object/usdt-wallet.ts b/src/graphql/shared/types/object/usdt-wallet.ts index 3845ae696..26099a257 100644 --- a/src/graphql/shared/types/object/usdt-wallet.ts +++ b/src/graphql/shared/types/object/usdt-wallet.ts @@ -29,6 +29,7 @@ import Lnurl from "../scalar/lnurl" import { resolveCashWalletHistoryWalletsForWalletObject } from "./cash-wallet-history" import { TransactionConnection } from "./transaction" +import WalletSplit, { resolveWalletSplits } from "./wallet-split" const UsdtWallet = GT.Object({ name: "UsdtWallet", @@ -167,6 +168,12 @@ const UsdtWallet = GT.Object({ ) }, }, + splits: { + type: GT.NonNullList(WalletSplit), + description: + "Payment split rules on this wallet. Visible only to the wallet's owner.", + resolve: resolveWalletSplits, + }, }), }) diff --git a/src/graphql/shared/types/object/wallet-split.ts b/src/graphql/shared/types/object/wallet-split.ts new file mode 100644 index 000000000..3d92567c2 --- /dev/null +++ b/src/graphql/shared/types/object/wallet-split.ts @@ -0,0 +1,52 @@ +import { GT } from "@graphql/index" +import { mapError } from "@graphql/error-map" + +import { getPaymentSplits } from "@app/payment-splits/get-splits" + +import Timestamp from "../scalar/timestamp" +import WalletId from "../scalar/wallet-id" + +type WalletSplitSource = { + id: string + destinationWalletId: string + percent: number + enabled: boolean + createdAt: Date +} + +const WalletSplit = GT.Object({ + name: "WalletSplit", + description: + "A payment split rule: this percentage of every external Lightning receive on the wallet is paid on to the destination wallet.", + fields: () => ({ + id: { type: GT.NonNullID }, + destinationWalletId: { type: GT.NonNull(WalletId) }, + percent: { + type: GT.NonNull(GT.Int), + description: "Whole percent of each receive, from 1 to 100.", + }, + enabled: { type: GT.NonNull(GT.Boolean) }, + createdAt: { type: GT.NonNull(Timestamp) }, + }), +}) + +/** + * Resolver for the `splits` field on UsdWallet/UsdtWallet. Split rules are + * private to the wallet's owner: a session whose account does not own the + * wallet gets an empty list. + */ +export const resolveWalletSplits = async ( + source: Wallet, + _args: unknown, + ctx: unknown, +): Promise => { + const domainAccount = (ctx as { domainAccount?: { id: string } } | undefined) + ?.domainAccount + if (domainAccount && domainAccount.id !== source.accountId) return [] + + const splits = await getPaymentSplits(source.id) + if (splits instanceof Error) throw mapError(splits) + return splits +} + +export default WalletSplit diff --git a/test/flash/unit/graphql/api-key-scope-enforcement.spec.ts b/test/flash/unit/graphql/api-key-scope-enforcement.spec.ts index b01962445..ea51ca613 100644 --- a/test/flash/unit/graphql/api-key-scope-enforcement.spec.ts +++ b/test/flash/unit/graphql/api-key-scope-enforcement.spec.ts @@ -182,6 +182,8 @@ describe("apiKeyNestedFieldScopes completeness (nested escalation net)", () => { "transactionsCount", "csvTransactions", "wallets", + // ENG-653: payment split rules reveal who a wallet pays and how much. + "splits", ] it("gates every sensitive field on each guarded type", () => { diff --git a/test/flash/unit/graphql/public/root/mutation/wallet-splits-set.spec.ts b/test/flash/unit/graphql/public/root/mutation/wallet-splits-set.spec.ts new file mode 100644 index 000000000..f00807da7 --- /dev/null +++ b/test/flash/unit/graphql/public/root/mutation/wallet-splits-set.spec.ts @@ -0,0 +1,215 @@ +const mockFindWalletById = jest.fn() +const mockReplaceForWallet = jest.fn() +const mockFindRuleById = jest.fn() +const mockDeleteRuleById = jest.fn() + +jest.mock("@services/mongoose", () => ({ + AccountsRepository: jest.fn(() => ({ findByUsername: jest.fn() })), + WalletsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindWalletById(...args), + listByAccountId: jest.fn(), + })), +})) + +jest.mock("@services/mongoose/payment-split-rules", () => ({ + PaymentSplitRulesRepository: jest.fn(() => ({ + replaceForWallet: (...args: unknown[]) => mockReplaceForWallet(...args), + findById: (...args: unknown[]) => mockFindRuleById(...args), + deleteById: (...args: unknown[]) => mockDeleteRuleById(...args), + })), +})) + +jest.mock("@services/lock", () => ({ + LockService: jest.fn(() => ({ + lockWalletId: async (_id: string, fn: () => Promise) => fn(), + })), +})) + +jest.mock("@services/logger", () => ({ + baseLogger: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + child: jest.fn(() => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn() })), + }, +})) + +import { PaymentSplitsConfig } from "@config" +import { CouldNotFindWalletFromIdError } from "@domain/errors" +import { WalletCurrency } from "@domain/shared" +import WalletSplitDeleteMutation from "@graphql/public/root/mutation/wallet-split-delete" +import WalletSplitsSetMutation from "@graphql/public/root/mutation/wallet-splits-set" + +const OWNER_ID = "owner-account" +const SOURCE_ID = "11111111-1111-4111-8111-111111111111" +const DEST_ID = "22222222-2222-4222-8222-222222222222" +const FOREIGN_ID = "33333333-3333-4333-8333-333333333333" + +const walletsById: Record = { + [SOURCE_ID]: { id: SOURCE_ID, accountId: OWNER_ID, currency: WalletCurrency.Usdt }, + [DEST_ID]: { id: DEST_ID, accountId: "dest-account", currency: WalletCurrency.Usdt }, + [FOREIGN_ID]: { + id: FOREIGN_ID, + accountId: "someone-else", + currency: WalletCurrency.Usdt, + }, +} + +type Result = { + errors: Array<{ message: string; code?: string }> + splits?: unknown[] + success?: boolean +} + +const ctx = { domainAccount: { id: OWNER_ID } } as unknown as GraphQLPublicContextAuth + +const resolveSet = (input: Record) => + ( + WalletSplitsSetMutation as unknown as { + resolve: (s: null, a: unknown, c: unknown, i: never) => Promise + } + ).resolve(null, { input }, ctx, {} as never) + +const resolveDelete = (input: Record) => + ( + WalletSplitDeleteMutation as unknown as { + resolve: (s: null, a: unknown, c: unknown, i: never) => Promise + } + ).resolve(null, { input }, ctx, {} as never) + +const originalEnabled = PaymentSplitsConfig.enabled + +beforeEach(() => { + jest.clearAllMocks() + PaymentSplitsConfig.enabled = true + mockFindWalletById.mockImplementation( + async (id: string) => walletsById[id] ?? new CouldNotFindWalletFromIdError(), + ) + mockReplaceForWallet.mockImplementation( + async (walletId: string, rules: { destinationWalletId: string; percent: number }[]) => + rules.map((r, i) => ({ + id: `rule-${i}`, + walletId, + enabled: true, + createdAt: new Date(), + ...r, + })), + ) +}) + +afterAll(() => { + PaymentSplitsConfig.enabled = originalEnabled +}) + +describe("walletSplitsSet", () => { + it("returns the new split set", async () => { + const result = await resolveSet({ + walletId: SOURCE_ID, + splits: [{ destination: DEST_ID, percent: 30 }], + }) + + expect(result.errors).toEqual([]) + expect(result.splits).toEqual([ + expect.objectContaining({ destinationWalletId: DEST_ID, percent: 30 }), + ]) + }) + + it("rejects another account's walletId", async () => { + const result = await resolveSet({ + walletId: FOREIGN_ID, + splits: [{ destination: DEST_ID, percent: 30 }], + }) + + expect(result.splits).toBeUndefined() + expect(result.errors).toHaveLength(1) + expect(mockReplaceForWallet).not.toHaveBeenCalled() + }) + + it("maps an over-100% set to a user-readable validation error", async () => { + walletsById[FOREIGN_ID] = { + id: FOREIGN_ID, + accountId: "someone-else", + currency: WalletCurrency.Usdt, + } + const result = await resolveSet({ + walletId: SOURCE_ID, + splits: [ + { destination: DEST_ID, percent: 70 }, + { destination: FOREIGN_ID, percent: 31 }, + ], + }) + + expect(result.errors).toEqual([ + expect.objectContaining({ + message: "Split percentages must add up to more than 0 and at most 100", + }), + ]) + }) + + it("maps a self split", async () => { + const result = await resolveSet({ + walletId: SOURCE_ID, + splits: [{ destination: SOURCE_ID, percent: 10 }], + }) + + expect(result.errors[0].message).toBe("A wallet cannot split payments to itself") + }) + + it("maps an unknown destination", async () => { + const result = await resolveSet({ + walletId: SOURCE_ID, + splits: [{ destination: "44444444-4444-4444-8444-444444444444", percent: 10 }], + }) + + expect(result.errors[0].message).toMatch(/^No wallet found for/) + }) + + it("maps the disabled flag", async () => { + PaymentSplitsConfig.enabled = false + + const result = await resolveSet({ + walletId: SOURCE_ID, + splits: [{ destination: DEST_ID, percent: 10 }], + }) + + expect(result.errors[0].message).toBe("Payment splits are not available") + }) + + it("passes an input validation error through", async () => { + const result = await resolveSet({ + walletId: new Error("Invalid value for WalletId"), + splits: [], + }) + + expect(result.errors[0].message).toBe("Invalid value for WalletId") + }) +}) + +describe("walletSplitDelete", () => { + it("deletes the caller's split", async () => { + mockFindRuleById.mockResolvedValue({ id: "rule-1", walletId: SOURCE_ID }) + mockDeleteRuleById.mockResolvedValue(true) + + expect(await resolveDelete({ walletId: SOURCE_ID, splitId: "rule-1" })).toEqual({ + errors: [], + success: true, + }) + }) + + it("maps a split on another wallet to not-found", async () => { + mockFindRuleById.mockResolvedValue({ id: "rule-1", walletId: DEST_ID }) + + const result = await resolveDelete({ walletId: SOURCE_ID, splitId: "rule-1" }) + + expect(result.success).toBe(false) + expect(result.errors[0].message).toBe("Payment split not found") + expect(mockDeleteRuleById).not.toHaveBeenCalled() + }) + + it("rejects another account's walletId", async () => { + const result = await resolveDelete({ walletId: FOREIGN_ID, splitId: "rule-1" }) + + expect(result.success).toBe(false) + expect(mockFindRuleById).not.toHaveBeenCalled() + }) +}) diff --git a/test/flash/unit/graphql/shared/wallet-split.spec.ts b/test/flash/unit/graphql/shared/wallet-split.spec.ts new file mode 100644 index 000000000..de28519e1 --- /dev/null +++ b/test/flash/unit/graphql/shared/wallet-split.spec.ts @@ -0,0 +1,48 @@ +const mockListByWalletId = jest.fn() + +jest.mock("@services/mongoose/payment-split-rules", () => ({ + PaymentSplitRulesRepository: jest.fn(() => ({ + listByWalletId: (...args: unknown[]) => mockListByWalletId(...args), + })), +})) + +import { resolveWalletSplits } from "@graphql/shared/types/object/wallet-split" + +const wallet = { id: "wallet-1", accountId: "owner" } as unknown as Wallet +const rule = { + id: "rule-1", + walletId: "wallet-1", + destinationWalletId: "dest", + percent: 10, + enabled: true, + createdAt: new Date(), +} + +beforeEach(() => { + jest.clearAllMocks() + mockListByWalletId.mockResolvedValue([rule]) +}) + +describe("Wallet.splits resolver", () => { + it("returns the wallet's split rules to its owner", async () => { + expect( + await resolveWalletSplits(wallet, {}, { domainAccount: { id: "owner" } }), + ).toEqual([rule]) + expect(mockListByWalletId).toHaveBeenCalledWith("wallet-1") + }) + + it("returns an empty list to any other account", async () => { + expect( + await resolveWalletSplits(wallet, {}, { domainAccount: { id: "intruder" } }), + ).toEqual([]) + expect(mockListByWalletId).not.toHaveBeenCalled() + }) + + it("surfaces a repository failure as a GraphQL error", async () => { + mockListByWalletId.mockResolvedValue(new Error("mongo down")) + + await expect( + resolveWalletSplits(wallet, {}, { domainAccount: { id: "owner" } }), + ).rejects.toBeDefined() + }) +}) From d26e2dbb54c20af3bcc58570dff2372398cb84d5 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 23:14:32 -0700 Subject: [PATCH 09/13] fix(payment-splits): address review on cadence, lock retries, Flash-to-Flash, empty set - Cadence: run reconcile, executor and stale sweep every minute inside the ibex-webhook workload (startPaymentSplitsLoop, Redis lease for reconcile). The daily galoy cron keeps only a backstop reconcile. The reconcile lookback is derived from the caller's cadence (cadence plus margin: 25 min for the loop, 26 h for the cron) and pages through IBEX until it passes the lookback. The executor drains due rows within executorTimeBudgetMs instead of a fixed batch, so the 1/2/4/8 minute backoff is real. - Lock errors: retry only ResourceAttemptsLockServiceError raised before the send guard passed. Any other lock error, or one after the guard passed (the payment may have gone out), marks the row unknown and alerts ops. - Flash-to-Flash (decision 1): bolt11 pays mark the hash (origin ln-send) before paying, fail-closed; LNURL pays mark the IBEX-returned hash after paying, and the executor re-checks the marker before every payout (skipped_internal). A call-site test classifies every IBEX pay call. - walletSplitsSet with an empty list clears all splits. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G4SaLJF5SWq6sjBEaWJFow --- dev/apollo-federation/supergraph.graphql | 4 +- dev/config/base-config.yaml | 5 +- .../payment-splits/execute-pending-splits.ts | 153 +++++++++++++++--- src/app/payment-splits/index.ts | 1 + src/app/payment-splits/reconcile-splits.ts | 71 +++++++- src/app/payment-splits/split-loop.ts | 102 ++++++++++++ src/app/payments/mark-outgoing-invoice.ts | 81 ++++++++++ src/config/schema.ts | 32 +++- src/config/schema.types.d.ts | 5 +- src/config/yaml.ts | 5 +- src/domain/bitcoin/lightning/index.ts | 2 +- src/domain/bitcoin/lightning/ln-invoice.ts | 10 ++ src/domain/payment-splits/errors.ts | 4 +- src/domain/payment-splits/index.ts | 21 +++ src/domain/payment-splits/validate.ts | 7 +- .../root/mutation/ln-invoice-payment-send.ts | 9 ++ .../ln-noamount-usd-invoice-payment-send.ts | 9 ++ .../root/mutation/lnurl-payment-send.ts | 8 + .../public/root/mutation/wallet-splits-set.ts | 3 +- src/graphql/public/schema.graphql | 4 +- src/servers/cron.ts | 34 +--- src/servers/ibex-webhook-server.ts | 8 +- .../ibex/webhook-server/routes/on-receive.ts | 11 +- .../mongoose/internal-transfer-invoices.ts | 6 +- .../mongoose/payment-split-executions.ts | 9 +- src/services/mongoose/schema.ts | 16 +- .../add-invoice-call-sites.spec.ts | 42 ++++- .../execute-pending-splits.spec.ts | 142 +++++++++++++++- .../payment-splits/reconcile-splits.spec.ts | 90 +++++++++-- .../app/payment-splits/set-splits.spec.ts | 12 ++ .../app/payment-splits/split-loop.spec.ts | 121 ++++++++++++++ .../payments/mark-outgoing-invoice.spec.ts | 86 ++++++++++ .../domain/payment-splits/validate.spec.ts | 8 +- .../graphql/ln-invoice-payment-send.spec.ts | 51 ++++++ ...-noamount-usd-invoice-payment-send.spec.ts | 41 +++++ .../root/mutation/lnurl-payment-send.spec.ts | 39 +++++ 36 files changed, 1146 insertions(+), 106 deletions(-) create mode 100644 src/app/payment-splits/split-loop.ts create mode 100644 src/app/payments/mark-outgoing-invoice.ts create mode 100644 test/flash/unit/app/payment-splits/split-loop.spec.ts create mode 100644 test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts diff --git a/dev/apollo-federation/supergraph.graphql b/dev/apollo-federation/supergraph.graphql index 155c032e5..a141c1790 100644 --- a/dev/apollo-federation/supergraph.graphql +++ b/dev/apollo-federation/supergraph.graphql @@ -3584,7 +3584,9 @@ input WalletSplitInput input WalletSplitsSetInput @join__type(graph: PUBLIC) { - """The complete split set. Replaces every existing split.""" + """ + The complete split set. Replaces every existing split; an empty list clears them all. + """ splits: [WalletSplitInput!]! """The wallet whose incoming payments are split.""" diff --git a/dev/config/base-config.yaml b/dev/config/base-config.yaml index 5d065c6d3..ec2bfb213 100644 --- a/dev/config/base-config.yaml +++ b/dev/config/base-config.yaml @@ -59,9 +59,12 @@ fygaro: # ENG-653: Flash-native payment splits for incoming payments. paymentSplits: enabled: false # flags-off baseline; enable per environment via config overrides - executorBatchSize: 50 maxAttempts: 5 minShareCents: 1 + loopIntervalMs: 60000 # executor + stale-sweep cadence inside the ibex-webhook pod + executorTimeBudgetMs: 45000 # one executor pass drains due rows for at most this long + reconcileIntervalMs: 900000 # webhook-safety-net reconcile cadence inside the same loop + cronIntervalMs: 86400000 # galoy cronjob schedule (chart: 0 2 * * *); sizes the cron reconcile lookback bridge: enabled: false # flags-off baseline; enable per environment via config overrides diff --git a/src/app/payment-splits/execute-pending-splits.ts b/src/app/payment-splits/execute-pending-splits.ts index c12a0d175..733eeafa9 100644 --- a/src/app/payment-splits/execute-pending-splits.ts +++ b/src/app/payment-splits/execute-pending-splits.ts @@ -5,13 +5,15 @@ import { authorizeSend } from "@app/payments/authorize-send" import { intraledgerPaymentSendWalletIdForUsdWallet } from "@app/payments/send-intraledger" import { PaymentSendStatus } from "@domain/bitcoin/lightning" -import { LockServiceError } from "@domain/lock" +import { LockServiceError, ResourceAttemptsLockServiceError } from "@domain/lock" import { InternalTransferMarkerError, splitRetryBackoffMs } from "@domain/payment-splits" import { SendLimitsUnavailableError } from "@domain/payments/errors" import { RateLimitServiceError } from "@domain/rate-limit/errors" +import { notifyOpsEvent } from "@services/alerts/ops-events" import { IbexError } from "@services/ibex/errors" import { baseLogger } from "@services/logger" +import { isInternalTransferInvoice } from "@services/mongoose/internal-transfer-invoices" import { AccountsRepository, WalletsRepository } from "@services/mongoose" import { PaymentSplitExecutionsRepository, @@ -28,17 +30,44 @@ export const splitIdempotencyKey = (executionId: string): IdempotencyKey => /** * Errors worth another attempt: raised before IBEX paid anything and expected * to clear on their own (rate limits, a send guard that could not reach Redis - * or a price, a busy idempotency lock, the internal-transfer marker write, an - * IBEX 429). Everything else — limits, KYC level, insufficient balance, an - * inactive account, an unknown IBEX error — is terminal on the first attempt, - * so an error whose payment outcome is unknown is never re-sent. + * or a price, the internal-transfer marker write, an IBEX 429, and a busy + * idempotency lock that was never acquired). Everything else — limits, KYC + * level, insufficient balance, an inactive account, an unknown IBEX error — is + * terminal on the first attempt, so an error whose payment outcome is unknown + * is never re-sent. + * + * Lock errors are the subtle case. redlock also returns a LockServiceError when + * the locked routine throws or when releasing the lock fails, and both can + * happen AFTER the IBEX payment succeeded, replacing the Success result. So a + * lock error is retryable only when it is ResourceAttemptsLockServiceError AND + * the send never reached the payment (`paymentAttempted` false: the authorize + * hook, which runs immediately before the payment, never passed). */ -export const isRetryableSplitError = (error: Error): boolean => - error instanceof RateLimitServiceError || - error instanceof SendLimitsUnavailableError || - error instanceof LockServiceError || - error instanceof InternalTransferMarkerError || - (error instanceof IbexError && error.httpCode === 429) +export const isRetryableSplitError = ( + error: Error, + { paymentAttempted }: { paymentAttempted: boolean }, +): boolean => { + if (error instanceof LockServiceError) { + return error instanceof ResourceAttemptsLockServiceError && !paymentAttempted + } + return ( + error instanceof RateLimitServiceError || + error instanceof SendLimitsUnavailableError || + error instanceof InternalTransferMarkerError || + (error instanceof IbexError && error.httpCode === 429) + ) +} + +/** + * A lock error the send may have paid through (see isRetryableSplitError). + * Treated like a row the stale sweep finds stuck in `sending`: `unknown` plus + * an ops alert, never resent. + */ +const isAmbiguousLockError = ( + error: Error, + { paymentAttempted }: { paymentAttempted: boolean }, +): boolean => + error instanceof LockServiceError && !isRetryableSplitError(error, { paymentAttempted }) const formatCents = (cents: number): string => `US$${(cents / 100).toFixed(2)}` @@ -61,7 +90,13 @@ const notifyOwner = async ( logContext: { executionId: execution.id, walletId: execution.walletId }, }) -type ExecuteOutcome = "sent" | "ibex_pending" | "failed" | "retry" +type ExecuteOutcome = + | "sent" + | "ibex_pending" + | "failed" + | "retry" + | "unknown" + | "skipped_internal" const executeOne = async ( execution: PaymentSplitExecutionRecord, @@ -82,11 +117,45 @@ const executeOne = async ( return "failed" as const } + const retry = async (error: Error) => { + await executions.finishClaimed(execution.id, { + status: "pending", + attempts, + nextAttemptAt: new Date(now.getTime() + splitRetryBackoffMs(attempts)), + lastError: error.name, + now, + }) + return "retry" as const + } + + // Re-check the internal-transfer marker at pay time. An LNURL send from + // another Flash user is marked only after IBEX returns, so its receive can be + // recorded before the marker exists (ENG-653 decision 1). + const internal = await isInternalTransferInvoice(execution.triggerKey) + if (internal instanceof Error) { + // Nothing was sent: the same retry budget as any pre-payment error. + if (attempts < PaymentSplitsConfig.maxAttempts) return retry(internal) + return fail(internal) + } + if (internal) { + await executions.finishClaimed(execution.id, { + status: "skipped_internal", + attempts: execution.attempts, + now, + }) + return "skipped_internal" + } + const wallet = await WalletsRepository().findById(execution.walletId as WalletId) if (wallet instanceof Error) return fail(wallet) const account = await AccountsRepository().findById(wallet.accountId) if (account instanceof Error) return fail(account) + // Set once the ENG-573 guard passes. withPaymentIdempotency runs `authorize` + // inside the idempotency lock immediately before the payment, so this is the + // last point at which no money can have moved yet. + let paymentAttempted = false + const result = await intraledgerPaymentSendWalletIdForUsdWallet({ senderWalletId: wallet.id, recipientWalletId: execution.destinationWalletId as WalletId, @@ -99,25 +168,49 @@ const executeOne = async ( // A split is a send out of the user's own wallet: it goes through the same // ENG-573 guard (attempt budget, amount sanity, daily limit) as a manual // intraledger send. - authorize: () => - authorizeSend({ + authorize: async () => { + const authorized = await authorizeSend({ senderAccount: account, senderWalletId: wallet.id, amount: { currency: "USD", cents: execution.shareCents }, kind: "intraledger", - }), + }) + if (!(authorized instanceof Error)) paymentAttempted = true + return authorized + }, }) if (result instanceof Error) { - if (isRetryableSplitError(result) && attempts < PaymentSplitsConfig.maxAttempts) { + if (isAmbiguousLockError(result, { paymentAttempted })) { await executions.finishClaimed(execution.id, { - status: "pending", + status: "unknown", attempts, - nextAttemptAt: new Date(now.getTime() + splitRetryBackoffMs(attempts)), lastError: result.name, now, }) - return "retry" + logger.error( + { executionId: execution.id, walletId: execution.walletId, error: result }, + "payment split lock error after the send may have paid; marked unknown", + ) + notifyOpsEvent({ + flow: "transfer", + phase: "split-unknown", + status: "failed", + amount: { value: (execution.shareCents / 100).toFixed(2), currency: "USD" }, + meta: { + executionId: execution.id, + senderWalletId: execution.walletId, + recipientWalletId: execution.destinationWalletId, + reason: result.name, + }, + }) + return "unknown" + } + if ( + isRetryableSplitError(result, { paymentAttempted }) && + attempts < PaymentSplitsConfig.maxAttempts + ) { + return retry(result) } logger.warn( { executionId: execution.id, walletId: execution.walletId, error: result }, @@ -152,27 +245,41 @@ export type ExecutePendingSplitsSummary = { ibexPending: number failed: number retried: number + unknown: number + skippedInternal: number } /** - * Cron executor: claims due `pending` executions one at a time (atomically, so + * Executor: claims due `pending` executions one at a time (atomically, so * concurrent runs never claim the same row) and pays each share as an * intraledger send from the receiving wallet to the split destination. + * + * Runs every PaymentSplitsConfig.loopIntervalMs from the split loop and keeps + * claiming until nothing is due or executorTimeBudgetMs is spent, so the + * throughput is not capped by a fixed batch per run. `now` is fixed per pass: + * a row retried in this pass is due at now + backoff and is picked up by a + * later pass, which is what makes the 1/2/4/8 minute backoff real. */ export const executePendingSplits = async ({ now = new Date(), -}: { now?: Date } = {}): Promise => { + clock = () => Date.now(), +}: { now?: Date; clock?: () => number } = {}): Promise< + ExecutePendingSplitsSummary | Error +> => { const summary: ExecutePendingSplitsSummary = { claimed: 0, sent: 0, ibexPending: 0, failed: 0, retried: 0, + unknown: 0, + skippedInternal: 0, } if (!PaymentSplitsConfig.enabled) return summary const executions = PaymentSplitExecutionsRepository() - for (let i = 0; i < PaymentSplitsConfig.executorBatchSize; i++) { + const deadline = clock() + PaymentSplitsConfig.executorTimeBudgetMs + while (clock() < deadline) { const execution = await executions.claimNextDue(now) if (execution instanceof Error) return execution if (execution === null) break @@ -182,6 +289,8 @@ export const executePendingSplits = async ({ if (outcome === "sent") summary.sent += 1 else if (outcome === "ibex_pending") summary.ibexPending += 1 else if (outcome === "failed") summary.failed += 1 + else if (outcome === "unknown") summary.unknown += 1 + else if (outcome === "skipped_internal") summary.skippedInternal += 1 else summary.retried += 1 } diff --git a/src/app/payment-splits/index.ts b/src/app/payment-splits/index.ts index 3c5dcb813..b61f16519 100644 --- a/src/app/payment-splits/index.ts +++ b/src/app/payment-splits/index.ts @@ -5,3 +5,4 @@ export * from "./record-split-executions" export * from "./execute-pending-splits" export * from "./sweep-stale-splits" export * from "./reconcile-splits" +export * from "./split-loop" diff --git a/src/app/payment-splits/reconcile-splits.ts b/src/app/payment-splits/reconcile-splits.ts index 386c4257d..b9af082c9 100644 --- a/src/app/payment-splits/reconcile-splits.ts +++ b/src/app/payment-splits/reconcile-splits.ts @@ -1,5 +1,7 @@ import { PaymentSplitsConfig } from "@config" +import { splitReconcileWindowMs } from "@domain/payment-splits" + import Ibex from "@services/ibex/client" import { baseLogger } from "@services/logger" import { WalletsRepository } from "@services/mongoose" @@ -15,34 +17,90 @@ const IBEX_LIGHTNING_RECEIVE = 1 // v1. Bridge virtual-account deposits arrive as the same type on the same // address, so a split could not tell them apart (ENG-653 decision 2). -// Transactions read per wallet per run. The window is short, so a page this -// size covers any plausible receive rate. +// Transactions read per IBEX page. Pages are read newest-settled first until +// one reaches past the lookback, so a busy wallet is covered whatever its +// receive rate; MAX_PAGES only stops a runaway loop (50k transactions). const TRANSACTIONS_PAGE_SIZE = 100 +const MAX_PAGES = 500 const PaymentHashRegex = /^[0-9a-f]{64}$/i export type ReconcileSplitsSummary = { + windowMs: number wallets: number receivesSeen: number recorded: number errors: number } +type IbexAccountTransaction = { + id?: string + createdAt?: string + amount?: number + transactionTypeId?: number +} + +/** + * Every transaction on `walletId` from the newest back to `since`, one IBEX + * page at a time (sort settledAt: newest settled first; pages are 0-based). + * Stops at the first page that reaches past `since` or comes back short. + */ +const readTransactionsSince = async ( + walletId: string, + since: number, +): Promise => { + const seen = new Set() + const transactions: IbexAccountTransaction[] = [] + + for (let page = 0; page < MAX_PAGES; page++) { + const batch = await Ibex.getAccountTransactions({ + account_id: walletId, + limit: TRANSACTIONS_PAGE_SIZE, + page, + sort: "settledAt", + }) + if (batch instanceof Error) return batch + + let oldest = Infinity + for (const tx of batch as IbexAccountTransaction[]) { + const createdAt = tx.createdAt ? new Date(tx.createdAt).getTime() : NaN + if (!Number.isNaN(createdAt)) oldest = Math.min(oldest, createdAt) + if (!tx.id || seen.has(tx.id)) continue + seen.add(tx.id) + transactions.push(tx) + } + + if (batch.length < TRANSACTIONS_PAGE_SIZE || oldest < since) return transactions + } + + logger.warn( + { walletId, pages: MAX_PAGES }, + "split reconcile: page cap reached before the lookback; older receives not read", + ) + return transactions +} + /** * Safety net for missed or failed webhooks: re-reads recent Lightning receives * on every wallet with enabled split rules and records any execution the * webhook did not. The trigger key is the payment hash — the same key the * webhook uses — so a receive the webhook already recorded is a duplicate-key * no-op. + * + * `cadenceMs` is how often the caller runs this. The lookback is derived from + * it (splitReconcileWindowMs: cadence plus a margin) so consecutive runs always + * overlap; a caller cannot hand in a window shorter than its own schedule. */ export const reconcileSplits = async ({ - windowMs, + cadenceMs, now = new Date(), }: { - windowMs: number + cadenceMs: number now?: Date }): Promise => { + const windowMs = splitReconcileWindowMs(cadenceMs) const summary: ReconcileSplitsSummary = { + windowMs, wallets: 0, receivesSeen: 0, recorded: 0, @@ -65,10 +123,7 @@ export const reconcileSplits = async ({ continue } - const transactions = await Ibex.getAccountTransactions({ - account_id: walletId, - limit: TRANSACTIONS_PAGE_SIZE, - }) + const transactions = await readTransactionsSince(walletId, since) if (transactions instanceof Error) { summary.errors += 1 logger.warn({ walletId, error: transactions }, "split reconcile: IBEX read failed") diff --git a/src/app/payment-splits/split-loop.ts b/src/app/payment-splits/split-loop.ts new file mode 100644 index 000000000..eb319d497 --- /dev/null +++ b/src/app/payment-splits/split-loop.ts @@ -0,0 +1,102 @@ +import { PaymentSplitsConfig } from "@config" + +import { baseLogger } from "@services/logger" +import { redis } from "@services/redis" + +import { executePendingSplits } from "./execute-pending-splits" +import { reconcileSplits } from "./reconcile-splits" +import { sweepStaleSplits } from "./sweep-stale-splits" + +const logger = baseLogger.child({ module: "payment-splits-loop" }) + +const RECONCILE_LEASE_KEY = "payment-splits:reconcile:lease" +const FIRST_RUN_DELAY_MS = 30 * 1000 + +/** + * One reconcile per reconcileIntervalMs across all replicas: a Redis NX lease + * that expires just before the next one is due. Fail-open, like the Fygaro + * treasury loop: a Redis blip must not silence the webhook safety net, and a + * duplicate reconcile only re-inserts rows the unique index rejects. + */ +const claimReconcileLease = async (intervalMs: number): Promise => { + const ttlSeconds = Math.max(1, Math.floor((intervalMs * 0.9) / 1000)) + try { + return (await redis.set(RECONCILE_LEASE_KEY, "1", "EX", ttlSeconds, "NX")) === "OK" + } catch (err) { + logger.warn( + { err }, + "payment splits loop: reconcile lease unavailable, running anyway", + ) + return true + } +} + +/** + * One tick of the split loop: reconcile (when its lease is due), execute every + * due row within the executor's time budget, then sweep rows stuck in + * `sending`. Executor claims and sweep moves are atomic per row, so ticks on + * several replicas never pay the same row twice. + */ +export const runPaymentSplitsTick = async ({ + now = new Date(), +}: { now?: Date } = {}): Promise => { + if (!PaymentSplitsConfig.enabled) return + + if (await claimReconcileLease(PaymentSplitsConfig.reconcileIntervalMs)) { + const reconciled = await reconcileSplits({ + cadenceMs: PaymentSplitsConfig.reconcileIntervalMs, + now, + }) + if (reconciled instanceof Error) { + logger.error({ err: reconciled }, "payment splits loop: reconcile failed") + } else if (reconciled.recorded > 0 || reconciled.errors > 0) { + logger.info({ result: reconciled }, "payment splits loop: reconcile finished") + } + } + + const executed = await executePendingSplits({ now }) + if (executed instanceof Error) { + logger.error({ err: executed }, "payment splits loop: executor failed") + } else if (executed.claimed > 0) { + logger.info({ result: executed }, "payment splits loop: executor finished") + } + + const swept = await sweepStaleSplits({ now }) + if (swept instanceof Error) { + logger.error({ err: swept }, "payment splits loop: stale sweep failed") + } +} + +/** + * Runs the split pipeline every loopIntervalMs inside the long-running + * ibex-webhook workload. The galoy cron is a one-shot k8s Job whose schedule + * the chart owns (daily at 02:00), far too slow to pay splits or to make the + * 1/2/4/8 minute retry backoff real, and this repo cannot promise a cadence + * through it. The webhook pod records the splits, so it is up whenever there + * is anything to pay. The daily cron keeps a long-lookback reconcile as the + * backstop for this loop being down. + */ +export const startPaymentSplitsLoop = (): NodeJS.Timeout | undefined => { + if (!PaymentSplitsConfig.enabled) return undefined + + const intervalMs = PaymentSplitsConfig.loopIntervalMs + let running = false + + // A tick that outlives the interval (a long executor drain) is not stacked + // on: the next fire is skipped until it finishes. + const fire = () => { + if (running) return + running = true + runPaymentSplitsTick() + .catch((err) => logger.error({ err }, "payment splits loop tick errored")) + .finally(() => { + running = false + }) + } + + setTimeout(fire, FIRST_RUN_DELAY_MS).unref() + const timer = setInterval(fire, intervalMs) + timer.unref() + logger.info({ intervalMs }, "payment splits loop started") + return timer +} diff --git a/src/app/payments/mark-outgoing-invoice.ts b/src/app/payments/mark-outgoing-invoice.ts new file mode 100644 index 000000000..bfd8b428e --- /dev/null +++ b/src/app/payments/mark-outgoing-invoice.ts @@ -0,0 +1,81 @@ +import { paymentHashFromBolt11 } from "@domain/bitcoin/lightning" +import { + InternalTransferMarkerError, + InternalTransferOrigin, +} from "@domain/payment-splits" +import { ErrorLevel } from "@domain/shared" + +import { recordInternalTransferInvoice } from "@services/mongoose/internal-transfer-invoices" +import { baseLogger } from "@services/logger" +import { recordExceptionInCurrentSpan } from "@services/tracing" + +const PaymentHashRegex = /^[0-9a-f]{64}$/i + +/** + * ENG-653 decision 1: a Flash-to-Flash payment never triggers the recipient's + * payment splits. A Flash user who scans another Flash user's bolt11 pays an + * invoice IBEX settles internally, and the recipient's on-receive webhook + * fires just like for an external payer. Flash cannot cheaply tell whether a + * bolt11 belongs to one of its own wallets, so it marks every bolt11 its users + * pay: an external invoice never comes back through Flash's receive webhook, + * so marking it is harmless. + * + * Fails closed, like the intraledger marker: when the hash is readable but the + * marker cannot be written, nothing is paid. A bolt11 the decoder cannot parse + * is paid unmarked: every invoice IBEX mints for a Flash wallet decodes (the + * intraledger marker depends on that), so an undecodable one is external. + */ +export const markOutgoingBolt11 = async ({ + paymentRequest, + senderWalletId, +}: { + paymentRequest: string + senderWalletId: WalletId +}): Promise => { + const paymentHash = paymentHashFromBolt11(paymentRequest) + if (!paymentHash || !PaymentHashRegex.test(paymentHash)) return true + + const recorded = await recordInternalTransferInvoice({ + paymentHash, + senderWalletId, + origin: InternalTransferOrigin.LnSend, + }) + if (recorded instanceof Error) return new InternalTransferMarkerError(recorded) + return true +} + +/** + * LNURL counterpart of markOutgoingBolt11. IBEX fetches the LNURL invoice and + * pays it in one call, so the hash is only known after the payment. The + * marker is written then; the split executor re-checks it before paying, so a + * receive recorded in the gap is skipped (`skipped_internal`), not paid. + * Best effort: the money already moved, so a failure is logged, not returned. + */ +export const markPaidLnurlHash = async ({ + paymentHash, + senderWalletId, +}: { + paymentHash: unknown + senderWalletId: WalletId +}): Promise => { + if (typeof paymentHash !== "string" || !PaymentHashRegex.test(paymentHash)) return + + const recorded = await recordInternalTransferInvoice({ + paymentHash, + senderWalletId, + origin: InternalTransferOrigin.LnurlSend, + }) + if (recorded instanceof Error) { + baseLogger.error( + { err: recorded, paymentHash, senderWalletId }, + "Payment splits: could not mark a paid LNURL hash", + ) + recordExceptionInCurrentSpan({ + error: recorded, + level: ErrorLevel.Critical, + fallbackMsg: + "Payment splits: could not mark a paid LNURL hash; a Flash payee's splits may fire", + attributes: { "paymentSplits.paymentHash": paymentHash }, + }) + } +} diff --git a/src/config/schema.ts b/src/config/schema.ts index 0a08a4c59..c19ac7bb4 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -1128,21 +1128,45 @@ export const configSchema = { type: "object", properties: { enabled: { type: "boolean", default: false }, - // Split executions the cron executor claims and pays per run. - executorBatchSize: { type: "integer", minimum: 1, default: 50 }, // Sends attempted per execution before a retryable error becomes a - // terminal `failed`. Backoff between attempts is 1, 2, 4, 8... minutes. + // terminal `failed`. Backoff between attempts is 1, 2, 4, 8... minutes, + // so loopIntervalMs must stay at or below one minute for those steps + // to be real. maxAttempts: { type: "integer", minimum: 1, default: 5 }, // A share below this many US cents is recorded as `skipped_min` and // never sent. minShareCents: { type: "integer", minimum: 1, default: 1 }, + // Cadence of the split loop (executor + stale sweep) that runs inside + // the long-running ibex-webhook workload. The galoy cron is a daily + // k8s Job, too slow to execute or retry splits. + loopIntervalMs: { + type: "integer", + minimum: 10000, + maximum: 60000, + default: 60000, + }, + // Wall-clock budget for one executor pass. The pass keeps claiming due + // rows until none are left or the budget is spent, so throughput is + // not capped by a fixed batch size. + executorTimeBudgetMs: { type: "integer", minimum: 1000, default: 45000 }, + // How often the loop re-reads recent IBEX receives to record splits + // a missed or failed webhook did not. The lookback is derived from + // this (see splitReconcileWindowMs). + reconcileIntervalMs: { type: "integer", minimum: 60000, default: 900000 }, + // Schedule of the galoy cronjob (owned by the chart; daily at 02:00). + // The cron reconcile is the backstop for loop downtime and looks back + // this long plus a margin. + cronIntervalMs: { type: "integer", minimum: 60000, default: 86400000 }, }, additionalProperties: false, default: { enabled: false, - executorBatchSize: 50, maxAttempts: 5, minShareCents: 1, + loopIntervalMs: 60000, + executorTimeBudgetMs: 45000, + reconcileIntervalMs: 900000, + cronIntervalMs: 86400000, }, }, frappe: { diff --git a/src/config/schema.types.d.ts b/src/config/schema.types.d.ts index 5f2643ea8..f7da0ffc9 100644 --- a/src/config/schema.types.d.ts +++ b/src/config/schema.types.d.ts @@ -130,9 +130,12 @@ type FygaroConfig = { type PaymentSplitsConfig = { enabled: boolean - executorBatchSize: number maxAttempts: number minShareCents: number + loopIntervalMs: number + executorTimeBudgetMs: number + reconcileIntervalMs: number + cronIntervalMs: number } type CashoutEmail = { diff --git a/src/config/yaml.ts b/src/config/yaml.ts index 43d8a8b88..f15818f3d 100644 --- a/src/config/yaml.ts +++ b/src/config/yaml.ts @@ -647,9 +647,12 @@ export const FygaroConfig = yamlConfig.fygaro as FygaroConfig export const PaymentSplitsConfig = (yamlConfig.paymentSplits ?? { enabled: false, - executorBatchSize: 50, maxAttempts: 5, minShareCents: 1, + loopIntervalMs: 60000, + executorTimeBudgetMs: 45000, + reconcileIntervalMs: 900000, + cronIntervalMs: 86400000, }) as PaymentSplitsConfig export const FrappeConfig = yamlConfig.frappe as FrappeConfig diff --git a/src/domain/bitcoin/lightning/index.ts b/src/domain/bitcoin/lightning/index.ts index 2ca1fc778..c57a165bd 100644 --- a/src/domain/bitcoin/lightning/index.ts +++ b/src/domain/bitcoin/lightning/index.ts @@ -2,7 +2,7 @@ import { createHash, randomBytes } from "crypto" import { InvalidPubKeyError } from "@domain/errors" -export { decodeInvoice } from "./ln-invoice" +export { decodeInvoice, paymentHashFromBolt11 } from "./ln-invoice" export { invoiceExpirationForCurrency, defaultTimeToExpiryInSeconds, diff --git a/src/domain/bitcoin/lightning/ln-invoice.ts b/src/domain/bitcoin/lightning/ln-invoice.ts index b1b79826d..5cb1522d6 100644 --- a/src/domain/bitcoin/lightning/ln-invoice.ts +++ b/src/domain/bitcoin/lightning/ln-invoice.ts @@ -66,6 +66,16 @@ export const decodeInvoice = ( } } +/** + * Just the payment hash of a bolt11, with none of decodeInvoice's policy + * (payment secret, amount). Undefined when the library cannot parse it. + */ +export const paymentHashFromBolt11 = (bolt11: string): PaymentHash | undefined => { + const decoded = safeDecode(bolt11.toLowerCase() as EncodedPaymentRequest) + if (decoded instanceof Error) return undefined + return typeof decoded.id === "string" ? (decoded.id as PaymentHash) : undefined +} + const safeDecode = (bolt11EncodedInvoice: EncodedPaymentRequest) => { try { return parsePaymentRequest({ request: bolt11EncodedInvoice }) diff --git a/src/domain/payment-splits/errors.ts b/src/domain/payment-splits/errors.ts index 88994ca59..ba1362a80 100644 --- a/src/domain/payment-splits/errors.ts +++ b/src/domain/payment-splits/errors.ts @@ -3,8 +3,8 @@ import { DomainError, ErrorLevel, ValidationError } from "@domain/shared" // ENG-653 payment splits. Every class exported here is an ApplicationError key // and therefore needs a case in src/graphql/error-map.ts. -// The percentages of a wallet's split set must sum to more than 0 and at most -// 100 (an empty set sums to 0). +// The percentages of a non-empty split set must sum to more than 0 and at most +// 100. An empty set is allowed: it clears the wallet's splits. export class SplitPercentExceededError extends ValidationError {} // A single percent is not an integer in [1, 100]. export class InvalidSplitPercentError extends ValidationError {} diff --git a/src/domain/payment-splits/index.ts b/src/domain/payment-splits/index.ts index 34be9051d..eafd69423 100644 --- a/src/domain/payment-splits/index.ts +++ b/src/domain/payment-splits/index.ts @@ -9,6 +9,9 @@ export const PaymentSplitExecutionStatus = { IbexPending: "ibex_pending", Failed: "failed", SkippedMin: "skipped_min", + // The trigger turned out to be a Flash-to-Flash payment (its hash was marked + // after the receive was recorded, e.g. an LNURL send): never paid. + SkippedInternal: "skipped_internal", Unknown: "unknown", } as const @@ -18,6 +21,12 @@ export type PaymentSplitExecutionStatus = export const InternalTransferOrigin = { Intraledger: "intraledger", Cutover: "cutover", + // A Flash user paid a bolt11 (QR scan). Marked before paying, whether or + // not the payee turns out to be a Flash wallet. + LnSend: "ln-send", + // A Flash user paid a Lightning address / LNURL. IBEX fetches the invoice + // itself, so the hash is only known (and marked) after the payment returns. + LnurlSend: "lnurl-send", } as const export type InternalTransferOrigin = @@ -26,3 +35,15 @@ export type InternalTransferOrigin = // Backoff before attempt n+1 after the nth failed attempt: 1, 2, 4, 8, 16 min. export const splitRetryBackoffMs = (failedAttempts: number): number => 60_000 * 2 ** Math.max(0, failedAttempts - 1) + +// Extra lookback on top of a reconcile's run cadence, so a receive that landed +// just before the previous run (or while one run was late) is still read. +const RECONCILE_MIN_MARGIN_MS = 10 * 60_000 + +/** + * Lookback for a split reconcile that runs every `cadenceMs`. Always longer + * than the cadence, so consecutive runs overlap and no receive falls in a gap: + * a 15-minute loop reads 25 minutes, the daily cron reads 26 hours. + */ +export const splitReconcileWindowMs = (cadenceMs: number): number => + cadenceMs + Math.max(RECONCILE_MIN_MARGIN_MS, Math.ceil(cadenceMs / 12)) diff --git a/src/domain/payment-splits/validate.ts b/src/domain/payment-splits/validate.ts index d9280f156..767eb5a4d 100644 --- a/src/domain/payment-splits/validate.ts +++ b/src/domain/payment-splits/validate.ts @@ -20,7 +20,8 @@ export type SplitSetEntry = { /** * Validates a complete split set for one source wallet. The set replaces any - * existing rules, so the percent sum is checked over this set alone. + * existing rules, so the percent sum is checked over this set alone. An empty + * set is valid and clears the wallet's splits. */ export const validateSplitSet = ({ sourceWalletId, @@ -51,8 +52,10 @@ export const validateSplitSet = ({ } } + // An empty set is "clear all splits" (walletSplitsSet replaces every rule). + // Only a non-empty set must allocate more than 0 and at most 100 percent. const sum = entries.reduce((total, { percent }) => total + percent, 0) - if (sum <= 0 || sum > 100) { + if (entries.length > 0 && (sum <= 0 || sum > 100)) { return new SplitPercentExceededError( "Split percentages must add up to more than 0 and at most 100", ) diff --git a/src/graphql/public/root/mutation/ln-invoice-payment-send.ts b/src/graphql/public/root/mutation/ln-invoice-payment-send.ts index 3ad072331..b9ed38fe8 100644 --- a/src/graphql/public/root/mutation/ln-invoice-payment-send.ts +++ b/src/graphql/public/root/mutation/ln-invoice-payment-send.ts @@ -14,6 +14,7 @@ import Ibex from "@services/ibex/client" import { IbexError, InsufficientIbexBalance } from "@services/ibex/errors" import { paymentSendStatusOrPending } from "@services/ibex/payment-status" import { withPaymentIdempotency } from "@app/payments/idempotency" +import { markOutgoingBolt11 } from "@app/payments/mark-outgoing-invoice" import { authorizeSend, gateSend, @@ -149,6 +150,14 @@ const LnInvoicePaymentSendMutation = GT.Field< }) }, execute: async (): Promise => { + // ENG-653: the payee may be another Flash wallet, whose splits must + // not fire on a Flash-to-Flash payment. Fails closed. + const marked = await markOutgoingBolt11({ + paymentRequest, + senderWalletId: walletId, + }) + if (marked instanceof Error) return marked + const PayLightningInvoice = await Ibex.payInvoice({ invoice: paymentRequest as Bolt11, accountId: walletId, diff --git a/src/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.ts b/src/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.ts index f8a74e1fb..19b18b7a9 100644 --- a/src/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.ts +++ b/src/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.ts @@ -17,6 +17,7 @@ import Ibex from "@services/ibex/client" import { IbexError } from "@services/ibex/errors" import { withPaymentIdempotency } from "@app/payments/idempotency" +import { markOutgoingBolt11 } from "@app/payments/mark-outgoing-invoice" import { authorizeSend } from "@app/payments/authorize-send" import { paymentSendStatusOrPending } from "@services/ibex/payment-status" @@ -142,6 +143,14 @@ const LnNoAmountUsdInvoicePaymentSendMutation = GT.Field< }) if (usCents instanceof Error) return usCents + // ENG-653: the payee may be another Flash wallet, whose splits must + // not fire on a Flash-to-Flash payment. Fails closed. + const marked = await markOutgoingBolt11({ + paymentRequest, + senderWalletId: routedWalletId, + }) + if (marked instanceof Error) return marked + const PayLightningInvoice = await Ibex.payInvoice({ invoice: paymentRequest as Bolt11, accountId: routedWalletId, diff --git a/src/graphql/public/root/mutation/lnurl-payment-send.ts b/src/graphql/public/root/mutation/lnurl-payment-send.ts index 9c527484d..3aa3a259e 100644 --- a/src/graphql/public/root/mutation/lnurl-payment-send.ts +++ b/src/graphql/public/root/mutation/lnurl-payment-send.ts @@ -1,5 +1,6 @@ import { withPaymentIdempotency } from "@app/payments/idempotency" import { authorizeSend } from "@app/payments/authorize-send" +import { markPaidLnurlHash } from "@app/payments/mark-outgoing-invoice" import dedent from "dedent" import { resolveCashWalletMutationWalletIdForAccount } from "@app/cash-wallet-cutover" @@ -279,6 +280,13 @@ const LnurlPaymentSendMutation = GT.Field< params: paramsFromMetadata(metadata), }) if (payment instanceof IbexError) return payment + // ENG-653: the payee may be a Flash wallet (a Flash Lightning + // address). IBEX fetched the invoice itself, so its hash is only known + // now; the split executor re-checks this marker before paying. + await markPaidLnurlHash({ + paymentHash: payment.hash ?? payment.transaction?.payment?.hash, + senderWalletId: routedWalletId, + }) return lnurlPaymentSendStatusOrPending(payment) }, }) diff --git a/src/graphql/public/root/mutation/wallet-splits-set.ts b/src/graphql/public/root/mutation/wallet-splits-set.ts index 1834dc1b2..16f32d605 100644 --- a/src/graphql/public/root/mutation/wallet-splits-set.ts +++ b/src/graphql/public/root/mutation/wallet-splits-set.ts @@ -30,7 +30,8 @@ const WalletSplitsSetInput = GT.Input({ }, splits: { type: GT.NonNullList(WalletSplitInput), - description: "The complete split set. Replaces every existing split.", + description: + "The complete split set. Replaces every existing split; an empty list clears them all.", }, }), }) diff --git a/src/graphql/public/schema.graphql b/src/graphql/public/schema.graphql index d4857fc87..3d8155e8a 100644 --- a/src/graphql/public/schema.graphql +++ b/src/graphql/public/schema.graphql @@ -2942,7 +2942,9 @@ input WalletSplitInput { } input WalletSplitsSetInput { - """The complete split set. Replaces every existing split.""" + """ + The complete split set. Replaces every existing split; an empty list clears them all. + """ splits: [WalletSplitInput!]! """The wallet whose incoming payments are split.""" diff --git a/src/servers/cron.ts b/src/servers/cron.ts index b1540912f..fd63d9356 100644 --- a/src/servers/cron.ts +++ b/src/servers/cron.ts @@ -30,11 +30,7 @@ import { reconcileBridgeAndIbexWithdrawals, } from "@services/bridge/reconciliation" import { checkFygaroTreasuryFloat } from "@services/fygaro/float-monitor" -import { - executePendingSplits, - reconcileSplits, - sweepStaleSplits, -} from "@app/payment-splits" +import { reconcileSplits } from "@app/payment-splits" import { elapsedSinceTimestamp, sleep } from "@utils" import { rebalancingInternalChannels } from "@services/lnd/rebalancing" @@ -99,32 +95,20 @@ const reconcileBridgeWithdrawalsJob = async () => { if (result instanceof Error) throw result } -// ENG-653 payment splits. Reconcile first so a receive whose webhook was -// missed is recorded before the executor runs; the sweep then flags any row a -// crashed run left in `sending`. All three self-guard on the config flag. +// ENG-653 payment splits. Execution, retries and the stale sweep run every +// minute in the ibex-webhook workload (startPaymentSplitsLoop); this daily Job +// is far too slow for them. It keeps only a long-lookback reconcile as the +// backstop for that loop being down: the lookback is derived from +// paymentSplits.cronIntervalMs (the chart's schedule) plus a margin, so it +// always reaches back past the previous run. Self-guards on the config flag. const reconcileSplitPaymentsJob = async () => { if (!PaymentSplitsConfig.enabled) return - const result = await reconcileSplits({ windowMs: RECONCILE_WINDOW_MS }) + const result = await reconcileSplits({ cadenceMs: PaymentSplitsConfig.cronIntervalMs }) if (result instanceof Error) throw result logger.info({ result }, "payment split reconcile finished") } -const executeSplitPaymentsJob = async () => { - if (!PaymentSplitsConfig.enabled) return - - const result = await executePendingSplits() - if (result instanceof Error) throw result - logger.info({ result }, "payment split executor finished") -} - -const sweepStaleSplitsJob = async () => { - if (!PaymentSplitsConfig.enabled) return - - const result = await sweepStaleSplits() - if (result instanceof Error) throw result -} - // Bankowner treasury float check for Fygaro auto-credit. Runs every cron // invocation on whatever schedule the chart gives this Job; the fygaro-webhook // workload ALSO runs it every fygaro.float.checkIntervalMs (treasury-loop.ts) @@ -183,8 +167,6 @@ const main = async () => { reconcileBridgeDepositsJob, reconcileBridgeWithdrawalsJob, reconcileSplitPaymentsJob, - executeSplitPaymentsJob, - sweepStaleSplitsJob, checkFygaroFloatJob, evidenceRetentionJob, deleteExpiredPaymentFlows, diff --git a/src/servers/ibex-webhook-server.ts b/src/servers/ibex-webhook-server.ts index d51aaadcb..db6834719 100644 --- a/src/servers/ibex-webhook-server.ts +++ b/src/servers/ibex-webhook-server.ts @@ -1,3 +1,4 @@ +import { startPaymentSplitsLoop } from "@app/payment-splits/split-loop" import WebookServer from "@services/ibex/webhook-server" import { setupMongoConnection } from "@services/mongodb" import { warnIfDevContext } from "@utils/dev-context" @@ -10,6 +11,11 @@ if (require.main === module) { warnIfDevContext() setupMongoConnection() - .then(async () => WebookServer.start()) + .then(async () => { + WebookServer.start() + // ENG-653: split reconcile, executor and stale sweep run here, every + // paymentSplits.loopIntervalMs; the daily galoy cron is too slow. + startPaymentSplitsLoop() + }) .catch(exitOnBootFailure) } diff --git a/src/services/ibex/webhook-server/routes/on-receive.ts b/src/services/ibex/webhook-server/routes/on-receive.ts index c6210331d..36795c13b 100644 --- a/src/services/ibex/webhook-server/routes/on-receive.ts +++ b/src/services/ibex/webhook-server/routes/on-receive.ts @@ -73,8 +73,11 @@ const fetchPaymentContext = async ( // ENG-653: record (never send) the payment splits a Lightning/LNURL receive // triggers. Failure here must not fail the webhook: IBEX would retry it and -// re-run the receive notification, and the reconcile cron records any receive -// this step missed. Duplicate deliveries are duplicate-key no-ops. +// re-run the receive notification. The split reconcile records any receive +// this step missed: every paymentSplits.reconcileIntervalMs from this pod's +// split loop (lookback = interval plus margin) and once a day from the galoy +// cron (lookback = a day plus margin). Duplicate deliveries are duplicate-key +// no-ops. const recordPaymentSplits = async ( req: PaymentRequest, _resp: Response, @@ -98,14 +101,14 @@ const recordPaymentSplits = async ( if (result instanceof Error) { logger.error( { err: result, walletId: receiverWallet.id, paymentHash }, - "Failed to record payment splits; the reconcile cron will retry", + "Failed to record payment splits; the split reconcile will pick it up", ) } } } catch (err) { logger.error( { err }, - "Failed to record payment splits; the reconcile cron will retry", + "Failed to record payment splits; the split reconcile will pick it up", ) } diff --git a/src/services/mongoose/internal-transfer-invoices.ts b/src/services/mongoose/internal-transfer-invoices.ts index 97735a0bd..fa22dddba 100644 --- a/src/services/mongoose/internal-transfer-invoices.ts +++ b/src/services/mongoose/internal-transfer-invoices.ts @@ -1,3 +1,5 @@ +import type { InternalTransferOrigin } from "@domain/payment-splits" + import { parseRepositoryError } from "./utils" import { InternalTransferInvoice } from "./schema" @@ -20,8 +22,8 @@ export const recordInternalTransferInvoice = async ({ }: { paymentHash: string senderWalletId?: string - recipientWalletId: string - origin: "intraledger" | "cutover" + recipientWalletId?: string + origin: InternalTransferOrigin }): Promise => { try { await InternalTransferInvoice.create({ diff --git a/src/services/mongoose/payment-split-executions.ts b/src/services/mongoose/payment-split-executions.ts index 7de8a2ad8..5b396f792 100644 --- a/src/services/mongoose/payment-split-executions.ts +++ b/src/services/mongoose/payment-split-executions.ts @@ -8,6 +8,7 @@ export type PaymentSplitExecutionStatusValue = | "ibex_pending" | "failed" | "skipped_min" + | "skipped_internal" | "unknown" export type PaymentSplitExecutionRecord = { @@ -114,7 +115,13 @@ export const PaymentSplitExecutionsRepository = () => { const finishClaimed = async ( id: string, update: { - status: "pending" | "sent" | "ibex_pending" | "failed" + status: + | "pending" + | "sent" + | "ibex_pending" + | "failed" + | "skipped_internal" + | "unknown" attempts: number nextAttemptAt?: Date lastError?: string diff --git a/src/services/mongoose/schema.ts b/src/services/mongoose/schema.ts index 70a06714f..fc1da3a43 100644 --- a/src/services/mongoose/schema.ts +++ b/src/services/mongoose/schema.ts @@ -932,6 +932,7 @@ const PaymentSplitExecutionSchema = new Schema({ "ibex_pending", "failed", "skipped_min", + "skipped_internal", "unknown", ], required: true, @@ -953,15 +954,22 @@ export const PaymentSplitExecution = mongoose.model( // Payment hashes of invoices Flash minted for itself: the recipient side of an // intraledger send (P2P, Fygaro top-up credits, referral payouts, split -// payouts) and of cash-wallet cutover moves. A receive whose hash is here never -// triggers payment splits. TTL must stay above the split reconcile lookback. +// payouts) and of cash-wallet cutover moves, plus every bolt11 / LNURL a Flash +// user pays (the payee may be another Flash wallet). A receive whose hash is +// here never triggers payment splits. TTL must stay above the split reconcile +// lookback (cronIntervalMs plus margin). export const INTERNAL_TRANSFER_INVOICE_TTL_SECONDS = 30 * 24 * 60 * 60 const InternalTransferInvoiceSchema = new Schema({ paymentHash: { type: String, required: true, unique: true }, senderWalletId: { type: String }, - recipientWalletId: { type: String, required: true }, - origin: { type: String, enum: ["intraledger", "cutover"], required: true }, + // Unknown for ln-send / lnurl-send: Flash only knows the payer. + recipientWalletId: { type: String }, + origin: { + type: String, + enum: ["intraledger", "cutover", "ln-send", "lnurl-send"], + required: true, + }, createdAt: { type: Date, required: true, default: Date.now }, }) diff --git a/test/flash/unit/app/payment-splits/add-invoice-call-sites.spec.ts b/test/flash/unit/app/payment-splits/add-invoice-call-sites.spec.ts index a74717dae..b817dde18 100644 --- a/test/flash/unit/app/payment-splits/add-invoice-call-sites.spec.ts +++ b/test/flash/unit/app/payment-splits/add-invoice-call-sites.spec.ts @@ -11,8 +11,9 @@ const CLASSIFIED_CALL_SITES: Record = { "src/app/payments/send-intraledger.ts": "marked", // Marked internal (origin "cutover") before the invoice is handed back. "src/app/cash-wallet-cutover/runtime-services.ts": "marked", - // User-facing receive invoices: an external payer pays them; they SHOULD - // trigger splits. + // User-facing receive invoices: an external payer's payment SHOULD trigger + // splits. A Flash user paying one (QR scan, Lightning address) does not: + // the pay paths below mark the hash before / right after paying. "src/app/wallets/add-invoice-for-wallet.ts": "user-receive", "src/graphql/public/root/mutation/ln-noamount-invoice-create.ts": "user-receive", // Cashout: the user pays an invoice on Flash's own cashout wallet. Flash's @@ -27,6 +28,43 @@ const listTs = (dir: string): string[] => return path.endsWith(".ts") && !path.endsWith(".d.ts") ? [path] : [] }) +// ENG-653 decision 1: Flash-to-Flash payments never trigger splits. A Flash +// user can pay another Flash wallet through any of these, so each must mark the +// paid hash (or be classified as unable to reach a split wallet). +const CLASSIFIED_PAY_CALL_SITES: Record = { + // Pays the invoice it minted itself, after marking it (origin intraledger). + "src/app/payments/send-intraledger.ts": "marked", + // Pays the cutover invoice it minted and marked (origin cutover). + "src/app/cash-wallet-cutover/runtime-services.ts": "marked", + // Bolt11 pays: marked (origin ln-send) before paying, fail-closed. + "src/graphql/public/root/mutation/ln-invoice-payment-send.ts": "marked", + "src/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.ts": "marked", + // LNURL pay: IBEX fetches the invoice, so the hash is marked (origin + // lnurl-send) after paying and the executor re-checks before every payout. + "src/graphql/public/root/mutation/lnurl-payment-send.ts": "marked-after-pay", + // Cashout: pays Flash's own cashout wallet, which carries no split rules. + "src/app/offers/ValidOffer.ts": "flash-owned-recipient", + // Mentions Ibex.payToLnurl in a doc comment only. + "src/services/ibex/payment-status.ts": "comment-only", +} + +const listCallers = (pattern: RegExp): string[] => { + const root = join(__dirname, "../../../../..") + return listTs(join(root, "src")) + .filter((path) => !path.endsWith(join("services", "ibex", "client.ts"))) + .filter((path) => pattern.test(readFileSync(path, "utf8"))) + .map((path) => relative(root, path).split("\\").join("/")) + .sort() +} + +describe("Ibex.payInvoice / Ibex.payToLnurl call sites", () => { + it("are all classified for the Flash-to-Flash split marker", () => { + expect(listCallers(/\bIbex\.(payInvoice|payToLnurl)\b/)).toEqual( + Object.keys(CLASSIFIED_PAY_CALL_SITES).sort(), + ) + }) +}) + describe("Ibex.addInvoice call sites", () => { it("are all classified for the payment-splits marker", () => { const root = join(__dirname, "../../../../..") diff --git a/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts index cb45f2ca7..c59da40d3 100644 --- a/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts +++ b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts @@ -3,6 +3,8 @@ const mockAuthorizeSend = jest.fn() const mockNotify = jest.fn() const mockFindWalletById = jest.fn() const mockFindAccountById = jest.fn() +const mockIsInternal = jest.fn() +const mockNotifyOps = jest.fn() jest.mock("@app/payments/send-intraledger", () => ({ intraledgerPaymentSendWalletIdForUsdWallet: (...args: unknown[]) => mockSend(...args), @@ -25,6 +27,14 @@ jest.mock("@services/mongoose", () => ({ })), })) +jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ + isInternalTransferInvoice: (...args: unknown[]) => mockIsInternal(...args), +})) + +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: (...args: unknown[]) => mockNotifyOps(...args), +})) + jest.mock("@services/mongoose/payment-split-executions", () => ({ PaymentSplitExecutionsRepository: jest.fn(() => mockStore.repo), })) @@ -35,8 +45,17 @@ import { splitIdempotencyKey, } from "@app/payment-splits/execute-pending-splits" import { PaymentSendStatus } from "@domain/bitcoin/lightning" -import { InactiveAccountError, IntraledgerLimitsExceededError } from "@domain/errors" -import { InternalTransferMarkerError } from "@domain/payment-splits" +import { + InactiveAccountError, + IntraledgerLimitsExceededError, + RepositoryError, +} from "@domain/errors" +import { + ResourceAttemptsLockServiceError, + ResourceExpiredLockServiceError, + UnknownLockServiceError, +} from "@domain/lock" +import { InternalTransferMarkerError, splitRetryBackoffMs } from "@domain/payment-splits" import { SendLimitsUnavailableError } from "@domain/payments/errors" import { PaymentSendRateLimiterExceededError } from "@domain/rate-limit/errors" import { WalletCurrency } from "@domain/shared" @@ -80,8 +99,9 @@ beforeEach(() => { Object.assign(PaymentSplitsConfig, { enabled: true, maxAttempts: 5, - executorBatchSize: 50, + executorTimeBudgetMs: 45_000, }) + mockIsInternal.mockResolvedValue(false) mockFindWalletById.mockResolvedValue({ id: SOURCE, accountId: OWNER.id, @@ -109,6 +129,8 @@ describe("executePendingSplits", () => { ibexPending: 0, failed: 0, retried: 0, + unknown: 0, + skippedInternal: 0, }) expect(mockSend).toHaveBeenCalledWith( expect.objectContaining({ @@ -211,9 +233,14 @@ describe("executePendingSplits", () => { expect(mockNotify).toHaveBeenCalledTimes(1) }) + it("runs every minute in production, so the 1-minute backoff step is real", () => { + expect(PaymentSplitsConfig.loopIntervalMs).toBeLessThanOrEqual(splitRetryBackoffMs(1)) + }) + it.each([ ["send guard unavailable", new SendLimitsUnavailableError("redis")], ["marker write failure", new InternalTransferMarkerError("mongo")], + ["a busy idempotency lock", new ResourceAttemptsLockServiceError()], ])("treats %s as retryable", async (_label, error) => { await seed() mockSend.mockResolvedValue(error) @@ -264,11 +291,10 @@ describe("executePendingSplits", () => { ).toEqual([1, 0]) }) - it("claims at most executorBatchSize rows per run", async () => { - PaymentSplitsConfig.executorBatchSize = 2 - for (let i = 0; i < 3; i++) { + const seedMany = async (count: number) => { + for (let i = 0; i < count; i++) { await mockStore.repo.insert({ - triggerKey: `${i}`.repeat(64), + triggerKey: `${i}`.padStart(64, "c"), ruleId: "rule-1", walletId: SOURCE, destinationWalletId: DEST, @@ -279,10 +305,110 @@ describe("executePendingSplits", () => { now: T0, }) } + } + + it("drains every due row in one pass, with no fixed batch cap", async () => { + await seedMany(120) mockSend.mockResolvedValue(PaymentSendStatus.Success) + expect(await executePendingSplits({ now: T0 })).toMatchObject({ + claimed: 120, + sent: 120, + }) + expect([...mockStore.rows.values()].every((r) => r.status === "sent")).toBe(true) + }) + + it("stops claiming once the time budget is spent", async () => { + PaymentSplitsConfig.executorTimeBudgetMs = 1000 + await seedMany(3) + mockSend.mockResolvedValue(PaymentSendStatus.Success) + // Each clock read advances 600ms: the deadline is set at 600 + 1000, the + // first check (1200) claims one row, the second (1800) is past it. + let t = 0 + const clock = () => (t += 600) + + expect(await executePendingSplits({ now: T0, clock })).toMatchObject({ claimed: 1 }) expect(await executePendingSplits({ now: T0 })).toMatchObject({ claimed: 2 }) - expect(await executePendingSplits({ now: T0 })).toMatchObject({ claimed: 1 }) + }) + + describe("lock errors", () => { + // The send reached the payment: the authorize hook (which runs + // immediately before Ibex.payInvoice) passed, then the lock layer erred. + const afterAuthorize = + (error: Error) => + async (args: { authorize: () => Promise }): Promise => { + await args.authorize() + return error + } + + it.each([ + [ + "UnknownLockServiceError (routine threw / release failed)", + new UnknownLockServiceError("release"), + ], + ["ResourceExpiredLockServiceError", new ResourceExpiredLockServiceError("expired")], + [ + "ResourceAttemptsLockServiceError from the release", + new ResourceAttemptsLockServiceError(), + ], + ])( + "never retries %s after the payment was reached; marks unknown and alerts ops", + async (_label, error) => { + await seed() + mockSend.mockImplementation(afterAuthorize(error)) + + const summary = await executePendingSplits({ now: T0 }) + + expect(summary).toMatchObject({ claimed: 1, unknown: 1, retried: 0, failed: 0 }) + expect(onlyRow()).toMatchObject({ status: "unknown", attempts: 1 }) + expect(mockNotifyOps).toHaveBeenCalledWith( + expect.objectContaining({ phase: "split-unknown", status: "failed" }), + ) + + // Never re-sent, however long the executor keeps running. + await executePendingSplits({ now: minutes(60) }) + expect(mockSend).toHaveBeenCalledTimes(1) + }, + ) + + it.each([ + ["UnknownLockServiceError", new UnknownLockServiceError("boom")], + ["ResourceExpiredLockServiceError", new ResourceExpiredLockServiceError("expired")], + ])("never retries %s even before the payment was reached", async (_label, error) => { + await seed() + mockSend.mockResolvedValue(error) + + await executePendingSplits({ now: T0 }) + + expect(onlyRow()).toMatchObject({ status: "unknown" }) + await executePendingSplits({ now: minutes(60) }) + expect(mockSend).toHaveBeenCalledTimes(1) + }) + }) + + describe("internal-transfer re-check", () => { + it("skips a row whose hash was marked internal after it was recorded", async () => { + await seed() + mockIsInternal.mockResolvedValue(true) + + const summary = await executePendingSplits({ now: T0 }) + + expect(summary).toMatchObject({ claimed: 1, skippedInternal: 1 }) + expect(mockIsInternal).toHaveBeenCalledWith("a".repeat(64)) + expect(onlyRow()).toMatchObject({ status: "skipped_internal" }) + expect(mockSend).not.toHaveBeenCalled() + expect(mockNotify).not.toHaveBeenCalled() + }) + + it("retries when the marker cannot be read, without sending", async () => { + await seed() + mockIsInternal.mockResolvedValue(new RepositoryError("mongo")) + + await executePendingSplits({ now: T0 }) + + expect(onlyRow()).toMatchObject({ status: "pending", attempts: 1 }) + expect(mockSend).not.toHaveBeenCalled() + }) }) it("never sends a skipped_min row", async () => { diff --git a/test/flash/unit/app/payment-splits/reconcile-splits.spec.ts b/test/flash/unit/app/payment-splits/reconcile-splits.spec.ts index 70123157f..7e71aef21 100644 --- a/test/flash/unit/app/payment-splits/reconcile-splits.spec.ts +++ b/test/flash/unit/app/payment-splits/reconcile-splits.spec.ts @@ -37,6 +37,7 @@ jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ import { PaymentSplitsConfig } from "@config" import { recordSplitExecutions } from "@app/payment-splits/record-split-executions" import { reconcileSplits } from "@app/payment-splits/reconcile-splits" +import { splitReconcileWindowMs } from "@domain/payment-splits" import { WalletCurrency } from "@domain/shared" import { createFakeExecutionsStore } from "./fake-executions-store" @@ -47,7 +48,9 @@ const mockStore = createFakeExecutionsStore() const WALLET = "source-wallet" const NOW = new Date("2026-10-11T12:00:00Z") -const WINDOW_MS = 15 * 60 * 1000 +// The split loop reconciles every 15 minutes; its lookback is 25 minutes. +const CADENCE_MS = 15 * 60 * 1000 +const WINDOW_MS = 25 * 60 * 1000 const RULE_CREATED = new Date("2026-10-11T11:00:00Z") const HASH_1 = "1".repeat(64) const HASH_2 = "2".repeat(64) @@ -98,9 +101,15 @@ describe("reconcileSplits", () => { it("records a missed Lightning receive keyed by its payment hash", async () => { mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, minutesAgo(5))]) - const summary = await reconcileSplits({ windowMs: WINDOW_MS, now: NOW }) + const summary = await reconcileSplits({ cadenceMs: CADENCE_MS, now: NOW }) - expect(summary).toEqual({ wallets: 1, receivesSeen: 1, recorded: 1, errors: 0 }) + expect(summary).toEqual({ + windowMs: WINDOW_MS, + wallets: 1, + receivesSeen: 1, + recorded: 1, + errors: 0, + }) expect(mockGetAccountTransactions).toHaveBeenCalledWith( expect.objectContaining({ account_id: WALLET }), ) @@ -122,7 +131,7 @@ describe("reconcileSplits", () => { }) mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, minutesAgo(5))]) - const summary = await reconcileSplits({ windowMs: WINDOW_MS, now: NOW }) + const summary = await reconcileSplits({ cadenceMs: CADENCE_MS, now: NOW }) expect(summary).toMatchObject({ receivesSeen: 1, recorded: 0 }) expect(mockStore.rows.size).toBe(1) @@ -141,7 +150,7 @@ describe("reconcileSplits", () => { ]) mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, minutesAgo(5))]) - expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + expect(await reconcileSplits({ cadenceMs: CADENCE_MS, now: NOW })).toMatchObject({ recorded: 0, }) expect(mockStore.rows.size).toBe(0) @@ -154,7 +163,7 @@ describe("reconcileSplits", () => { tx("tx-3", 3, minutesAgo(1)), ]) - expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + expect(await reconcileSplits({ cadenceMs: CADENCE_MS, now: NOW })).toMatchObject({ receivesSeen: 0, recorded: 0, }) @@ -163,9 +172,9 @@ describe("reconcileSplits", () => { }) it("ignores receives older than the lookback window", async () => { - mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, minutesAgo(16))]) + mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, minutesAgo(26))]) - expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + expect(await reconcileSplits({ cadenceMs: CADENCE_MS, now: NOW })).toMatchObject({ receivesSeen: 0, }) expect(mockGetTransactionDetails).not.toHaveBeenCalled() @@ -178,7 +187,7 @@ describe("reconcileSplits", () => { tx("tx-2", 1, minutesAgo(3)), ]) - expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + expect(await reconcileSplits({ cadenceMs: CADENCE_MS, now: NOW })).toMatchObject({ receivesSeen: 2, recorded: 1, }) @@ -191,17 +200,76 @@ describe("reconcileSplits", () => { .mockResolvedValueOnce(new Error("ibex down")) .mockResolvedValueOnce([tx("tx-1", 1, minutesAgo(2))]) - expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + expect(await reconcileSplits({ cadenceMs: CADENCE_MS, now: NOW })).toMatchObject({ wallets: 2, errors: 1, recorded: 1, }) }) + it("pages through IBEX until a page reaches past the lookback", async () => { + // A busy merchant wallet: the newest 100 transactions are all sends, and + // the missed receive sits on the second page. + const sends = Array.from({ length: 100 }, (_, i) => + tx(`send-${i}`, 2, minutesAgo(1 + i * 0.1)), + ) + mockGetAccountTransactions.mockImplementation(async ({ page }: { page: number }) => { + if (page === 0) return sends + if (page === 1) + return [ + tx("tx-1", 1, minutesAgo(20)), + ...Array.from({ length: 99 }, (_, i) => tx(`old-${i}`, 2, minutesAgo(30 + i))), + ] + throw new Error(`page ${page} must not be read`) + }) + + expect(await reconcileSplits({ cadenceMs: CADENCE_MS, now: NOW })).toMatchObject({ + receivesSeen: 1, + recorded: 1, + errors: 0, + }) + expect(mockGetAccountTransactions).toHaveBeenCalledTimes(2) + expect(mockGetAccountTransactions).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ account_id: WALLET, page: 0, sort: "settledAt" }), + ) + expect(mockGetAccountTransactions).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ account_id: WALLET, page: 1 }), + ) + }) + + it("a daily cron run reaches back to a receive from the previous afternoon", async () => { + // Cron fires at 02:00; the webhook for a 14:00 receive the day before + // failed. The lookback derived from a 24h cadence covers it. + const cronNow = new Date("2026-10-12T02:00:00Z") + mockGetAccountTransactions.mockResolvedValue([tx("tx-1", 1, "2026-10-11T14:00:00Z")]) + + const summary = await reconcileSplits({ + cadenceMs: 24 * 60 * 60 * 1000, + now: cronNow, + }) + + expect(summary).toMatchObject({ receivesSeen: 1, recorded: 1 }) + expect(summary instanceof Error ? 0 : summary.windowMs).toBeGreaterThan( + 24 * 60 * 60 * 1000, + ) + }) + + it("never looks back less than the configured cadence of either caller", () => { + for (const cadenceMs of [ + PaymentSplitsConfig.reconcileIntervalMs, + PaymentSplitsConfig.cronIntervalMs, + ]) { + expect(splitReconcileWindowMs(cadenceMs)).toBeGreaterThan(cadenceMs) + } + expect(PaymentSplitsConfig.cronIntervalMs).toBeGreaterThanOrEqual(24 * 60 * 60 * 1000) + }) + it("does nothing while the feature flag is off", async () => { PaymentSplitsConfig.enabled = false - expect(await reconcileSplits({ windowMs: WINDOW_MS, now: NOW })).toMatchObject({ + expect(await reconcileSplits({ cadenceMs: CADENCE_MS, now: NOW })).toMatchObject({ wallets: 0, }) expect(mockListWalletIds).not.toHaveBeenCalled() diff --git a/test/flash/unit/app/payment-splits/set-splits.spec.ts b/test/flash/unit/app/payment-splits/set-splits.spec.ts index 2a3766633..09a62ce76 100644 --- a/test/flash/unit/app/payment-splits/set-splits.spec.ts +++ b/test/flash/unit/app/payment-splits/set-splits.spec.ts @@ -212,6 +212,18 @@ describe("setPaymentSplits", () => { expect(mockReplaceForWallet).not.toHaveBeenCalled() }) + it("clears every split when given an empty set", async () => { + const result = await setPaymentSplits({ + account: OWNER, + walletId: SOURCE_ID, + splits: [], + }) + + expect(result).toEqual([]) + expect(mockLockWalletId).toHaveBeenCalledWith(SOURCE_ID, expect.any(Function)) + expect(mockReplaceForWallet).toHaveBeenCalledWith(SOURCE_ID, []) + }) + it("rejects a set over 100% without writing", async () => { mockFindWalletById.mockImplementation(async (id: string) => id === SOURCE_ID diff --git a/test/flash/unit/app/payment-splits/split-loop.spec.ts b/test/flash/unit/app/payment-splits/split-loop.spec.ts new file mode 100644 index 000000000..964e4c3ff --- /dev/null +++ b/test/flash/unit/app/payment-splits/split-loop.spec.ts @@ -0,0 +1,121 @@ +const mockReconcile = jest.fn() +const mockExecute = jest.fn() +const mockSweep = jest.fn() +const mockRedisSet = jest.fn() + +jest.mock("@app/payment-splits/reconcile-splits", () => ({ + reconcileSplits: (...args: unknown[]) => mockReconcile(...args), +})) +jest.mock("@app/payment-splits/execute-pending-splits", () => ({ + executePendingSplits: (...args: unknown[]) => mockExecute(...args), +})) +jest.mock("@app/payment-splits/sweep-stale-splits", () => ({ + sweepStaleSplits: (...args: unknown[]) => mockSweep(...args), +})) +jest.mock("@services/redis", () => ({ + redis: { set: (...args: unknown[]) => mockRedisSet(...args) }, +})) + +import { PaymentSplitsConfig } from "@config" +import { + runPaymentSplitsTick, + startPaymentSplitsLoop, +} from "@app/payment-splits/split-loop" +import { splitRetryBackoffMs } from "@domain/payment-splits" + +const NOW = new Date("2026-10-11T12:00:00Z") +const original = { ...PaymentSplitsConfig } + +beforeEach(() => { + jest.clearAllMocks() + Object.assign(PaymentSplitsConfig, original, { enabled: true }) + mockReconcile.mockResolvedValue({ recorded: 0, errors: 0 }) + mockExecute.mockResolvedValue({ claimed: 0 }) + mockSweep.mockResolvedValue({ markedUnknown: 0 }) + mockRedisSet.mockResolvedValue("OK") +}) + +afterAll(() => { + Object.assign(PaymentSplitsConfig, original) +}) + +describe("runPaymentSplitsTick", () => { + it("reconciles on its own cadence, then executes and sweeps", async () => { + await runPaymentSplitsTick({ now: NOW }) + + expect(mockReconcile).toHaveBeenCalledWith({ + cadenceMs: PaymentSplitsConfig.reconcileIntervalMs, + now: NOW, + }) + expect(mockRedisSet).toHaveBeenCalledWith( + "payment-splits:reconcile:lease", + "1", + "EX", + Math.floor((PaymentSplitsConfig.reconcileIntervalMs * 0.9) / 1000), + "NX", + ) + expect(mockExecute).toHaveBeenCalledWith({ now: NOW }) + expect(mockSweep).toHaveBeenCalledWith({ now: NOW }) + }) + + it("skips the reconcile while another tick holds the lease, but still executes", async () => { + mockRedisSet.mockResolvedValue(null) + + await runPaymentSplitsTick({ now: NOW }) + + expect(mockReconcile).not.toHaveBeenCalled() + expect(mockExecute).toHaveBeenCalledTimes(1) + expect(mockSweep).toHaveBeenCalledTimes(1) + }) + + it("reconciles anyway when Redis is unreachable (fail-open)", async () => { + mockRedisSet.mockRejectedValue(new Error("redis down")) + + await runPaymentSplitsTick({ now: NOW }) + + expect(mockReconcile).toHaveBeenCalledTimes(1) + }) + + it("keeps executing when the reconcile fails", async () => { + mockReconcile.mockResolvedValue(new Error("mongo")) + + await runPaymentSplitsTick({ now: NOW }) + + expect(mockExecute).toHaveBeenCalledTimes(1) + }) + + it("does nothing while the feature flag is off", async () => { + PaymentSplitsConfig.enabled = false + + await runPaymentSplitsTick({ now: NOW }) + + expect(mockReconcile).not.toHaveBeenCalled() + expect(mockExecute).not.toHaveBeenCalled() + }) +}) + +describe("startPaymentSplitsLoop", () => { + it("is not started while the feature flag is off", () => { + PaymentSplitsConfig.enabled = false + expect(startPaymentSplitsLoop()).toBeUndefined() + }) + + it("runs often enough for the executor's 1-minute retry step", () => { + expect(PaymentSplitsConfig.loopIntervalMs).toBeLessThanOrEqual(splitRetryBackoffMs(1)) + expect(PaymentSplitsConfig.reconcileIntervalMs).toBeLessThan( + PaymentSplitsConfig.cronIntervalMs, + ) + }) + + it("starts a timer when enabled", () => { + jest.useFakeTimers() + try { + const timer = startPaymentSplitsLoop() + expect(timer).toBeDefined() + clearInterval(timer) + } finally { + jest.clearAllTimers() + jest.useRealTimers() + } + }) +}) diff --git a/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts b/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts new file mode 100644 index 000000000..ccd2af3e7 --- /dev/null +++ b/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts @@ -0,0 +1,86 @@ +const mockRecordInternalTransferInvoice = jest.fn() +jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ + recordInternalTransferInvoice: (...args: unknown[]) => + mockRecordInternalTransferInvoice(...args), +})) + +jest.mock("@services/tracing", () => ({ + recordExceptionInCurrentSpan: jest.fn(), +})) + +import { + markOutgoingBolt11, + markPaidLnurlHash, +} from "@app/payments/mark-outgoing-invoice" +import { RepositoryError } from "@domain/errors" +import { InternalTransferMarkerError } from "@domain/payment-splits" + +// BOLT11 spec test vector; its payment hash is 0001…0102. +const BOLT11 = + "lnbc20u1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kxqrrsssp5m6kmam774klwlh4dhmhaatd7al02m0h0m6kmam774klwlh4dhmhs9qypqqqcqpf3cwux5979a8j28d4ydwahx00saa68wq3az7v9jdgzkghtxnkf3z5t7q5suyq2dl9tqwsap8j0wptc82cpyvey9gf6zyylzrm60qtcqsq7egtsq" +const HASH = "0001020304050607080900010203040506070809000102030405060708090102" +const SENDER = "sender-wallet" as WalletId + +beforeEach(() => { + jest.clearAllMocks() + mockRecordInternalTransferInvoice.mockResolvedValue(true) +}) + +describe("markOutgoingBolt11", () => { + it("marks the bolt11's payment hash as an ln-send from the paying wallet", async () => { + expect( + await markOutgoingBolt11({ paymentRequest: BOLT11, senderWalletId: SENDER }), + ).toBe(true) + expect(mockRecordInternalTransferInvoice).toHaveBeenCalledWith({ + paymentHash: HASH, + senderWalletId: SENDER, + origin: "ln-send", + }) + }) + + it("fails closed when the marker cannot be written", async () => { + mockRecordInternalTransferInvoice.mockResolvedValue(new RepositoryError("mongo")) + + expect( + await markOutgoingBolt11({ paymentRequest: BOLT11, senderWalletId: SENDER }), + ).toBeInstanceOf(InternalTransferMarkerError) + }) + + it("lets an undecodable request through unmarked (IBEX-minted invoices always decode)", async () => { + expect( + await markOutgoingBolt11({ + paymentRequest: "lnbc1garbage", + senderWalletId: SENDER, + }), + ).toBe(true) + expect(mockRecordInternalTransferInvoice).not.toHaveBeenCalled() + }) +}) + +describe("markPaidLnurlHash", () => { + it("marks the paid hash as an lnurl-send", async () => { + await markPaidLnurlHash({ paymentHash: HASH, senderWalletId: SENDER }) + + expect(mockRecordInternalTransferInvoice).toHaveBeenCalledWith({ + paymentHash: HASH, + senderWalletId: SENDER, + origin: "lnurl-send", + }) + }) + + it.each([undefined, null, "not-a-hash"])( + "ignores a missing or malformed hash (%p)", + async (paymentHash) => { + await markPaidLnurlHash({ paymentHash, senderWalletId: SENDER }) + expect(mockRecordInternalTransferInvoice).not.toHaveBeenCalled() + }, + ) + + it("never throws after the money moved, even if the write fails", async () => { + mockRecordInternalTransferInvoice.mockResolvedValue(new RepositoryError("mongo")) + + await expect( + markPaidLnurlHash({ paymentHash: HASH, senderWalletId: SENDER }), + ).resolves.toBeUndefined() + }) +}) diff --git a/test/flash/unit/domain/payment-splits/validate.spec.ts b/test/flash/unit/domain/payment-splits/validate.spec.ts index 5b6000259..e101420e9 100644 --- a/test/flash/unit/domain/payment-splits/validate.spec.ts +++ b/test/flash/unit/domain/payment-splits/validate.spec.ts @@ -36,8 +36,12 @@ describe("validateSplitSet", () => { ) }) - it("rejects a 0 sum (the empty set)", () => { - expect(validate([])).toBeInstanceOf(SplitPercentExceededError) + it("accepts the empty set: it clears the wallet's splits", () => { + expect(validate([])).toBe(true) + }) + + it("still rejects an empty set on a non-split currency", () => { + expect(validate([], WalletCurrency.Btc)).toBeInstanceOf(SplitCurrencyMismatchError) }) it("rejects non-integer and out-of-range percents", () => { diff --git a/test/flash/unit/graphql/ln-invoice-payment-send.spec.ts b/test/flash/unit/graphql/ln-invoice-payment-send.spec.ts index adf3cfc8e..cf3218f2f 100644 --- a/test/flash/unit/graphql/ln-invoice-payment-send.spec.ts +++ b/test/flash/unit/graphql/ln-invoice-payment-send.spec.ts @@ -82,9 +82,20 @@ jest.mock("@app/payments/idempotency", () => ({ mockWithPaymentIdempotency(...args), })) +// ENG-653: the payee may be another Flash wallet; the resolver marks the +// bolt11's hash before paying so the payee's splits never fire. Default-allow. +const mockMarkOutgoingBolt11 = jest.fn() +jest.mock("@app/payments/mark-outgoing-invoice", () => ({ + markOutgoingBolt11: async (args: unknown) => { + const result = await mockMarkOutgoingBolt11(args) + return result === undefined ? true : result + }, +})) + import { ErrorLevel } from "@domain/shared" import { WithdrawalLimitsExceededError } from "@domain/errors" import { LnInvoiceDecodeError } from "@domain/bitcoin/lightning/errors" +import { InternalTransferMarkerError } from "@domain/payment-splits" import LnInvoicePaymentSendMutation from "@graphql/public/root/mutation/ln-invoice-payment-send" import { IbexError, @@ -447,3 +458,43 @@ describe("ENG-573 decode gate against a real bolt11", () => { }) }) }) + +describe("lnInvoicePaymentSend Flash-to-Flash split marker (ENG-653)", () => { + beforeEach(() => { + jest.clearAllMocks() + mockSendGuardMode.mockReturnValue("log-only") + mockPayInvoice.mockResolvedValue({ + transaction: { payment: { status: { id: 2 } } }, + }) + }) + + it("marks the bolt11 for the paying wallet before paying it", async () => { + const order: string[] = [] + mockMarkOutgoingBolt11.mockImplementationOnce(async () => { + order.push("mark") + return true + }) + mockPayInvoice.mockImplementation(async () => { + order.push("pay") + return { transaction: { payment: { status: { id: 2 } } } } + }) + + const result = await resolvePayment() + + expect(mockMarkOutgoingBolt11).toHaveBeenCalledWith({ + paymentRequest: "lnbc1", + senderWalletId: "wallet-1", + }) + expect(order).toEqual(["mark", "pay"]) + expect(result.status).toBe("success") + }) + + it("pays nothing when the marker cannot be written", async () => { + mockMarkOutgoingBolt11.mockResolvedValueOnce(new InternalTransferMarkerError("mongo")) + + const result = await resolvePayment() + + expect(mockPayInvoice).not.toHaveBeenCalled() + expect(result.status).toBe("failed") + }) +}) diff --git a/test/flash/unit/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.spec.ts b/test/flash/unit/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.spec.ts index b56d97cf8..104d566cb 100644 --- a/test/flash/unit/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.spec.ts +++ b/test/flash/unit/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.spec.ts @@ -51,7 +51,18 @@ jest.mock("@app/wallets", () => ({ ) => mockUsdWalletAmountFromWalletId(...args), })) +// ENG-653: the payee may be another Flash wallet; the resolver marks the +// bolt11's hash before paying so the payee's splits never fire. Default-allow. +const mockMarkOutgoingBolt11 = jest.fn() +jest.mock("@app/payments/mark-outgoing-invoice", () => ({ + markOutgoingBolt11: async (args: unknown) => { + const result = await mockMarkOutgoingBolt11(args) + return result === undefined ? true : result + }, +})) + import { IdempotencyKeyReuseError, WithdrawalLimitsExceededError } from "@domain/errors" +import { InternalTransferMarkerError } from "@domain/payment-splits" import { ErrorLevel, USDTAmount } from "@domain/shared" import LnNoAmountUsdInvoicePaymentSendMutation from "@graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send" import { IbexError, UnconfirmedIbexPayment } from "@services/ibex/errors" @@ -115,6 +126,36 @@ describe("LnNoAmountUsdInvoicePaymentSendMutation", () => { ) }) + it("marks the bolt11 for the routed wallet before paying it (ENG-653)", async () => { + const order: string[] = [] + mockMarkOutgoingBolt11.mockImplementationOnce(async () => { + order.push("mark") + return true + }) + mockPayInvoice.mockImplementation(async () => { + order.push("pay") + return { status: 0, transaction: { payment: { status: { id: 2 } } } } + }) + + const result = await resolveMutation() + + expect(mockMarkOutgoingBolt11).toHaveBeenCalledWith({ + paymentRequest: "lnbc1noamount", + senderWalletId: routedWalletId, + }) + expect(order).toEqual(["mark", "pay"]) + expect(result).toEqual({ errors: [], status: "success" }) + }) + + it("pays nothing when the split marker cannot be written (ENG-653)", async () => { + mockMarkOutgoingBolt11.mockResolvedValueOnce(new InternalTransferMarkerError("mongo")) + + const result = await resolveMutation() + + expect(mockPayInvoice).not.toHaveBeenCalled() + expect(result.status).toBe("failed") + }) + it("pays the routed wallet with the resolved cent amount", async () => { const result = await resolveMutation() diff --git a/test/flash/unit/graphql/public/root/mutation/lnurl-payment-send.spec.ts b/test/flash/unit/graphql/public/root/mutation/lnurl-payment-send.spec.ts index 08ccee236..2a769eadc 100644 --- a/test/flash/unit/graphql/public/root/mutation/lnurl-payment-send.spec.ts +++ b/test/flash/unit/graphql/public/root/mutation/lnurl-payment-send.spec.ts @@ -59,6 +59,13 @@ jest.mock("axios", () => ({ get: (...args: Parameters) => mockAxiosGet(...args), })) +// ENG-653: IBEX fetches the LNURL invoice itself, so the hash is marked after +// the payment returns. Recorded here so the wiring can be asserted. +const mockMarkPaidLnurlHash = jest.fn() +jest.mock("@app/payments/mark-outgoing-invoice", () => ({ + markPaidLnurlHash: (...args: unknown[]) => mockMarkPaidLnurlHash(...args), +})) + import LnurlPaymentSendMutation from "@graphql/public/root/mutation/lnurl-payment-send" import { IdempotencyKeyReuseError, WithdrawalLimitsExceededError } from "@domain/errors" import { paymentAmountFromNumber, USDTAmount, WalletCurrency } from "@domain/shared" @@ -180,6 +187,38 @@ describe("LnurlPaymentSendMutation", () => { expect(result).toEqual({ errors: [], status: "success" }) }) + it("marks the paid hash for the routed wallet after IBEX pays (ENG-653)", async () => { + const hash = "ab".repeat(32) + mockPayToLnurl.mockResolvedValue({ + hash, + transaction: { payment: { status: { id: 2 } } }, + }) + + await resolveMutation() + + expect(mockMarkPaidLnurlHash).toHaveBeenCalledWith({ + paymentHash: hash, + senderWalletId: routedWalletId, + }) + }) + + it("falls back to the payment-level hash and never marks a failed IBEX call (ENG-653)", async () => { + const hash = "cd".repeat(32) + mockPayToLnurl.mockResolvedValueOnce({ + transaction: { payment: { hash, status: { id: 2 } } }, + }) + await resolveMutation() + expect(mockMarkPaidLnurlHash).toHaveBeenLastCalledWith({ + paymentHash: hash, + senderWalletId: routedWalletId, + }) + + mockMarkPaidLnurlHash.mockClear() + mockPayToLnurl.mockResolvedValueOnce(new IbexError(new Error("boom"))) + await resolveMutation() + expect(mockMarkPaidLnurlHash).not.toHaveBeenCalled() + }) + it("rejects converted msats below LNURL minSendable before calling IBEX pay", async () => { mockAxiosGet.mockResolvedValueOnce({ data: { From 84c38c85b38bfd57512ef3aee4d0a21a4c4ad6ca Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 23:25:47 -0700 Subject: [PATCH 10/13] fix(payment-splits): withdraw rejected ln-send markers, grace new rows, drain loop on SIGTERM - payMarkedBolt11 marks the bolt11, pays, and deletes its own ln-send marker when IBEX rejects the payment outright (insufficient balance, corroborated FAILED, other 4xx). Pending, unreadable, 5xx, timeouts and already-paid keep the marker. A rejected Flash attempt no longer turns off the payee's splits for a later external payment of the same invoice. - New pending split executions become claimable only after paymentSplits.claimGraceMs (default 2 min), so a post-payment LNURL internal marker lands before the executor's pay-time re-check. - startPaymentSplitsLoop returns stop(): no new tick, the executor claims no new row, and the in-flight send finishes. The ibex-webhook server awaits it on SIGTERM/SIGINT (bounded at 25s) and re-raises the signal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G4SaLJF5SWq6sjBEaWJFow --- dev/config/base-config.yaml | 1 + .../payment-splits/execute-pending-splits.ts | 10 +- .../payment-splits/record-split-executions.ts | 7 + src/app/payment-splits/split-loop.ts | 82 ++++++- src/app/payments/mark-outgoing-invoice.ts | 104 +++++++- src/config/schema.ts | 7 + src/config/schema.types.d.ts | 1 + src/config/yaml.ts | 1 + .../root/mutation/ln-invoice-payment-send.ts | 31 +-- .../ln-noamount-usd-invoice-payment-send.ts | 25 +- src/servers/ibex-webhook-server.ts | 12 +- .../mongoose/internal-transfer-invoices.ts | 64 ++++- .../mongoose/payment-split-executions.ts | 5 +- .../payment-splits-repositories.spec.ts | 14 ++ .../execute-pending-splits.spec.ts | 18 ++ .../payment-splits/fake-executions-store.ts | 5 +- .../record-split-executions.spec.ts | 18 ++ .../app/payment-splits/split-loop.spec.ts | 121 +++++++++- .../payments/mark-outgoing-invoice.spec.ts | 133 ++++++++++- .../graphql/ln-invoice-payment-send.spec.ts | 14 +- .../graphql/ln-send-marker-withdrawal.spec.ts | 224 ++++++++++++++++++ ...-noamount-usd-invoice-payment-send.spec.ts | 14 +- 22 files changed, 832 insertions(+), 79 deletions(-) create mode 100644 test/flash/unit/graphql/ln-send-marker-withdrawal.spec.ts diff --git a/dev/config/base-config.yaml b/dev/config/base-config.yaml index ec2bfb213..9106be4bc 100644 --- a/dev/config/base-config.yaml +++ b/dev/config/base-config.yaml @@ -63,6 +63,7 @@ paymentSplits: minShareCents: 1 loopIntervalMs: 60000 # executor + stale-sweep cadence inside the ibex-webhook pod executorTimeBudgetMs: 45000 # one executor pass drains due rows for at most this long + claimGraceMs: 120000 # a new pending split is not claimable for this long (lets a post-payment LNURL internal marker land) reconcileIntervalMs: 900000 # webhook-safety-net reconcile cadence inside the same loop cronIntervalMs: 86400000 # galoy cronjob schedule (chart: 0 2 * * *); sizes the cron reconcile lookback diff --git a/src/app/payment-splits/execute-pending-splits.ts b/src/app/payment-splits/execute-pending-splits.ts index 733eeafa9..526e41229 100644 --- a/src/app/payment-splits/execute-pending-splits.ts +++ b/src/app/payment-splits/execute-pending-splits.ts @@ -259,11 +259,14 @@ export type ExecutePendingSplitsSummary = { * throughput is not capped by a fixed batch per run. `now` is fixed per pass: * a row retried in this pass is due at now + backoff and is picked up by a * later pass, which is what makes the 1/2/4/8 minute backoff real. + * + * `shouldStop` lets the loop drain on SIGTERM (see startPaymentSplitsLoop). */ export const executePendingSplits = async ({ now = new Date(), clock = () => Date.now(), -}: { now?: Date; clock?: () => number } = {}): Promise< + shouldStop = () => false, +}: { now?: Date; clock?: () => number; shouldStop?: () => boolean } = {}): Promise< ExecutePendingSplitsSummary | Error > => { const summary: ExecutePendingSplitsSummary = { @@ -279,7 +282,10 @@ export const executePendingSplits = async ({ const executions = PaymentSplitExecutionsRepository() const deadline = clock() + PaymentSplitsConfig.executorTimeBudgetMs - while (clock() < deadline) { + // `shouldStop` is checked before every claim, so on shutdown the row in + // flight finishes and no new row moves to `sending` (where a kill would + // strand it for the stale sweep to mark `unknown`). + while (clock() < deadline && !shouldStop()) { const execution = await executions.claimNextDue(now) if (execution instanceof Error) return execution if (execution === null) break diff --git a/src/app/payment-splits/record-split-executions.ts b/src/app/payment-splits/record-split-executions.ts index a74ae208e..757d095ad 100644 --- a/src/app/payment-splits/record-split-executions.ts +++ b/src/app/payment-splits/record-split-executions.ts @@ -59,6 +59,12 @@ export const recordSplitExecutions = async ({ if (internal) return NOTHING } + // ENG-653 decision 1: an LNURL send from another Flash user is marked + // internal only after payToLnurl returns, so this receive can be recorded + // before the marker exists. The executor re-checks the marker at pay time; + // holding the row back for claimGraceMs lets that marker land first. + const firstClaimableAt = new Date(now.getTime() + PaymentSplitsConfig.claimGraceMs) + const executions = PaymentSplitExecutionsRepository() const result = { ...NOTHING } for (const rule of rules) { @@ -81,6 +87,7 @@ export const recordSplitExecutions = async ({ triggerAmountMinor: share.triggerAmountMinor, shareCents: share.shareCents, status: share.status === "ok" ? "pending" : "skipped_min", + nextAttemptAt: firstClaimableAt, now, }) if (inserted instanceof Error) return inserted diff --git a/src/app/payment-splits/split-loop.ts b/src/app/payment-splits/split-loop.ts index eb319d497..18c06b287 100644 --- a/src/app/payment-splits/split-loop.ts +++ b/src/app/payment-splits/split-loop.ts @@ -39,10 +39,14 @@ const claimReconcileLease = async (intervalMs: number): Promise => { */ export const runPaymentSplitsTick = async ({ now = new Date(), -}: { now?: Date } = {}): Promise => { + shouldStop = () => false, +}: { now?: Date; shouldStop?: () => boolean } = {}): Promise => { if (!PaymentSplitsConfig.enabled) return - if (await claimReconcileLease(PaymentSplitsConfig.reconcileIntervalMs)) { + if ( + !shouldStop() && + (await claimReconcileLease(PaymentSplitsConfig.reconcileIntervalMs)) + ) { const reconciled = await reconcileSplits({ cadenceMs: PaymentSplitsConfig.reconcileIntervalMs, now, @@ -54,19 +58,32 @@ export const runPaymentSplitsTick = async ({ } } - const executed = await executePendingSplits({ now }) + if (shouldStop()) return + + const executed = await executePendingSplits({ now, shouldStop }) if (executed instanceof Error) { logger.error({ err: executed }, "payment splits loop: executor failed") } else if (executed.claimed > 0) { logger.info({ result: executed }, "payment splits loop: executor finished") } + if (shouldStop()) return + const swept = await sweepStaleSplits({ now }) if (swept instanceof Error) { logger.error({ err: swept }, "payment splits loop: stale sweep failed") } } +export type PaymentSplitsLoop = { + /** + * Stops the loop for shutdown: no further tick starts, the tick in flight + * claims no new row, and the returned promise settles once that tick (and + * the split send it may be in the middle of) has finished. + */ + stop: () => Promise +} + /** * Runs the split pipeline every loopIntervalMs inside the long-running * ibex-webhook workload. The galoy cron is a one-shot k8s Job whose schedule @@ -75,28 +92,71 @@ export const runPaymentSplitsTick = async ({ * through it. The webhook pod records the splits, so it is up whenever there * is anything to pay. The daily cron keeps a long-lookback reconcile as the * backstop for this loop being down. + * + * The caller must `await loop.stop()` on SIGTERM. A pod killed mid-send leaves + * its row in `sending`, which the stale sweep later marks `unknown` for ops. */ -export const startPaymentSplitsLoop = (): NodeJS.Timeout | undefined => { +export const startPaymentSplitsLoop = (): PaymentSplitsLoop | undefined => { if (!PaymentSplitsConfig.enabled) return undefined const intervalMs = PaymentSplitsConfig.loopIntervalMs - let running = false + let stopping = false + let inFlight: Promise | null = null + const shouldStop = () => stopping // A tick that outlives the interval (a long executor drain) is not stacked // on: the next fire is skipped until it finishes. const fire = () => { - if (running) return - running = true - runPaymentSplitsTick() + if (stopping || inFlight) return + inFlight = runPaymentSplitsTick({ shouldStop }) .catch((err) => logger.error({ err }, "payment splits loop tick errored")) .finally(() => { - running = false + inFlight = null }) } - setTimeout(fire, FIRST_RUN_DELAY_MS).unref() + const firstRun = setTimeout(fire, FIRST_RUN_DELAY_MS) + firstRun.unref() const timer = setInterval(fire, intervalMs) timer.unref() logger.info({ intervalMs }, "payment splits loop started") - return timer + + const stop = async (): Promise => { + stopping = true + clearTimeout(firstRun) + clearInterval(timer) + if (inFlight) await inFlight + logger.info("payment splits loop stopped") + } + + return { stop } +} + +const SHUTDOWN_SIGNALS: NodeJS.Signals[] = ["SIGTERM", "SIGINT"] +// Below the pod's 30s termination grace period, so a send that hangs can never +// be the reason the pod is SIGKILLed instead of exiting. +const SHUTDOWN_TIMEOUT_MS = 25_000 + +/** + * On SIGTERM/SIGINT, stops the loop and lets the split send in flight finish + * (bounded by SHUTDOWN_TIMEOUT_MS), then hands the signal back so the process + * dies the way it would have without this hook. The listeners remove + * themselves first, so the re-raised signal is not caught again. + */ +export const stopPaymentSplitsLoopOnShutdown = ( + loop: PaymentSplitsLoop, + { timeoutMs = SHUTDOWN_TIMEOUT_MS }: { timeoutMs?: number } = {}, +): void => { + const onSignal = (signal: NodeJS.Signals): void => { + for (const s of SHUTDOWN_SIGNALS) process.removeListener(s, onSignal) + logger.info({ signal }, "payment splits loop: stopping for shutdown") + + Promise.race([ + loop.stop(), + new Promise((resolve) => setTimeout(resolve, timeoutMs).unref()), + ]) + .catch((err) => logger.error({ err }, "payment splits loop: stop failed")) + .then(() => process.kill(process.pid, signal)) + } + for (const s of SHUTDOWN_SIGNALS) process.on(s, onSignal) } diff --git a/src/app/payments/mark-outgoing-invoice.ts b/src/app/payments/mark-outgoing-invoice.ts index bfd8b428e..50b9df7e5 100644 --- a/src/app/payments/mark-outgoing-invoice.ts +++ b/src/app/payments/mark-outgoing-invoice.ts @@ -1,11 +1,21 @@ -import { paymentHashFromBolt11 } from "@domain/bitcoin/lightning" +import { PaymentSendStatus, paymentHashFromBolt11 } from "@domain/bitcoin/lightning" import { InternalTransferMarkerError, InternalTransferOrigin, } from "@domain/payment-splits" import { ErrorLevel } from "@domain/shared" -import { recordInternalTransferInvoice } from "@services/mongoose/internal-transfer-invoices" +import { + CompletedInvoice, + FailedIbexPayment, + IbexError, + InsufficientIbexBalance, +} from "@services/ibex/errors" +import { + deleteInternalTransferInvoice, + insertInternalTransferInvoice, + recordInternalTransferInvoice, +} from "@services/mongoose/internal-transfer-invoices" import { baseLogger } from "@services/logger" import { recordExceptionInCurrentSpan } from "@services/tracing" @@ -25,23 +35,101 @@ const PaymentHashRegex = /^[0-9a-f]{64}$/i * is paid unmarked: every invoice IBEX mints for a Flash wallet decodes (the * intraledger marker depends on that), so an undecodable one is external. */ -export const markOutgoingBolt11 = async ({ +type OutgoingBolt11Marker = { paymentHash: string; created: boolean } | null + +const markOutgoing = async ({ paymentRequest, senderWalletId, }: { paymentRequest: string senderWalletId: WalletId -}): Promise => { +}): Promise => { const paymentHash = paymentHashFromBolt11(paymentRequest) - if (!paymentHash || !PaymentHashRegex.test(paymentHash)) return true + if (!paymentHash || !PaymentHashRegex.test(paymentHash)) return null - const recorded = await recordInternalTransferInvoice({ + const inserted = await insertInternalTransferInvoice({ paymentHash, senderWalletId, origin: InternalTransferOrigin.LnSend, }) - if (recorded instanceof Error) return new InternalTransferMarkerError(recorded) - return true + if (inserted instanceof Error) return new InternalTransferMarkerError(inserted) + return { paymentHash, created: inserted.created } +} + +export const markOutgoingBolt11 = async (args: { + paymentRequest: string + senderWalletId: WalletId +}): Promise => { + const marked = await markOutgoing(args) + return marked instanceof Error ? marked : true +} + +/** + * True only when IBEX refused the payment before any money moved: an + * insufficient balance, a corroborated FAILED status, or any other 4xx the + * request was rejected with (no route, expired invoice, rate limit). Never true + * for a pending or unreadable outcome, a transport error or a 5xx (the payment + * may still settle), nor for an already-paid invoice (someone else's payment of + * it settled, and that payer may be another Flash wallet whose marker this is). + */ +export const isDefinitivePaymentRejection = (result: unknown): boolean => { + if (result === PaymentSendStatus.Failure) return true + if (result instanceof InsufficientIbexBalance) return true + if (result instanceof FailedIbexPayment) return true + if (result instanceof CompletedInvoice) return false + if (result instanceof IbexError) { + const code = result.httpCode + return code !== undefined && code >= 400 && code < 500 && code !== 408 + } + return false +} + +/** + * Marks the bolt11 (markOutgoingBolt11), runs `pay`, and withdraws the marker + * again when IBEX rejected the payment outright. Without that, a Flash user's + * failed attempt on a merchant's QR would leave a 30-day marker behind and the + * merchant's splits would not fire when a wallet outside Flash paid the same + * invoice afterwards. + * + * The marker is withdrawn only when this call created it, so a marker written + * by an earlier or concurrent payment of the same hash is left alone. + */ +export const payMarkedBolt11 = async ({ + paymentRequest, + senderWalletId, + pay, +}: { + paymentRequest: string + senderWalletId: WalletId + pay: () => Promise +}): Promise => { + const marker = await markOutgoing({ paymentRequest, senderWalletId }) + if (marker instanceof Error) return marker + + const result = await pay() + + if (marker?.created && isDefinitivePaymentRejection(result)) { + const removed = await deleteInternalTransferInvoice({ + paymentHash: marker.paymentHash, + senderWalletId, + origin: InternalTransferOrigin.LnSend, + }) + if (removed instanceof Error) { + baseLogger.error( + { err: removed, paymentHash: marker.paymentHash, senderWalletId }, + "Payment splits: could not withdraw the marker of a rejected ln-send", + ) + recordExceptionInCurrentSpan({ + error: removed, + level: ErrorLevel.Critical, + fallbackMsg: + "Payment splits: a rejected ln-send left its marker; the payee's splits will not fire if another wallet pays this invoice", + attributes: { "paymentSplits.paymentHash": marker.paymentHash }, + }) + } + } + + return result } /** diff --git a/src/config/schema.ts b/src/config/schema.ts index c19ac7bb4..ba63b75af 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -1149,6 +1149,12 @@ export const configSchema = { // rows until none are left or the budget is spent, so throughput is // not capped by a fixed batch size. executorTimeBudgetMs: { type: "integer", minimum: 1000, default: 45000 }, + // A newly recorded pending execution is not claimable until this long + // after it was recorded. An LNURL send from another Flash user is + // marked internal only after payToLnurl returns, so the payee's receive + // can be recorded first; the grace lets that marker land before the + // executor's pay-time re-check (ENG-653 decision 1). + claimGraceMs: { type: "integer", minimum: 0, default: 120000 }, // How often the loop re-reads recent IBEX receives to record splits // a missed or failed webhook did not. The lookback is derived from // this (see splitReconcileWindowMs). @@ -1165,6 +1171,7 @@ export const configSchema = { minShareCents: 1, loopIntervalMs: 60000, executorTimeBudgetMs: 45000, + claimGraceMs: 120000, reconcileIntervalMs: 900000, cronIntervalMs: 86400000, }, diff --git a/src/config/schema.types.d.ts b/src/config/schema.types.d.ts index f7da0ffc9..d3fad2bf3 100644 --- a/src/config/schema.types.d.ts +++ b/src/config/schema.types.d.ts @@ -134,6 +134,7 @@ type PaymentSplitsConfig = { minShareCents: number loopIntervalMs: number executorTimeBudgetMs: number + claimGraceMs: number reconcileIntervalMs: number cronIntervalMs: number } diff --git a/src/config/yaml.ts b/src/config/yaml.ts index f15818f3d..dfec29107 100644 --- a/src/config/yaml.ts +++ b/src/config/yaml.ts @@ -651,6 +651,7 @@ export const PaymentSplitsConfig = (yamlConfig.paymentSplits ?? { minShareCents: 1, loopIntervalMs: 60000, executorTimeBudgetMs: 45000, + claimGraceMs: 120000, reconcileIntervalMs: 900000, cronIntervalMs: 86400000, }) as PaymentSplitsConfig diff --git a/src/graphql/public/root/mutation/ln-invoice-payment-send.ts b/src/graphql/public/root/mutation/ln-invoice-payment-send.ts index b9ed38fe8..c6b36a044 100644 --- a/src/graphql/public/root/mutation/ln-invoice-payment-send.ts +++ b/src/graphql/public/root/mutation/ln-invoice-payment-send.ts @@ -14,7 +14,7 @@ import Ibex from "@services/ibex/client" import { IbexError, InsufficientIbexBalance } from "@services/ibex/errors" import { paymentSendStatusOrPending } from "@services/ibex/payment-status" import { withPaymentIdempotency } from "@app/payments/idempotency" -import { markOutgoingBolt11 } from "@app/payments/mark-outgoing-invoice" +import { payMarkedBolt11 } from "@app/payments/mark-outgoing-invoice" import { authorizeSend, gateSend, @@ -151,23 +151,24 @@ const LnInvoicePaymentSendMutation = GT.Field< }, execute: async (): Promise => { // ENG-653: the payee may be another Flash wallet, whose splits must - // not fire on a Flash-to-Flash payment. Fails closed. - const marked = await markOutgoingBolt11({ + // not fire on a Flash-to-Flash payment. Fails closed. The marker is + // withdrawn when IBEX rejects the payment outright. + return payMarkedBolt11({ paymentRequest, senderWalletId: walletId, + pay: async () => { + const PayLightningInvoice = await Ibex.payInvoice({ + invoice: paymentRequest as Bolt11, + accountId: walletId, + }) + + if (PayLightningInvoice instanceof IbexError) { + return PayLightningInvoice + } + + return paymentSendStatusOrPending(PayLightningInvoice) + }, }) - if (marked instanceof Error) return marked - - const PayLightningInvoice = await Ibex.payInvoice({ - invoice: paymentRequest as Bolt11, - accountId: walletId, - }) - - if (PayLightningInvoice instanceof IbexError) { - return PayLightningInvoice - } - - return paymentSendStatusOrPending(PayLightningInvoice) }, }) diff --git a/src/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.ts b/src/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.ts index 19b18b7a9..5efe2f267 100644 --- a/src/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.ts +++ b/src/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.ts @@ -17,7 +17,7 @@ import Ibex from "@services/ibex/client" import { IbexError } from "@services/ibex/errors" import { withPaymentIdempotency } from "@app/payments/idempotency" -import { markOutgoingBolt11 } from "@app/payments/mark-outgoing-invoice" +import { payMarkedBolt11 } from "@app/payments/mark-outgoing-invoice" import { authorizeSend } from "@app/payments/authorize-send" import { paymentSendStatusOrPending } from "@services/ibex/payment-status" @@ -144,20 +144,21 @@ const LnNoAmountUsdInvoicePaymentSendMutation = GT.Field< if (usCents instanceof Error) return usCents // ENG-653: the payee may be another Flash wallet, whose splits must - // not fire on a Flash-to-Flash payment. Fails closed. - const marked = await markOutgoingBolt11({ + // not fire on a Flash-to-Flash payment. Fails closed. The marker is + // withdrawn when IBEX rejects the payment outright. + return payMarkedBolt11({ paymentRequest, senderWalletId: routedWalletId, + pay: async () => { + const PayLightningInvoice = await Ibex.payInvoice({ + invoice: paymentRequest as Bolt11, + accountId: routedWalletId, + send: usCents, + }) + if (PayLightningInvoice instanceof IbexError) return PayLightningInvoice + return paymentSendStatusOrPending(PayLightningInvoice) + }, }) - if (marked instanceof Error) return marked - - const PayLightningInvoice = await Ibex.payInvoice({ - invoice: paymentRequest as Bolt11, - accountId: routedWalletId, - send: usCents, - }) - if (PayLightningInvoice instanceof IbexError) return PayLightningInvoice - return paymentSendStatusOrPending(PayLightningInvoice) }, }) diff --git a/src/servers/ibex-webhook-server.ts b/src/servers/ibex-webhook-server.ts index db6834719..a32c1b8a6 100644 --- a/src/servers/ibex-webhook-server.ts +++ b/src/servers/ibex-webhook-server.ts @@ -1,4 +1,7 @@ -import { startPaymentSplitsLoop } from "@app/payment-splits/split-loop" +import { + startPaymentSplitsLoop, + stopPaymentSplitsLoopOnShutdown, +} from "@app/payment-splits/split-loop" import WebookServer from "@services/ibex/webhook-server" import { setupMongoConnection } from "@services/mongodb" import { warnIfDevContext } from "@utils/dev-context" @@ -14,8 +17,11 @@ if (require.main === module) { .then(async () => { WebookServer.start() // ENG-653: split reconcile, executor and stale sweep run here, every - // paymentSplits.loopIntervalMs; the daily galoy cron is too slow. - startPaymentSplitsLoop() + // paymentSplits.loopIntervalMs; the daily galoy cron is too slow. On + // SIGTERM (rollout, kube-monkey) the split send in flight finishes and + // no new one starts, so a deploy does not strand a row in `sending`. + const splitsLoop = startPaymentSplitsLoop() + if (splitsLoop) stopPaymentSplitsLoopOnShutdown(splitsLoop) }) .catch(exitOnBootFailure) } diff --git a/src/services/mongoose/internal-transfer-invoices.ts b/src/services/mongoose/internal-transfer-invoices.ts index fa22dddba..e77d57d63 100644 --- a/src/services/mongoose/internal-transfer-invoices.ts +++ b/src/services/mongoose/internal-transfer-invoices.ts @@ -9,22 +9,24 @@ const isDuplicateKeyError = (error: unknown): boolean => // Typed RepositoryErrors, so every failure maps cleanly through error-map. const toError = (error: unknown): RepositoryError => parseRepositoryError(error) +type InternalTransferInvoiceFields = { + paymentHash: string + senderWalletId?: string + recipientWalletId?: string + origin: InternalTransferOrigin +} + /** - * Marks a payment hash as an invoice Flash minted for one of its own transfers - * (ENG-653), so a receive on it never triggers the recipient's payment splits. - * Recording the same hash twice is a no-op. + * Writes the marker and says whether this call created it (`created: false` + * when the hash was already marked). A caller that may later withdraw its + * marker (deleteInternalTransferInvoice) must only do so when it created it. */ -export const recordInternalTransferInvoice = async ({ +export const insertInternalTransferInvoice = async ({ paymentHash, senderWalletId, recipientWalletId, origin, -}: { - paymentHash: string - senderWalletId?: string - recipientWalletId?: string - origin: InternalTransferOrigin -}): Promise => { +}: InternalTransferInvoiceFields): Promise<{ created: boolean } | RepositoryError> => { try { await InternalTransferInvoice.create({ paymentHash: paymentHash.toLowerCase(), @@ -33,9 +35,47 @@ export const recordInternalTransferInvoice = async ({ origin, createdAt: new Date(), }) - return true + return { created: true } + } catch (error) { + if (isDuplicateKeyError(error)) return { created: false } + return toError(error) + } +} + +/** + * Marks a payment hash as an invoice Flash minted for one of its own transfers + * (ENG-653), so a receive on it never triggers the recipient's payment splits. + * Recording the same hash twice is a no-op. + */ +export const recordInternalTransferInvoice = async ( + fields: InternalTransferInvoiceFields, +): Promise => { + const inserted = await insertInternalTransferInvoice(fields) + return inserted instanceof Error ? inserted : true +} + +/** + * Withdraws a marker this process wrote for a payment that was then rejected + * outright. Matches on origin and sender as well as the hash, so it can never + * remove a marker some other transfer wrote. Returns whether a row was removed. + */ +export const deleteInternalTransferInvoice = async ({ + paymentHash, + senderWalletId, + origin, +}: { + paymentHash: string + senderWalletId: string + origin: InternalTransferOrigin +}): Promise => { + try { + const result = await InternalTransferInvoice.deleteOne({ + paymentHash: { $eq: paymentHash.toLowerCase() }, + senderWalletId: { $eq: senderWalletId }, + origin: { $eq: origin }, + }) + return result.deletedCount > 0 } catch (error) { - if (isDuplicateKeyError(error)) return true return toError(error) } } diff --git a/src/services/mongoose/payment-split-executions.ts b/src/services/mongoose/payment-split-executions.ts index 5b396f792..be4e6d494 100644 --- a/src/services/mongoose/payment-split-executions.ts +++ b/src/services/mongoose/payment-split-executions.ts @@ -70,14 +70,15 @@ export const PaymentSplitExecutionsRepository = () => { triggerAmountMinor: string shareCents: number status: "pending" | "skipped_min" + nextAttemptAt?: Date now?: Date }): Promise<{ inserted: true; id: string } | { inserted: false } | RepositoryError> => { - const { now = new Date(), ...fields } = data + const { now = new Date(), nextAttemptAt = now, ...fields } = data try { const doc = await PaymentSplitExecutionModel.create({ ...fields, attempts: 0, - nextAttemptAt: now, + nextAttemptAt, createdAt: now, updatedAt: now, }) diff --git a/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts b/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts index 325cf4ece..fa7aa98ce 100644 --- a/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts +++ b/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts @@ -94,6 +94,20 @@ describe("payment split repositories (ENG-653)", () => { expect(claimed[0]).toMatchObject({ status: "sending" }) }) + it("does not claim a row before its first nextAttemptAt", async () => { + const repo = PaymentSplitExecutionsRepository() + const now = new Date() + const claimableAt = new Date(now.getTime() + 120_000) + await repo.insert({ + ...executionFields("f".repeat(64), "rule-1"), + nextAttemptAt: claimableAt, + now, + }) + + expect(await repo.claimNextDue(now)).toBeNull() + expect(await repo.claimNextDue(claimableAt)).toMatchObject({ status: "sending" }) + }) + it("has a TTL index on internal-transfer markers that outlives the reconcile lookback", async () => { const indexes = await InternalTransferInvoice.collection.indexes() const ttl = indexes.find((index) => index.expireAfterSeconds !== undefined) diff --git a/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts index c59da40d3..87584dbbe 100644 --- a/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts +++ b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts @@ -331,6 +331,24 @@ describe("executePendingSplits", () => { expect(await executePendingSplits({ now: T0 })).toMatchObject({ claimed: 2 }) }) + it("finishes the row in flight but claims no new one once stopping", async () => { + await seedMany(3) + let stopping = false + mockSend.mockImplementation(async () => { + // SIGTERM lands while the first split is being sent. + stopping = true + return PaymentSendStatus.Success + }) + + expect( + await executePendingSplits({ now: T0, shouldStop: () => stopping }), + ).toMatchObject({ claimed: 1, sent: 1 }) + const statuses = [...mockStore.rows.values()].map((r) => r.status) + expect(statuses.filter((s) => s === "sent")).toHaveLength(1) + expect(statuses.filter((s) => s === "pending")).toHaveLength(2) + expect(statuses).not.toContain("sending") + }) + describe("lock errors", () => { // The send reached the payment: the authorize hook (which runs // immediately before Ibex.payInvoice) passed, then the lock layer erred. diff --git a/test/flash/unit/app/payment-splits/fake-executions-store.ts b/test/flash/unit/app/payment-splits/fake-executions-store.ts index 8852a2aed..cc7f4bb1a 100644 --- a/test/flash/unit/app/payment-splits/fake-executions-store.ts +++ b/test/flash/unit/app/payment-splits/fake-executions-store.ts @@ -32,9 +32,10 @@ export const createFakeExecutionsStore = () => { triggerAmountMinor: string shareCents: number status: string + nextAttemptAt?: Date now?: Date }) => { - const { now = new Date(), ...fields } = data + const { now = new Date(), nextAttemptAt = now, ...fields } = data const dup = [...rows.values()].find( (r) => r.triggerKey === fields.triggerKey && r.ruleId === fields.ruleId, ) @@ -44,7 +45,7 @@ export const createFakeExecutionsStore = () => { ...fields, id, attempts: 0, - nextAttemptAt: now, + nextAttemptAt, createdAt: now, updatedAt: now, }) diff --git a/test/flash/unit/app/payment-splits/record-split-executions.spec.ts b/test/flash/unit/app/payment-splits/record-split-executions.spec.ts index 08a99a37a..8e956886c 100644 --- a/test/flash/unit/app/payment-splits/record-split-executions.spec.ts +++ b/test/flash/unit/app/payment-splits/record-split-executions.spec.ts @@ -92,6 +92,24 @@ describe("recordSplitExecutions", () => { ]) }) + it("holds a freshly recorded row back from the executor for claimGraceMs", async () => { + // An LNURL send from another Flash user is marked internal only after + // payToLnurl returns. A row claimable at once could be paid by a loop tick + // landing before that marker is written (ENG-653 decision 1). + const now = new Date("2026-10-11T12:00:00Z") + const grace = PaymentSplitsConfig.claimGraceMs + expect(grace).toBeGreaterThanOrEqual(60_000) + + await record({ now }) + + expect(await mockStore.repo.claimNextDue(now)).toBeNull() + expect( + await mockStore.repo.claimNextDue(new Date(now.getTime() + grace - 1)), + ).toBeNull() + const claimed = await mockStore.repo.claimNextDue(new Date(now.getTime() + grace)) + expect(claimed).toMatchObject({ status: "sending", triggerKey: HASH }) + }) + it("records one row per rule however many times the same trigger is replayed", async () => { for (let i = 0; i < 5; i++) await record() diff --git a/test/flash/unit/app/payment-splits/split-loop.spec.ts b/test/flash/unit/app/payment-splits/split-loop.spec.ts index 964e4c3ff..65673188e 100644 --- a/test/flash/unit/app/payment-splits/split-loop.spec.ts +++ b/test/flash/unit/app/payment-splits/split-loop.spec.ts @@ -20,6 +20,7 @@ import { PaymentSplitsConfig } from "@config" import { runPaymentSplitsTick, startPaymentSplitsLoop, + stopPaymentSplitsLoopOnShutdown, } from "@app/payment-splits/split-loop" import { splitRetryBackoffMs } from "@domain/payment-splits" @@ -54,7 +55,10 @@ describe("runPaymentSplitsTick", () => { Math.floor((PaymentSplitsConfig.reconcileIntervalMs * 0.9) / 1000), "NX", ) - expect(mockExecute).toHaveBeenCalledWith({ now: NOW }) + expect(mockExecute).toHaveBeenCalledWith({ + now: NOW, + shouldStop: expect.any(Function), + }) expect(mockSweep).toHaveBeenCalledWith({ now: NOW }) }) @@ -107,15 +111,122 @@ describe("startPaymentSplitsLoop", () => { ) }) - it("starts a timer when enabled", () => { + it("starts a timer when enabled", async () => { jest.useFakeTimers() try { - const timer = startPaymentSplitsLoop() - expect(timer).toBeDefined() - clearInterval(timer) + const loop = startPaymentSplitsLoop() + expect(loop).toBeDefined() + await loop?.stop() } finally { jest.clearAllTimers() jest.useRealTimers() } }) }) + +describe("startPaymentSplitsLoop stop()", () => { + // Lets the tick's awaits (reconcile lease, reconcile) run under fake timers. + const flushMicrotasks = async () => { + for (let i = 0; i < 20; i++) await Promise.resolve() + } + const deferred = () => { + let resolve!: (value: unknown) => void + const promise = new Promise((r) => { + resolve = r + }) + return { promise, resolve } + } + + afterEach(() => { + jest.clearAllTimers() + jest.useRealTimers() + }) + + it("waits for the tick in flight, which claims no new row once stopping", async () => { + jest.useFakeTimers() + const send = deferred() + const claimsAfterStop: boolean[] = [] + // Stands in for the executor: one send in flight, then a claim check of the + // kind executePendingSplits makes before every claimNextDue. + mockExecute.mockImplementation( + async ({ shouldStop }: { shouldStop: () => boolean }) => { + await send.promise + claimsAfterStop.push(!shouldStop()) + return { claimed: 1 } + }, + ) + + const loop = startPaymentSplitsLoop()! + jest.advanceTimersByTime(30_000) + await flushMicrotasks() + expect(mockExecute).toHaveBeenCalledTimes(1) + + let stopped = false + const stopping = loop.stop().then(() => { + stopped = true + }) + await flushMicrotasks() + expect(stopped).toBe(false) + + send.resolve(undefined) + await stopping + + expect(stopped).toBe(true) + // The executor saw the stop flag after its send, so it claims nothing more. + expect(claimsAfterStop).toEqual([false]) + // The sweep is skipped once stopping. + expect(mockSweep).not.toHaveBeenCalled() + }) + + it("starts no tick after stop()", async () => { + jest.useFakeTimers() + const loop = startPaymentSplitsLoop()! + + await loop.stop() + jest.advanceTimersByTime(30_000 + PaymentSplitsConfig.loopIntervalMs * 3) + await flushMicrotasks() + + expect(mockExecute).not.toHaveBeenCalled() + expect(mockReconcile).not.toHaveBeenCalled() + }) +}) + +describe("runPaymentSplitsTick shouldStop", () => { + it("does nothing once stopping", async () => { + await runPaymentSplitsTick({ now: NOW, shouldStop: () => true }) + + expect(mockReconcile).not.toHaveBeenCalled() + expect(mockExecute).not.toHaveBeenCalled() + expect(mockSweep).not.toHaveBeenCalled() + }) +}) + +describe("stopPaymentSplitsLoopOnShutdown", () => { + it("stops the loop on SIGTERM, then re-raises the signal with its listeners gone", async () => { + const before = process.listenerCount("SIGTERM") + let finishStop!: () => void + const stop = jest.fn( + () => + new Promise((resolve) => { + finishStop = resolve + }), + ) + const kill = jest.spyOn(process, "kill").mockImplementation(() => true) + try { + stopPaymentSplitsLoopOnShutdown({ stop }) + expect(process.listenerCount("SIGTERM")).toBe(before + 1) + + process.emit("SIGTERM", "SIGTERM") + expect(stop).toHaveBeenCalledTimes(1) + expect(process.listenerCount("SIGTERM")).toBe(before) + await Promise.resolve() + expect(kill).not.toHaveBeenCalled() + + finishStop() + await new Promise((r) => setImmediate(r)) + expect(kill).toHaveBeenCalledWith(process.pid, "SIGTERM") + } finally { + kill.mockRestore() + } + }) +}) diff --git a/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts b/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts index ccd2af3e7..238e7fa9d 100644 --- a/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts +++ b/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts @@ -1,7 +1,13 @@ const mockRecordInternalTransferInvoice = jest.fn() +const mockInsertInternalTransferInvoice = jest.fn() +const mockDeleteInternalTransferInvoice = jest.fn() jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ recordInternalTransferInvoice: (...args: unknown[]) => mockRecordInternalTransferInvoice(...args), + insertInternalTransferInvoice: (...args: unknown[]) => + mockInsertInternalTransferInvoice(...args), + deleteInternalTransferInvoice: (...args: unknown[]) => + mockDeleteInternalTransferInvoice(...args), })) jest.mock("@services/tracing", () => ({ @@ -9,11 +15,22 @@ jest.mock("@services/tracing", () => ({ })) import { + isDefinitivePaymentRejection, markOutgoingBolt11, markPaidLnurlHash, + payMarkedBolt11, } from "@app/payments/mark-outgoing-invoice" +import { PaymentSendStatus } from "@domain/bitcoin/lightning" import { RepositoryError } from "@domain/errors" import { InternalTransferMarkerError } from "@domain/payment-splits" +import { ErrorLevel } from "@domain/shared" +import { + CompletedInvoice, + FailedIbexPayment, + IbexError, + InsufficientIbexBalance, + UnconfirmedIbexPayment, +} from "@services/ibex/errors" // BOLT11 spec test vector; its payment hash is 0001…0102. const BOLT11 = @@ -24,6 +41,8 @@ const SENDER = "sender-wallet" as WalletId beforeEach(() => { jest.clearAllMocks() mockRecordInternalTransferInvoice.mockResolvedValue(true) + mockInsertInternalTransferInvoice.mockResolvedValue({ created: true }) + mockDeleteInternalTransferInvoice.mockResolvedValue(true) }) describe("markOutgoingBolt11", () => { @@ -31,7 +50,7 @@ describe("markOutgoingBolt11", () => { expect( await markOutgoingBolt11({ paymentRequest: BOLT11, senderWalletId: SENDER }), ).toBe(true) - expect(mockRecordInternalTransferInvoice).toHaveBeenCalledWith({ + expect(mockInsertInternalTransferInvoice).toHaveBeenCalledWith({ paymentHash: HASH, senderWalletId: SENDER, origin: "ln-send", @@ -39,7 +58,7 @@ describe("markOutgoingBolt11", () => { }) it("fails closed when the marker cannot be written", async () => { - mockRecordInternalTransferInvoice.mockResolvedValue(new RepositoryError("mongo")) + mockInsertInternalTransferInvoice.mockResolvedValue(new RepositoryError("mongo")) expect( await markOutgoingBolt11({ paymentRequest: BOLT11, senderWalletId: SENDER }), @@ -53,7 +72,7 @@ describe("markOutgoingBolt11", () => { senderWalletId: SENDER, }), ).toBe(true) - expect(mockRecordInternalTransferInvoice).not.toHaveBeenCalled() + expect(mockInsertInternalTransferInvoice).not.toHaveBeenCalled() }) }) @@ -84,3 +103,111 @@ describe("markPaidLnurlHash", () => { ).resolves.toBeUndefined() }) }) + +const httpError = (httpCode: number) => { + const err = new IbexError(new Error(`IBEX ${httpCode}`)) + Object.defineProperty(err, "httpCode", { value: httpCode }) + return err +} + +describe("isDefinitivePaymentRejection", () => { + it.each([ + [ + "insufficient balance", + new InsufficientIbexBalance(new Error("400"), ErrorLevel.Info), + ], + ["a corroborated FAILED response", new FailedIbexPayment("failed")], + ["a FAILED status", PaymentSendStatus.Failure], + ["an IBEX 400 (no route, expired)", httpError(400)], + ["an IBEX 429", httpError(429)], + ])("is true for %s", (_label, result) => { + expect(isDefinitivePaymentRejection(result)).toBe(true) + }) + + it.each([ + ["success", PaymentSendStatus.Success], + ["pending", PaymentSendStatus.Pending], + ["an unreadable response", new UnconfirmedIbexPayment("unreadable")], + ["an already-paid invoice", new CompletedInvoice(new Error("400"))], + ["an IBEX 5xx", httpError(502)], + ["a request timeout", httpError(408)], + ["a transport error with no status", new IbexError(new Error("ECONNRESET"))], + ["a non-IBEX error", new Error("lock")], + ])("is false for %s", (_label, result) => { + expect(isDefinitivePaymentRejection(result)).toBe(false) + }) +}) + +describe("payMarkedBolt11", () => { + const run = (result: unknown) => + payMarkedBolt11({ + paymentRequest: BOLT11, + senderWalletId: SENDER, + pay: async () => result, + }) + + it("marks before paying and keeps the marker when the payment succeeds", async () => { + const order: string[] = [] + mockInsertInternalTransferInvoice.mockImplementation(async () => { + order.push("mark") + return { created: true } + }) + + const result = await payMarkedBolt11({ + paymentRequest: BOLT11, + senderWalletId: SENDER, + pay: async () => { + order.push("pay") + return PaymentSendStatus.Success + }, + }) + + expect(result).toBe(PaymentSendStatus.Success) + expect(order).toEqual(["mark", "pay"]) + expect(mockDeleteInternalTransferInvoice).not.toHaveBeenCalled() + }) + + it("keeps the marker while the payment is pending", async () => { + expect(await run(PaymentSendStatus.Pending)).toBe(PaymentSendStatus.Pending) + expect(mockDeleteInternalTransferInvoice).not.toHaveBeenCalled() + }) + + it("withdraws its own marker when IBEX rejects the payment outright", async () => { + const rejected = new InsufficientIbexBalance(new Error("400"), ErrorLevel.Info) + + expect(await run(rejected)).toBe(rejected) + expect(mockDeleteInternalTransferInvoice).toHaveBeenCalledWith({ + paymentHash: HASH, + senderWalletId: SENDER, + origin: "ln-send", + }) + }) + + it("leaves a marker it did not create, even on a rejection", async () => { + mockInsertInternalTransferInvoice.mockResolvedValue({ created: false }) + + await run(PaymentSendStatus.Failure) + + expect(mockDeleteInternalTransferInvoice).not.toHaveBeenCalled() + }) + + it("pays nothing when the marker cannot be written", async () => { + mockInsertInternalTransferInvoice.mockResolvedValue(new RepositoryError("mongo")) + const pay = jest.fn() + + const result = await payMarkedBolt11({ + paymentRequest: BOLT11, + senderWalletId: SENDER, + pay, + }) + + expect(result).toBeInstanceOf(InternalTransferMarkerError) + expect(pay).not.toHaveBeenCalled() + }) + + it("still returns the payment result when the withdrawal fails", async () => { + mockDeleteInternalTransferInvoice.mockResolvedValue(new RepositoryError("mongo")) + + expect(await run(PaymentSendStatus.Failure)).toBe(PaymentSendStatus.Failure) + }) +}) diff --git a/test/flash/unit/graphql/ln-invoice-payment-send.spec.ts b/test/flash/unit/graphql/ln-invoice-payment-send.spec.ts index cf3218f2f..80db48972 100644 --- a/test/flash/unit/graphql/ln-invoice-payment-send.spec.ts +++ b/test/flash/unit/graphql/ln-invoice-payment-send.spec.ts @@ -86,9 +86,19 @@ jest.mock("@app/payments/idempotency", () => ({ // bolt11's hash before paying so the payee's splits never fire. Default-allow. const mockMarkOutgoingBolt11 = jest.fn() jest.mock("@app/payments/mark-outgoing-invoice", () => ({ - markOutgoingBolt11: async (args: unknown) => { + // Mark (via the mock), then pay. Marker withdrawal on a rejected payment is + // covered against the real helper in ln-send-marker-withdrawal.spec.ts. + payMarkedBolt11: async ({ + pay, + ...args + }: { + paymentRequest: string + senderWalletId: string + pay: () => Promise + }) => { const result = await mockMarkOutgoingBolt11(args) - return result === undefined ? true : result + if (result instanceof Error) return result + return pay() }, })) diff --git a/test/flash/unit/graphql/ln-send-marker-withdrawal.spec.ts b/test/flash/unit/graphql/ln-send-marker-withdrawal.spec.ts new file mode 100644 index 000000000..861cb3826 --- /dev/null +++ b/test/flash/unit/graphql/ln-send-marker-withdrawal.spec.ts @@ -0,0 +1,224 @@ +// ENG-653 review: a Flash user's rejected attempt to pay a merchant's bolt11 +// must not leave the ln-send marker behind. If it did, the merchant's splits +// would be skipped as Flash-to-Flash when a wallet outside Flash paid the same +// invoice afterwards. This spec runs both bolt11 send resolvers against the real +// marker helper and the real split recorder, with the marker collection and +// the split stores held in memory. + +const markers = new Map() +jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ + insertInternalTransferInvoice: async (args: { + paymentHash: string + senderWalletId?: string + origin: string + }) => { + const key = args.paymentHash.toLowerCase() + if (markers.has(key)) return { created: false } + markers.set(key, { senderWalletId: args.senderWalletId, origin: args.origin }) + return { created: true } + }, + recordInternalTransferInvoice: async () => true, + deleteInternalTransferInvoice: async (args: { + paymentHash: string + senderWalletId: string + origin: string + }) => { + const key = args.paymentHash.toLowerCase() + const found = markers.get(key) + if (!found) return false + if (found.senderWalletId !== args.senderWalletId || found.origin !== args.origin) { + return false + } + return markers.delete(key) + }, + isInternalTransferInvoice: async (paymentHash: string) => + markers.has(paymentHash.toLowerCase()), +})) + +const mockListRules = jest.fn() +jest.mock("@services/mongoose/payment-split-rules", () => ({ + PaymentSplitRulesRepository: jest.fn(() => ({ + listByWalletId: (...args: unknown[]) => mockListRules(...args), + })), +})) +jest.mock("@services/mongoose/payment-split-executions", () => ({ + PaymentSplitExecutionsRepository: jest.fn(() => mockStore.repo), +})) + +jest.mock("@app/payments/authorize-send", () => ({ + authorizeSend: async () => true, + gateSend: async () => true, + SendRejectionReasons: { undecodableInvoice: "undecodable-invoice" }, +})) + +jest.mock("@app/payments/idempotency", () => ({ + withPaymentIdempotency: async ({ + authorize, + execute, + }: { + authorize?: () => Promise + execute: () => Promise + }) => { + const authorized = await authorize?.() + if (authorized instanceof Error) return authorized + return execute() + }, +})) + +jest.mock("@app/cash-wallet-cutover", () => ({ + resolveCashWalletMutationWalletIdForAccount: async ({ + walletId, + }: { + walletId: string + }) => walletId, +})) + +jest.mock("@app/wallets", () => ({ + usdWalletAmountFromWalletId: async () => ({ toIbex: () => 50 }), +})) + +const mockPayInvoice = jest.fn() +jest.mock("@services/ibex/client", () => ({ + __esModule: true, + default: { payInvoice: (...args: unknown[]) => mockPayInvoice(...args) }, +})) + +jest.mock("@services/tracing", () => ({ + addAttributesToCurrentSpan: jest.fn(), + addEventToCurrentSpan: jest.fn(), + recordExceptionInCurrentSpan: jest.fn(), +})) + +import { PaymentSplitsConfig } from "@config" +import { recordSplitExecutions } from "@app/payment-splits/record-split-executions" +import { ErrorLevel, WalletCurrency } from "@domain/shared" +import LnInvoicePaymentSendMutation from "@graphql/public/root/mutation/ln-invoice-payment-send" +import LnNoAmountUsdInvoicePaymentSendMutation from "@graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send" +import { IbexError, InsufficientIbexBalance } from "@services/ibex/errors" + +import { createFakeExecutionsStore } from "../app/payment-splits/fake-executions-store" + +const mockStore = createFakeExecutionsStore() + +// BOLT11 spec test vector; its payment hash is 0001…0102. +const BOLT11 = + "lnbc20u1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kxqrrsssp5m6kmam774klwlh4dhmhaatd7al02m0h0m6kmam774klwlh4dhmhs9qypqqqcqpf3cwux5979a8j28d4ydwahx00saa68wq3az7v9jdgzkghtxnkf3z5t7q5suyq2dl9tqwsap8j0wptc82cpyvey9gf6zyylzrm60qtcqsq7egtsq" +const HASH = "0001020304050607080900010203040506070809000102030405060708090102" +const PAYER_WALLET = "flash-payer-wallet" +const MERCHANT_WALLET = "merchant-wallet" + +type Resolve = ( + source: null, + args: { input: Record }, + ctx: { domainAccount: Record }, +) => Promise<{ status?: string; errors: { message: string }[] }> + +const resolvers: [string, () => ReturnType][] = [ + [ + "lnInvoicePaymentSend", + () => + (LnInvoicePaymentSendMutation.resolve as unknown as Resolve)( + null, + { input: { walletId: PAYER_WALLET, paymentRequest: BOLT11 } }, + { domainAccount: { id: "payer-account" } }, + ), + ], + [ + "lnNoAmountUsdInvoicePaymentSend", + () => + (LnNoAmountUsdInvoicePaymentSendMutation.resolve as unknown as Resolve)( + null, + { input: { walletId: PAYER_WALLET, paymentRequest: BOLT11, amount: 5000 } }, + { domainAccount: { id: "payer-account" } }, + ), + ], +] + +// The merchant's wallet receives the invoice later, from a wallet outside +// Flash (the IBEX on-receive webhook calls the recorder with the hash). +const merchantReceives = () => + recordSplitExecutions({ + walletId: MERCHANT_WALLET, + triggerKey: HASH, + paymentHash: HASH, + amount: 50, + currency: WalletCurrency.Usdt, + }) + +const originalEnabled = PaymentSplitsConfig.enabled + +beforeEach(() => { + jest.clearAllMocks() + markers.clear() + mockStore.rows.clear() + PaymentSplitsConfig.enabled = true + mockListRules.mockResolvedValue([ + { + id: "rule-1", + walletId: MERCHANT_WALLET, + destinationWalletId: "partner-wallet", + percent: 10, + enabled: true, + createdAt: new Date("2026-01-01T00:00:00Z"), + }, + ]) +}) + +afterAll(() => { + PaymentSplitsConfig.enabled = originalEnabled +}) + +describe.each(resolvers)("%s ln-send marker (ENG-653)", (_name, pay) => { + it("withdraws the marker when IBEX rejects for insufficient balance, so a later external payment records splits", async () => { + mockPayInvoice.mockResolvedValue( + new InsufficientIbexBalance(new Error("Bad Request"), ErrorLevel.Info), + ) + + const result = await pay() + + expect(result.status).toBe("failed") + expect(markers.has(HASH)).toBe(false) + expect(await merchantReceives()).toEqual({ + recorded: 1, + duplicates: 0, + skippedMin: 0, + }) + }) + + it("withdraws the marker on an IBEX 4xx rejection (no route, expired)", async () => { + const rejected = new IbexError(new Error("no route")) + Object.defineProperty(rejected, "httpCode", { value: 400 }) + mockPayInvoice.mockResolvedValue(rejected) + + await pay() + + expect(markers.has(HASH)).toBe(false) + }) + + it("keeps the marker when the payment settles, so the payee's splits do not fire", async () => { + mockPayInvoice.mockResolvedValue({ transaction: { payment: { status: { id: 2 } } } }) + + expect((await pay()).status).toBe("success") + expect(markers.has(HASH)).toBe(true) + expect(await merchantReceives()).toEqual({ + recorded: 0, + duplicates: 0, + skippedMin: 0, + }) + }) + + it("keeps the marker when the outcome is unknown (transport error)", async () => { + mockPayInvoice.mockResolvedValue(new IbexError(new Error("ECONNRESET"))) + + await pay() + + expect(markers.has(HASH)).toBe(true) + }) + + it("keeps the marker while the payment is in flight", async () => { + mockPayInvoice.mockResolvedValue({ transaction: { payment: { status: { id: 1 } } } }) + + expect((await pay()).status).toBe("pending") + expect(markers.has(HASH)).toBe(true) + }) +}) diff --git a/test/flash/unit/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.spec.ts b/test/flash/unit/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.spec.ts index 104d566cb..4bb18cace 100644 --- a/test/flash/unit/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.spec.ts +++ b/test/flash/unit/graphql/public/root/mutation/ln-noamount-usd-invoice-payment-send.spec.ts @@ -55,9 +55,19 @@ jest.mock("@app/wallets", () => ({ // bolt11's hash before paying so the payee's splits never fire. Default-allow. const mockMarkOutgoingBolt11 = jest.fn() jest.mock("@app/payments/mark-outgoing-invoice", () => ({ - markOutgoingBolt11: async (args: unknown) => { + // Mark (via the mock), then pay. Marker withdrawal on a rejected payment is + // covered against the real helper in ln-send-marker-withdrawal.spec.ts. + payMarkedBolt11: async ({ + pay, + ...args + }: { + paymentRequest: string + senderWalletId: string + pay: () => Promise + }) => { const result = await mockMarkOutgoingBolt11(args) - return result === undefined ? true : result + if (result instanceof Error) return result + return pay() }, })) From 988150e133c425c67e367f309923a30f741156a4 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 23:32:25 -0700 Subject: [PATCH 11/13] fix(payment-splits): one internal-transfer marker per sender The InternalTransferInvoice marker was unique on paymentHash alone. When two Flash wallets paid the same bolt11 concurrently, the second insert was a duplicate and owned no row; if the first payment was then rejected by IBEX, its withdrawal deleted the only marker and the second (successful) Flash-to-Flash payment could trigger the payee's splits (decision 1). The unique index is now (paymentHash, senderWalletId, origin). Each payer owns its own row and deleteInternalTransferInvoice still removes only the caller's row; isInternalTransferInvoice stays a hash-only lookup, so the hash is internal while any row exists. No reader decides policy from a single row's sender or origin. The collection is new in this PR and unreleased; graphql-main-server boots with syncIndexes(), which drops the old paymentHash_1 index wherever the branch may already have run, so no migration is needed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G4SaLJF5SWq6sjBEaWJFow --- src/app/payments/mark-outgoing-invoice.ts | 6 +- .../mongoose/internal-transfer-invoices.ts | 17 +++-- src/services/mongoose/schema.ts | 13 +++- .../payment-splits-repositories.spec.ts | 58 ++++++++++++++++ .../payments/mark-outgoing-invoice.spec.ts | 66 +++++++++++++++++++ .../graphql/ln-send-marker-withdrawal.spec.ts | 37 +++++------ 6 files changed, 170 insertions(+), 27 deletions(-) diff --git a/src/app/payments/mark-outgoing-invoice.ts b/src/app/payments/mark-outgoing-invoice.ts index 50b9df7e5..dd5f75068 100644 --- a/src/app/payments/mark-outgoing-invoice.ts +++ b/src/app/payments/mark-outgoing-invoice.ts @@ -91,8 +91,10 @@ export const isDefinitivePaymentRejection = (result: unknown): boolean => { * merchant's splits would not fire when a wallet outside Flash paid the same * invoice afterwards. * - * The marker is withdrawn only when this call created it, so a marker written - * by an earlier or concurrent payment of the same hash is left alone. + * The marker is withdrawn only when this call created it, and only this + * sender's row is removed: markers are one row per (hash, sender, origin), so + * a concurrent payment of the same bolt11 by another Flash wallet keeps its + * own row and the hash stays internal. */ export const payMarkedBolt11 = async ({ paymentRequest, diff --git a/src/services/mongoose/internal-transfer-invoices.ts b/src/services/mongoose/internal-transfer-invoices.ts index e77d57d63..4c4239276 100644 --- a/src/services/mongoose/internal-transfer-invoices.ts +++ b/src/services/mongoose/internal-transfer-invoices.ts @@ -17,8 +17,10 @@ type InternalTransferInvoiceFields = { } /** - * Writes the marker and says whether this call created it (`created: false` - * when the hash was already marked). A caller that may later withdraw its + * Writes this sender's marker for the hash and says whether this call created + * it (`created: false` when the same hash, sender and origin was already + * marked). Markers are one row per (hash, sender, origin): another wallet + * paying the same hash gets its own row. A caller that may later withdraw its * marker (deleteInternalTransferInvoice) must only do so when it created it. */ export const insertInternalTransferInvoice = async ({ @@ -45,7 +47,7 @@ export const insertInternalTransferInvoice = async ({ /** * Marks a payment hash as an invoice Flash minted for one of its own transfers * (ENG-653), so a receive on it never triggers the recipient's payment splits. - * Recording the same hash twice is a no-op. + * Recording the same hash, sender and origin twice is a no-op. */ export const recordInternalTransferInvoice = async ( fields: InternalTransferInvoiceFields, @@ -56,8 +58,9 @@ export const recordInternalTransferInvoice = async ( /** * Withdraws a marker this process wrote for a payment that was then rejected - * outright. Matches on origin and sender as well as the hash, so it can never - * remove a marker some other transfer wrote. Returns whether a row was removed. + * outright. Matches on origin and sender as well as the hash, so it only ever + * removes the caller's own row: another wallet's marker for the same hash + * stays, and the hash stays internal. Returns whether a row was removed. */ export const deleteInternalTransferInvoice = async ({ paymentHash, @@ -80,6 +83,10 @@ export const deleteInternalTransferInvoice = async ({ } } +/** + * True while ANY marker exists for the hash, whoever wrote it and whatever its + * origin: one Flash payer is enough to make the receive Flash-to-Flash. + */ export const isInternalTransferInvoice = async ( paymentHash: string, ): Promise => { diff --git a/src/services/mongoose/schema.ts b/src/services/mongoose/schema.ts index fc1da3a43..6a62709e4 100644 --- a/src/services/mongoose/schema.ts +++ b/src/services/mongoose/schema.ts @@ -961,7 +961,7 @@ export const PaymentSplitExecution = mongoose.model( export const INTERNAL_TRANSFER_INVOICE_TTL_SECONDS = 30 * 24 * 60 * 60 const InternalTransferInvoiceSchema = new Schema({ - paymentHash: { type: String, required: true, unique: true }, + paymentHash: { type: String, required: true }, senderWalletId: { type: String }, // Unknown for ln-send / lnurl-send: Flash only knows the payer. recipientWalletId: { type: String }, @@ -973,6 +973,17 @@ const InternalTransferInvoiceSchema = new Schema({ createdAt: { type: Date, required: true, default: Date.now }, }) +// One marker per (hash, sender, origin), not per hash: two Flash wallets may +// pay the same bolt11, and each must own a row it alone can withdraw when its +// own payment is rejected. A hash is internal while ANY row for it exists +// (isInternalTransferInvoice), served by this index's paymentHash prefix. +// Rows without a sender (cutover) index senderWalletId as null, so a repeat +// of the same hash and origin still collapses to one row. +InternalTransferInvoiceSchema.index( + { paymentHash: 1, senderWalletId: 1, origin: 1 }, + { unique: true }, +) + InternalTransferInvoiceSchema.index( { createdAt: 1 }, { expireAfterSeconds: INTERNAL_TRANSFER_INVOICE_TTL_SECONDS }, diff --git a/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts b/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts index fa7aa98ce..254ffb8a9 100644 --- a/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts +++ b/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts @@ -5,6 +5,8 @@ import { PaymentSplitRule, } from "@services/mongoose/schema" import { + deleteInternalTransferInvoice, + insertInternalTransferInvoice, isInternalTransferInvoice, recordInternalTransferInvoice, } from "@services/mongoose/internal-transfer-invoices" @@ -134,4 +136,60 @@ describe("payment split repositories (ENG-653)", () => { expect(await isInternalTransferInvoice("cd".repeat(32))).toBe(false) expect(await InternalTransferInvoice.countDocuments({})).toBe(1) }) + + it("keeps one marker per (hash, sender, origin), not one per hash", async () => { + const hash = "ef".repeat(32) + const lnSend = (senderWalletId: string) => ({ + paymentHash: hash, + senderWalletId, + origin: "ln-send" as const, + }) + + // Same sender and origin twice: one row, the second insert is a no-op. + expect(await insertInternalTransferInvoice(lnSend("sender-a"))).toEqual({ + created: true, + }) + expect(await insertInternalTransferInvoice(lnSend("sender-a"))).toEqual({ + created: false, + }) + expect(await InternalTransferInvoice.countDocuments({ paymentHash: hash })).toBe(1) + + // A second sender of the same hash owns a row of its own. + expect(await insertInternalTransferInvoice(lnSend("sender-b"))).toEqual({ + created: true, + }) + expect(await InternalTransferInvoice.countDocuments({ paymentHash: hash })).toBe(2) + + // The same sender under another origin is a distinct row too. + expect( + await insertInternalTransferInvoice({ + ...lnSend("sender-a"), + origin: "lnurl-send", + }), + ).toEqual({ created: true }) + expect(await InternalTransferInvoice.countDocuments({ paymentHash: hash })).toBe(3) + + // Withdrawing A's ln-send row leaves B's, so the hash stays internal. + expect(await deleteInternalTransferInvoice(lnSend("sender-a"))).toBe(true) + expect(await deleteInternalTransferInvoice(lnSend("sender-a"))).toBe(false) + expect(await isInternalTransferInvoice(hash)).toBe(true) + expect(await InternalTransferInvoice.countDocuments({ paymentHash: hash })).toBe(2) + + const indexes = await InternalTransferInvoice.collection.indexes() + expect(indexes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: { paymentHash: 1, senderWalletId: 1, origin: 1 }, + unique: true, + }), + ]), + ) + expect( + indexes.find( + (index) => + index.unique && + JSON.stringify(index.key) === JSON.stringify({ paymentHash: 1 }), + ), + ).toBeUndefined() + }) }) diff --git a/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts b/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts index 238e7fa9d..c84c4b109 100644 --- a/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts +++ b/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts @@ -1,6 +1,7 @@ const mockRecordInternalTransferInvoice = jest.fn() const mockInsertInternalTransferInvoice = jest.fn() const mockDeleteInternalTransferInvoice = jest.fn() +const mockIsInternalTransferInvoice = jest.fn() jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ recordInternalTransferInvoice: (...args: unknown[]) => mockRecordInternalTransferInvoice(...args), @@ -8,12 +9,15 @@ jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ mockInsertInternalTransferInvoice(...args), deleteInternalTransferInvoice: (...args: unknown[]) => mockDeleteInternalTransferInvoice(...args), + isInternalTransferInvoice: (...args: unknown[]) => + mockIsInternalTransferInvoice(...args), })) jest.mock("@services/tracing", () => ({ recordExceptionInCurrentSpan: jest.fn(), })) +import { isInternalTransferInvoice } from "@services/mongoose/internal-transfer-invoices" import { isDefinitivePaymentRejection, markOutgoingBolt11, @@ -211,3 +215,65 @@ describe("payMarkedBolt11", () => { expect(await run(PaymentSendStatus.Failure)).toBe(PaymentSendStatus.Failure) }) }) + +describe("payMarkedBolt11 with two Flash payers of the same bolt11", () => { + // In-memory stand-in for the marker collection with its real uniqueness: + // one row per (hash, sender, origin); a hash is internal while any row exists. + const rows = new Set() + const key = (a: { paymentHash: string; senderWalletId?: string; origin: string }) => + [a.paymentHash.toLowerCase(), a.senderWalletId ?? "", a.origin].join("|") + + beforeEach(() => { + rows.clear() + mockInsertInternalTransferInvoice.mockImplementation(async (a) => { + if (rows.has(key(a))) return { created: false } + rows.add(key(a)) + return { created: true } + }) + mockDeleteInternalTransferInvoice.mockImplementation(async (a) => rows.delete(key(a))) + mockIsInternalTransferInvoice.mockImplementation(async (hash: string) => + [...rows].some((r) => r.startsWith(`${hash.toLowerCase()}|`)), + ) + }) + + it("keeps the hash internal when A is rejected with a 400 after B marked it too", async () => { + const SENDER_A = "sender-a" as WalletId + const SENDER_B = "sender-b" as WalletId + const rejected = new IbexError(new Error("Bad Request")) + Object.defineProperty(rejected, "httpCode", { value: 400 }) + + // Both mark before either payment returns; A's is then rejected, B's settles. + let releaseA: () => void = () => undefined + const aPaid = new Promise((resolve) => { + releaseA = resolve + }) + const payA = payMarkedBolt11({ + paymentRequest: BOLT11, + senderWalletId: SENDER_A, + pay: async () => { + await aPaid + return rejected + }, + }) + const payB = payMarkedBolt11({ + paymentRequest: BOLT11, + senderWalletId: SENDER_B, + pay: async () => { + releaseA() + return PaymentSendStatus.Success + }, + }) + + expect(await payB).toBe(PaymentSendStatus.Success) + expect(await payA).toBe(rejected) + + expect(mockDeleteInternalTransferInvoice).toHaveBeenCalledTimes(1) + expect(mockDeleteInternalTransferInvoice).toHaveBeenCalledWith({ + paymentHash: HASH, + senderWalletId: SENDER_A, + origin: "ln-send", + }) + expect(rows).toEqual(new Set([`${HASH}|${SENDER_B}|ln-send`])) + expect(await isInternalTransferInvoice(HASH)).toBe(true) + }) +}) diff --git a/test/flash/unit/graphql/ln-send-marker-withdrawal.spec.ts b/test/flash/unit/graphql/ln-send-marker-withdrawal.spec.ts index 861cb3826..cdd0b6e52 100644 --- a/test/flash/unit/graphql/ln-send-marker-withdrawal.spec.ts +++ b/test/flash/unit/graphql/ln-send-marker-withdrawal.spec.ts @@ -5,16 +5,24 @@ // marker helper and the real split recorder, with the marker collection and // the split stores held in memory. -const markers = new Map() +// Mirrors the real collection: one row per (hash, sender, origin), unique. +const markers = new Set() +const markerKey = (args: { + paymentHash: string + senderWalletId?: string + origin: string +}) => [args.paymentHash.toLowerCase(), args.senderWalletId ?? "", args.origin].join("|") +const hashMarked = (paymentHash: string) => + [...markers].some((key) => key.startsWith(`${paymentHash.toLowerCase()}|`)) jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ insertInternalTransferInvoice: async (args: { paymentHash: string senderWalletId?: string origin: string }) => { - const key = args.paymentHash.toLowerCase() + const key = markerKey(args) if (markers.has(key)) return { created: false } - markers.set(key, { senderWalletId: args.senderWalletId, origin: args.origin }) + markers.add(key) return { created: true } }, recordInternalTransferInvoice: async () => true, @@ -22,17 +30,8 @@ jest.mock("@services/mongoose/internal-transfer-invoices", () => ({ paymentHash: string senderWalletId: string origin: string - }) => { - const key = args.paymentHash.toLowerCase() - const found = markers.get(key) - if (!found) return false - if (found.senderWalletId !== args.senderWalletId || found.origin !== args.origin) { - return false - } - return markers.delete(key) - }, - isInternalTransferInvoice: async (paymentHash: string) => - markers.has(paymentHash.toLowerCase()), + }) => markers.delete(markerKey(args)), + isInternalTransferInvoice: async (paymentHash: string) => hashMarked(paymentHash), })) const mockListRules = jest.fn() @@ -177,7 +176,7 @@ describe.each(resolvers)("%s ln-send marker (ENG-653)", (_name, pay) => { const result = await pay() expect(result.status).toBe("failed") - expect(markers.has(HASH)).toBe(false) + expect(hashMarked(HASH)).toBe(false) expect(await merchantReceives()).toEqual({ recorded: 1, duplicates: 0, @@ -192,14 +191,14 @@ describe.each(resolvers)("%s ln-send marker (ENG-653)", (_name, pay) => { await pay() - expect(markers.has(HASH)).toBe(false) + expect(hashMarked(HASH)).toBe(false) }) it("keeps the marker when the payment settles, so the payee's splits do not fire", async () => { mockPayInvoice.mockResolvedValue({ transaction: { payment: { status: { id: 2 } } } }) expect((await pay()).status).toBe("success") - expect(markers.has(HASH)).toBe(true) + expect(hashMarked(HASH)).toBe(true) expect(await merchantReceives()).toEqual({ recorded: 0, duplicates: 0, @@ -212,13 +211,13 @@ describe.each(resolvers)("%s ln-send marker (ENG-653)", (_name, pay) => { await pay() - expect(markers.has(HASH)).toBe(true) + expect(hashMarked(HASH)).toBe(true) }) it("keeps the marker while the payment is in flight", async () => { mockPayInvoice.mockResolvedValue({ transaction: { payment: { status: { id: 1 } } } }) expect((await pay()).status).toBe("pending") - expect(markers.has(HASH)).toBe(true) + expect(hashMarked(HASH)).toBe(true) }) }) From 952e26d9c536d0e38b72f745b89122e8526eb98f Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 23:40:50 -0700 Subject: [PATCH 12/13] fix(payment-splits): dedupe per payout target, unknown on ambiguous IBEX errors, wall-clock stamps Review fixes for ENG-653: - Execution dedupe is now unique on (triggerKey, walletId, destinationWalletId) instead of (triggerKey, ruleId). walletSplitsSet rewrites every rule with a fresh id, so a webhook replay after the merchant re-saved a split paid the same partner twice. The webhook also passes the receive time (settledAt/createdAt) as receivedAt, so the pre-rule guard covers it too. - After the send guard passed, an IBEX error that is not a definitive rejection (5xx, timeout, transport error) marks the row unknown and alerts ops instead of failing it and telling the owner it failed. - Claims and outcomes are stamped with wall-clock time, and each loop phase reads the clock when it starts, so a slow reconcile cannot let another replica's stale sweep take a row still being paid. A lost outcome (finishClaimed matched nothing) is logged at error level. - CompletedInvoice is a definitive rejection: the caller's ln-send marker is withdrawn, so a Flash user cannot suppress a merchant's splits by trying to pay an already-paid bolt11. - Transient repository errors before the send retry within the attempt budget; only a definite not-found fails at once. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G4SaLJF5SWq6sjBEaWJFow --- .../payment-splits/execute-pending-splits.ts | 128 ++++++++----- .../payment-splits/record-split-executions.ts | 11 +- src/app/payment-splits/split-loop.ts | 14 +- src/app/payments/mark-outgoing-invoice.ts | 18 +- .../ibex/webhook-server/routes/on-receive.ts | 14 ++ .../mongoose/payment-split-executions.ts | 20 +- src/services/mongoose/schema.ts | 16 +- .../execute-pending-splits.spec.ts | 178 +++++++++++++++++- .../payment-splits/fake-executions-store.ts | 13 +- .../record-split-executions.spec.ts | 28 +++ .../app/payment-splits/split-loop.spec.ts | 23 +++ .../payment-splits/sweep-stale-splits.spec.ts | 2 +- .../payments/mark-outgoing-invoice.spec.ts | 16 +- .../routes/on-receive-splits.spec.ts | 59 ++++++ 14 files changed, 465 insertions(+), 75 deletions(-) diff --git a/src/app/payment-splits/execute-pending-splits.ts b/src/app/payment-splits/execute-pending-splits.ts index 526e41229..bd3185080 100644 --- a/src/app/payment-splits/execute-pending-splits.ts +++ b/src/app/payment-splits/execute-pending-splits.ts @@ -2,9 +2,11 @@ import { PaymentSplitsConfig } from "@config" import { sendOutcomeNotificationBestEffort } from "@app/notifications/send-outcome-notification" import { authorizeSend } from "@app/payments/authorize-send" +import { isDefinitivePaymentRejection } from "@app/payments/mark-outgoing-invoice" import { intraledgerPaymentSendWalletIdForUsdWallet } from "@app/payments/send-intraledger" import { PaymentSendStatus } from "@domain/bitcoin/lightning" +import { CouldNotFindError } from "@domain/errors" import { LockServiceError, ResourceAttemptsLockServiceError } from "@domain/lock" import { InternalTransferMarkerError, splitRetryBackoffMs } from "@domain/payment-splits" import { SendLimitsUnavailableError } from "@domain/payments/errors" @@ -59,15 +61,29 @@ export const isRetryableSplitError = ( } /** - * A lock error the send may have paid through (see isRetryableSplitError). - * Treated like a row the stale sweep finds stuck in `sending`: `unknown` plus - * an ops alert, never resent. + * An error the send may have paid through. Treated like a row the stale sweep + * finds stuck in `sending`: `unknown` plus an ops alert, never resent, and no + * "split failed" push to the owner (who would then pay the partner by hand). + * + * - A lock error that is not a retryable pre-payment one (see + * isRetryableSplitError). + * - Once the ENG-573 guard passed (`paymentAttempted`), any IBEX error that is + * not a definitive rejection: a transport error, a timeout or a 5xx from + * payInvoice, where IBEX may already have debited (the KNOWN GAP in + * idempotency.ts). A definitive rejection (insufficient balance, a 4xx such + * as no route or a 429) moved no money and stays `failed` / retryable. */ -const isAmbiguousLockError = ( +export const isAmbiguousSplitSendError = ( error: Error, { paymentAttempted }: { paymentAttempted: boolean }, -): boolean => - error instanceof LockServiceError && !isRetryableSplitError(error, { paymentAttempted }) +): boolean => { + if (error instanceof LockServiceError) { + return !isRetryableSplitError(error, { paymentAttempted }) + } + return ( + paymentAttempted && error instanceof IbexError && !isDefinitivePaymentRejection(error) + ) +} const formatCents = (cents: number): string => `US$${(cents / 100).toFixed(2)}` @@ -101,55 +117,88 @@ type ExecuteOutcome = const executeOne = async ( execution: PaymentSplitExecutionRecord, now: Date, + stamp: () => Date, ): Promise => { const executions = PaymentSplitExecutionsRepository() const attempts = execution.attempts + 1 + // Records the outcome stamped with wall-clock time (not the pass's `now`), + // so updatedAt reflects when the row actually finished. A false result means + // the row was no longer `sending`: the stale sweep took it (marked it + // `unknown` and alerted ops) while this send ran, so the outcome below is + // the only record of what really happened. + const finish = async ( + update: Omit[1], "now">, + ): Promise => { + const finished = await executions.finishClaimed(execution.id, { + ...update, + now: stamp(), + }) + if (finished === true) return + logger.error( + { + executionId: execution.id, + walletId: execution.walletId, + destinationWalletId: execution.destinationWalletId, + outcome: update.status, + lastError: update.lastError, + error: finished instanceof Error ? finished : undefined, + }, + finished instanceof Error + ? "payment split outcome could not be recorded; the row stays in sending and the stale sweep will mark it unknown — resolve by hand with this outcome" + : "payment split outcome lost: the row was no longer sending (the stale sweep took it); resolve the unknown row by hand with this outcome", + ) + } + const fail = async (error: Error | string, accountId?: string) => { const lastError = typeof error === "string" ? error : error.name || error.message - await executions.finishClaimed(execution.id, { - status: "failed", - attempts, - lastError, - now, - }) + await finish({ status: "failed", attempts, lastError }) if (accountId) await notifyOwner(accountId, "failed", execution) return "failed" as const } const retry = async (error: Error) => { - await executions.finishClaimed(execution.id, { + await finish({ status: "pending", attempts, + // From the pass's `now`, so a row retried in this pass is due in a + // later pass (see executePendingSplits). nextAttemptAt: new Date(now.getTime() + splitRetryBackoffMs(attempts)), lastError: error.name, - now, }) return "retry" as const } + // Nothing has been sent yet: a transient error (a Mongo stepdown, a network + // blip) gets the same retry budget as any pre-payment error. Only a definite + // not-found (the wallet or account is gone) fails at once. + const failBeforeSend = async (error: Error) => { + if ( + !(error instanceof CouldNotFindError) && + attempts < PaymentSplitsConfig.maxAttempts + ) { + return retry(error) + } + return fail(error) + } + // Re-check the internal-transfer marker at pay time. An LNURL send from // another Flash user is marked only after IBEX returns, so its receive can be // recorded before the marker exists (ENG-653 decision 1). const internal = await isInternalTransferInvoice(execution.triggerKey) if (internal instanceof Error) { // Nothing was sent: the same retry budget as any pre-payment error. - if (attempts < PaymentSplitsConfig.maxAttempts) return retry(internal) - return fail(internal) + return failBeforeSend(internal) } if (internal) { - await executions.finishClaimed(execution.id, { - status: "skipped_internal", - attempts: execution.attempts, - now, - }) + await finish({ status: "skipped_internal", attempts: execution.attempts }) return "skipped_internal" } const wallet = await WalletsRepository().findById(execution.walletId as WalletId) - if (wallet instanceof Error) return fail(wallet) + if (wallet instanceof Error) return failBeforeSend(wallet) const account = await AccountsRepository().findById(wallet.accountId) - if (account instanceof Error) return fail(account) + if (account instanceof Error) return failBeforeSend(account) // Set once the ENG-573 guard passes. withPaymentIdempotency runs `authorize` // inside the idempotency lock immediately before the payment, so this is the @@ -181,16 +230,11 @@ const executeOne = async ( }) if (result instanceof Error) { - if (isAmbiguousLockError(result, { paymentAttempted })) { - await executions.finishClaimed(execution.id, { - status: "unknown", - attempts, - lastError: result.name, - now, - }) + if (isAmbiguousSplitSendError(result, { paymentAttempted })) { + await finish({ status: "unknown", attempts, lastError: result.name }) logger.error( { executionId: execution.id, walletId: execution.walletId, error: result }, - "payment split lock error after the send may have paid; marked unknown", + "payment split send may have paid despite the error; marked unknown", ) notifyOpsEvent({ flow: "transfer", @@ -220,7 +264,7 @@ const executeOne = async ( } if (result === PaymentSendStatus.Success) { - await executions.finishClaimed(execution.id, { status: "sent", attempts, now }) + await finish({ status: "sent", attempts }) await notifyOwner(account.id, "sent", execution) return "sent" } @@ -228,11 +272,7 @@ const executeOne = async ( if (result === PaymentSendStatus.Pending) { // Money has probably left the wallet: never re-pay (same stance as the // Fygaro credit). Ops resolves the final state from IBEX. - await executions.finishClaimed(execution.id, { - status: "ibex_pending", - attempts, - now, - }) + await finish({ status: "ibex_pending", attempts }) return "ibex_pending" } @@ -256,9 +296,12 @@ export type ExecutePendingSplitsSummary = { * * Runs every PaymentSplitsConfig.loopIntervalMs from the split loop and keeps * claiming until nothing is due or executorTimeBudgetMs is spent, so the - * throughput is not capped by a fixed batch per run. `now` is fixed per pass: - * a row retried in this pass is due at now + backoff and is picked up by a - * later pass, which is what makes the 1/2/4/8 minute backoff real. + * throughput is not capped by a fixed batch per run. `now` is fixed per pass + * and decides what is due: a row retried in this pass is due at now + backoff + * and is picked up by a later pass, which is what makes the 1/2/4/8 minute + * backoff real. Claims and outcomes are stamped with `clock()` (wall-clock + * time), never the pass's `now`, so a row's updatedAt never looks older to the + * stale sweep on another replica than it really is. * * `shouldStop` lets the loop drain on SIGTERM (see startPaymentSplitsLoop). */ @@ -281,17 +324,18 @@ export const executePendingSplits = async ({ if (!PaymentSplitsConfig.enabled) return summary const executions = PaymentSplitExecutionsRepository() + const stamp = () => new Date(clock()) const deadline = clock() + PaymentSplitsConfig.executorTimeBudgetMs // `shouldStop` is checked before every claim, so on shutdown the row in // flight finishes and no new row moves to `sending` (where a kill would // strand it for the stale sweep to mark `unknown`). while (clock() < deadline && !shouldStop()) { - const execution = await executions.claimNextDue(now) + const execution = await executions.claimNextDue(now, stamp()) if (execution instanceof Error) return execution if (execution === null) break summary.claimed += 1 - const outcome = await executeOne(execution, now) + const outcome = await executeOne(execution, now, stamp) if (outcome === "sent") summary.sent += 1 else if (outcome === "ibex_pending") summary.ibexPending += 1 else if (outcome === "failed") summary.failed += 1 diff --git a/src/app/payment-splits/record-split-executions.ts b/src/app/payment-splits/record-split-executions.ts index 757d095ad..a7e14bcbc 100644 --- a/src/app/payment-splits/record-split-executions.ts +++ b/src/app/payment-splits/record-split-executions.ts @@ -17,13 +17,16 @@ const NOTHING: RecordSplitExecutionsResult = { recorded: 0, duplicates: 0, skipp /** * Records one pending split execution per enabled rule on `walletId` for one * receive. Shared by the on-receive webhook and the reconcile cron, which both - * key the receive by its payment hash so the unique (triggerKey, ruleId) index - * dedupes them against each other and against webhook replays. + * key the receive by its payment hash so the unique (triggerKey, walletId, + * destinationWalletId) index dedupes them against each other and against + * webhook replays, including a replay after the rules were rewritten (new + * rule ids for the same destinations). * * Never sends: payouts happen only in the cron executor. * - * `receivedAt`, when known (the cron reads it from IBEX), keeps a rule from - * firing on a receive that settled before the rule was created. + * `receivedAt`, when known (the cron reads it from IBEX, the webhook from its + * body), keeps a rule from firing on a receive that settled before the rule + * was created. */ export const recordSplitExecutions = async ({ walletId, diff --git a/src/app/payment-splits/split-loop.ts b/src/app/payment-splits/split-loop.ts index 18c06b287..5d4c20077 100644 --- a/src/app/payment-splits/split-loop.ts +++ b/src/app/payment-splits/split-loop.ts @@ -36,12 +36,18 @@ const claimReconcileLease = async (intervalMs: number): Promise => { * due row within the executor's time budget, then sweep rows stuck in * `sending`. Executor claims and sweep moves are atomic per row, so ticks on * several replicas never pay the same row twice. + * + * Each phase reads the clock when it starts (unless a test pins `now`): the + * reconcile has no time bound, and an executor handed a `now` from before it + * would treat rows as due by a stale time while another replica's sweep, on + * a fresh clock, judges what is stuck. */ export const runPaymentSplitsTick = async ({ - now = new Date(), + now: pinnedNow, shouldStop = () => false, }: { now?: Date; shouldStop?: () => boolean } = {}): Promise => { if (!PaymentSplitsConfig.enabled) return + const currentTime = () => pinnedNow ?? new Date() if ( !shouldStop() && @@ -49,7 +55,7 @@ export const runPaymentSplitsTick = async ({ ) { const reconciled = await reconcileSplits({ cadenceMs: PaymentSplitsConfig.reconcileIntervalMs, - now, + now: currentTime(), }) if (reconciled instanceof Error) { logger.error({ err: reconciled }, "payment splits loop: reconcile failed") @@ -60,7 +66,7 @@ export const runPaymentSplitsTick = async ({ if (shouldStop()) return - const executed = await executePendingSplits({ now, shouldStop }) + const executed = await executePendingSplits({ now: currentTime(), shouldStop }) if (executed instanceof Error) { logger.error({ err: executed }, "payment splits loop: executor failed") } else if (executed.claimed > 0) { @@ -69,7 +75,7 @@ export const runPaymentSplitsTick = async ({ if (shouldStop()) return - const swept = await sweepStaleSplits({ now }) + const swept = await sweepStaleSplits({ now: currentTime() }) if (swept instanceof Error) { logger.error({ err: swept }, "payment splits loop: stale sweep failed") } diff --git a/src/app/payments/mark-outgoing-invoice.ts b/src/app/payments/mark-outgoing-invoice.ts index dd5f75068..f5d469939 100644 --- a/src/app/payments/mark-outgoing-invoice.ts +++ b/src/app/payments/mark-outgoing-invoice.ts @@ -66,17 +66,23 @@ export const markOutgoingBolt11 = async (args: { /** * True only when IBEX refused the payment before any money moved: an - * insufficient balance, a corroborated FAILED status, or any other 4xx the - * request was rejected with (no route, expired invoice, rate limit). Never true - * for a pending or unreadable outcome, a transport error or a 5xx (the payment - * may still settle), nor for an already-paid invoice (someone else's payment of - * it settled, and that payer may be another Flash wallet whose marker this is). + * insufficient balance, a corroborated FAILED status, an already-paid (or + * already-prepared) invoice, or any other 4xx the request was rejected with + * (no route, expired invoice, rate limit). Never true for a pending or + * unreadable outcome, a transport error or a 5xx (the payment may still + * settle). + * + * An already-paid invoice is a rejection of THIS caller: it paid nothing. + * Markers are one row per (hash, sender, origin), so a Flash wallet that did + * pay the invoice has its own row, and withdrawing this caller's row protects + * nobody. Keeping it would let any Flash user suppress a merchant's splits by + * attempting to pay a bolt11 an external customer already paid. */ export const isDefinitivePaymentRejection = (result: unknown): boolean => { if (result === PaymentSendStatus.Failure) return true if (result instanceof InsufficientIbexBalance) return true if (result instanceof FailedIbexPayment) return true - if (result instanceof CompletedInvoice) return false + if (result instanceof CompletedInvoice) return true if (result instanceof IbexError) { const code = result.httpCode return code !== undefined && code >= 400 && code < 500 && code !== 408 diff --git a/src/services/ibex/webhook-server/routes/on-receive.ts b/src/services/ibex/webhook-server/routes/on-receive.ts index 36795c13b..6fc84ea89 100644 --- a/src/services/ibex/webhook-server/routes/on-receive.ts +++ b/src/services/ibex/webhook-server/routes/on-receive.ts @@ -78,6 +78,19 @@ const fetchPaymentContext = async ( // split loop (lookback = interval plus margin) and once a day from the galoy // cron (lookback = a day plus margin). Duplicate deliveries are duplicate-key // no-ops. +// The receive time from the webhook body, so a rule created after the receive +// (a replay that lands after the merchant set or re-saved a split) never fires +// on it. Undefined when IBEX sent no parseable time. +const parseReceivedAt = (transaction: unknown): Date | undefined => { + const tx = transaction as { settledAt?: unknown; createdAt?: unknown } | undefined + for (const value of [tx?.settledAt, tx?.createdAt]) { + if (typeof value !== "string" && typeof value !== "number") continue + const date = new Date(value) + if (!Number.isNaN(date.getTime())) return date + } + return undefined +} + const recordPaymentSplits = async ( req: PaymentRequest, _resp: Response, @@ -97,6 +110,7 @@ const recordPaymentSplits = async ( paymentHash, amount, currency: receiverWallet.currency, + receivedAt: parseReceivedAt(transaction), }) if (result instanceof Error) { logger.error( diff --git a/src/services/mongoose/payment-split-executions.ts b/src/services/mongoose/payment-split-executions.ts index be4e6d494..f62e2a16c 100644 --- a/src/services/mongoose/payment-split-executions.ts +++ b/src/services/mongoose/payment-split-executions.ts @@ -57,9 +57,10 @@ const toExecution = (raw: RawExecution): PaymentSplitExecutionRecord => ({ export const PaymentSplitExecutionsRepository = () => { /** - * Inserts one execution. A duplicate (triggerKey, ruleId) — a webhook replay, - * or the reconcile cron finding a receive the webhook already recorded — is a - * typed no-op: `{ inserted: false }`. + * Inserts one execution. A duplicate (triggerKey, walletId, + * destinationWalletId) — a webhook replay, or the reconcile cron finding a + * receive the webhook already recorded, even under a rule rewritten since — + * is a typed no-op: `{ inserted: false }`. */ const insert = async (data: { triggerKey: string @@ -90,17 +91,22 @@ export const PaymentSplitExecutionsRepository = () => { } /** - * Atomically moves one due `pending` row to `sending`. Two concurrent - * claimers can never both get the same row: findOneAndUpdate matches on the - * status it is changing. + * Atomically moves one due (nextAttemptAt <= `now`) `pending` row to + * `sending`. Two concurrent claimers can never both get the same row: + * findOneAndUpdate matches on the status it is changing. + * + * The claim is stamped with `claimedAt` (wall-clock time by default), not + * `now`: the stale sweep reads updatedAt as "claimed at", and a stale `now` + * would let another replica's sweep take a row that is still being paid. */ const claimNextDue = async ( now: Date, + claimedAt: Date = new Date(), ): Promise => { try { const doc = await PaymentSplitExecutionModel.findOneAndUpdate( { status: "pending", nextAttemptAt: { $lte: now } }, - { $set: { status: "sending", updatedAt: now } }, + { $set: { status: "sending", updatedAt: claimedAt } }, { sort: { nextAttemptAt: 1 }, new: true }, ) .lean() diff --git a/src/services/mongoose/schema.ts b/src/services/mongoose/schema.ts index 6a62709e4..e3976f57a 100644 --- a/src/services/mongoose/schema.ts +++ b/src/services/mongoose/schema.ts @@ -911,9 +911,14 @@ PaymentSplitRuleSchema.index({ walletId: 1, enabled: 1 }) export const PaymentSplitRule = mongoose.model("PaymentSplitRule", PaymentSplitRuleSchema) -// One row per (receive, rule). The unique (triggerKey, ruleId) index is the -// dedupe between the webhook and the reconcile cron, and between webhook -// replays: a second insert for the same receive is a duplicate-key no-op. +// One row per (receive, payout target). The unique (triggerKey, walletId, +// destinationWalletId) index is the dedupe between the webhook and the +// reconcile cron, and between webhook replays: a second insert for the same +// receive is a duplicate-key no-op. It is keyed on the payout target, not the +// rule id, because walletSplitsSet replaces every rule with a fresh _id: a +// replay after the merchant re-saves (or deletes and re-adds) the same split +// must still be a duplicate, not a second payout. A wallet's rules never +// repeat a destination (SplitDuplicateDestinationError). const PaymentSplitExecutionSchema = new Schema({ triggerKey: { type: String, required: true }, ruleId: { type: String, required: true }, @@ -944,7 +949,10 @@ const PaymentSplitExecutionSchema = new Schema({ updatedAt: { type: Date, required: true, default: Date.now }, }) -PaymentSplitExecutionSchema.index({ triggerKey: 1, ruleId: 1 }, { unique: true }) +PaymentSplitExecutionSchema.index( + { triggerKey: 1, walletId: 1, destinationWalletId: 1 }, + { unique: true }, +) PaymentSplitExecutionSchema.index({ status: 1, nextAttemptAt: 1 }) export const PaymentSplitExecution = mongoose.model( diff --git a/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts index 87584dbbe..0842e684a 100644 --- a/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts +++ b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts @@ -5,6 +5,17 @@ const mockFindWalletById = jest.fn() const mockFindAccountById = jest.fn() const mockIsInternal = jest.fn() const mockNotifyOps = jest.fn() +jest.mock("@services/logger", () => { + const logger = { + trace: jest.fn(), + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + } + return { baseLogger: { ...logger, child: () => logger } } +}) jest.mock("@app/payments/send-intraledger", () => ({ intraledgerPaymentSendWalletIdForUsdWallet: (...args: unknown[]) => mockSend(...args), @@ -46,9 +57,12 @@ import { } from "@app/payment-splits/execute-pending-splits" import { PaymentSendStatus } from "@domain/bitcoin/lightning" import { + CouldNotFindAccountFromIdError, + CouldNotFindWalletFromIdError, InactiveAccountError, IntraledgerLimitsExceededError, RepositoryError, + UnknownRepositoryError, } from "@domain/errors" import { ResourceAttemptsLockServiceError, @@ -59,7 +73,7 @@ import { InternalTransferMarkerError, splitRetryBackoffMs } from "@domain/paymen import { SendLimitsUnavailableError } from "@domain/payments/errors" import { PaymentSendRateLimiterExceededError } from "@domain/rate-limit/errors" import { WalletCurrency } from "@domain/shared" -import { InsufficientIbexBalance } from "@services/ibex/errors" +import { IbexError, InsufficientIbexBalance } from "@services/ibex/errors" import { createFakeExecutionsStore } from "./fake-executions-store" @@ -67,6 +81,10 @@ import { createFakeExecutionsStore } from "./fake-executions-store" // is safe: the mock factory only captures the reference. const mockStore = createFakeExecutionsStore() +const mockLogger = ( + jest.requireMock("@services/logger") as { baseLogger: { error: jest.Mock } } +).baseLogger + const SOURCE = "source-wallet" const DEST = "dest-wallet" const OWNER = { id: "owner-account", level: 1 } @@ -404,6 +422,164 @@ describe("executePendingSplits", () => { }) }) + describe("IBEX errors after the send guard passed", () => { + // authorize passed, so Ibex.payInvoice was called: anything but a definite + // rejection may have debited (the KNOWN GAP in idempotency.ts). + const afterAuthorize = + (error: Error) => + async (args: { authorize: () => Promise }): Promise => { + await args.authorize() + return error + } + + const httpError = (httpCode?: number) => { + const err = new IbexError(new Error(`IBEX ${httpCode ?? "ECONNRESET"}`)) + if (httpCode !== undefined) { + Object.defineProperty(err, "httpCode", { value: httpCode }) + } + return err + } + + it.each([ + ["a 500", httpError(500)], + ["a 502", httpError(502)], + ["a 408 timeout", httpError(408)], + ["a transport error / timeout with no status", httpError()], + ])( + "marks %s unknown, alerts ops, never pushes the owner a failure, never re-sends", + async (_label, error) => { + await seed() + mockSend.mockImplementation(afterAuthorize(error)) + + const summary = await executePendingSplits({ now: T0 }) + + expect(summary).toMatchObject({ claimed: 1, unknown: 1, failed: 0, retried: 0 }) + expect(onlyRow()).toMatchObject({ status: "unknown", attempts: 1 }) + expect(mockNotifyOps).toHaveBeenCalledWith( + expect.objectContaining({ phase: "split-unknown", status: "failed" }), + ) + expect(mockNotify).not.toHaveBeenCalled() + + await executePendingSplits({ now: minutes(60) }) + expect(mockSend).toHaveBeenCalledTimes(1) + }, + ) + + it("fails a definitive 400 rejection and pushes the owner", async () => { + await seed() + mockSend.mockImplementation(afterAuthorize(httpError(400))) + + const summary = await executePendingSplits({ now: T0 }) + + expect(summary).toMatchObject({ claimed: 1, failed: 1, unknown: 0 }) + expect(onlyRow()).toMatchObject({ status: "failed", attempts: 1 }) + expect(mockNotifyOps).not.toHaveBeenCalled() + expect(mockNotify).toHaveBeenCalledWith( + expect.objectContaining({ phraseBase: "notification.paymentSplit.failed" }), + ) + }) + + it("still retries an IBEX 429 (rejected before anything moved)", async () => { + await seed() + mockSend.mockImplementation(afterAuthorize(httpError(429))) + + await executePendingSplits({ now: T0 }) + + expect(onlyRow()).toMatchObject({ status: "pending", attempts: 1 }) + expect(mockNotifyOps).not.toHaveBeenCalled() + }) + }) + + describe("repository errors before the send", () => { + it.each([ + ["the wallet lookup", "wallet"], + ["the account lookup", "account"], + ])( + "retries a transient error from %s instead of failing the split", + async (_label, which) => { + await seed() + const blip = new UnknownRepositoryError("primary stepdown") + if (which === "wallet") mockFindWalletById.mockResolvedValueOnce(blip) + else mockFindAccountById.mockResolvedValueOnce(blip) + mockSend.mockResolvedValue(PaymentSendStatus.Success) + + expect(await executePendingSplits({ now: T0 })).toMatchObject({ retried: 1 }) + expect(onlyRow()).toMatchObject({ status: "pending", attempts: 1 }) + expect(mockSend).not.toHaveBeenCalled() + expect(mockNotify).not.toHaveBeenCalled() + + await executePendingSplits({ now: minutes(1) }) + expect(onlyRow()).toMatchObject({ status: "sent", attempts: 2 }) + }, + ) + + it("fails a transient error once the attempt budget is spent", async () => { + await seed() + mockFindWalletById.mockResolvedValue(new UnknownRepositoryError("mongo down")) + + let now = T0 + for (const backoff of [1, 2, 4, 8]) { + await executePendingSplits({ now }) + now = new Date(now.getTime() + backoff * 60_000) + } + await executePendingSplits({ now }) + + expect(onlyRow()).toMatchObject({ status: "failed", attempts: 5 }) + expect(mockSend).not.toHaveBeenCalled() + }) + + it.each([ + ["the wallet is gone", "wallet", new CouldNotFindWalletFromIdError()], + ["the account is gone", "account", new CouldNotFindAccountFromIdError()], + ])("fails at once when %s", async (_label, which, error) => { + await seed() + if (which === "wallet") mockFindWalletById.mockResolvedValue(error) + else mockFindAccountById.mockResolvedValue(error) + + expect(await executePendingSplits({ now: T0 })).toMatchObject({ failed: 1 }) + expect(onlyRow()).toMatchObject({ status: "failed", attempts: 1 }) + expect(mockSend).not.toHaveBeenCalled() + }) + }) + + describe("wall-clock stamps", () => { + it("stamps the claim and the outcome with the clock, not the pass's now", async () => { + await seed() + // The pass's `now` is 11 minutes old by the time the row is claimed + // (a slow reconcile ran first). + const wallClock = minutes(11).getTime() + let updatedAtWhileSending: Date | undefined + mockSend.mockImplementation(async () => { + updatedAtWhileSending = onlyRow().updatedAt + return PaymentSendStatus.Success + }) + + await executePendingSplits({ now: T0, clock: () => wallClock }) + + expect(updatedAtWhileSending).toEqual(new Date(wallClock)) + expect(onlyRow()).toMatchObject({ status: "sent", updatedAt: new Date(wallClock) }) + }) + + it("logs an error when the stale sweep took the row while it was being paid", async () => { + await seed() + mockSend.mockImplementation(async () => { + // Another replica's sweep marks the row unknown mid-send. + await mockStore.repo.markNextStaleUnknown(new Date(8.64e15), new Date()) + return PaymentSendStatus.Success + }) + + await executePendingSplits({ now: T0 }) + + // The sweep's `unknown` is never overwritten, but the real outcome is + // logged for the human resolving it. + expect(onlyRow()).toMatchObject({ status: "unknown" }) + expect(mockLogger.error).toHaveBeenCalledWith( + expect.objectContaining({ executionId: onlyRow().id, outcome: "sent" }), + expect.stringContaining("outcome lost"), + ) + }) + }) + describe("internal-transfer re-check", () => { it("skips a row whose hash was marked internal after it was recorded", async () => { await seed() diff --git a/test/flash/unit/app/payment-splits/fake-executions-store.ts b/test/flash/unit/app/payment-splits/fake-executions-store.ts index cc7f4bb1a..81541b5f3 100644 --- a/test/flash/unit/app/payment-splits/fake-executions-store.ts +++ b/test/flash/unit/app/payment-splits/fake-executions-store.ts @@ -1,6 +1,6 @@ // In-memory stand-in for PaymentSplitExecutionsRepository that models the two -// properties the split pipeline relies on: the unique (triggerKey, ruleId) -// index and the atomic pending -> sending claim. +// properties the split pipeline relies on: the unique (triggerKey, walletId, +// destinationWalletId) index and the atomic pending -> sending claim. type Row = { id: string triggerKey: string @@ -37,7 +37,10 @@ export const createFakeExecutionsStore = () => { }) => { const { now = new Date(), nextAttemptAt = now, ...fields } = data const dup = [...rows.values()].find( - (r) => r.triggerKey === fields.triggerKey && r.ruleId === fields.ruleId, + (r) => + r.triggerKey === fields.triggerKey && + r.walletId === fields.walletId && + r.destinationWalletId === fields.destinationWalletId, ) if (dup) return { inserted: false as const } const id = `exec-${++seq}` @@ -51,13 +54,13 @@ export const createFakeExecutionsStore = () => { }) return { inserted: true as const, id } }, - claimNextDue: async (now: Date) => { + claimNextDue: async (now: Date, claimedAt: Date = new Date()) => { const due = [...rows.values()] .filter((r) => r.status === "pending" && r.nextAttemptAt <= now) .sort((a, b) => a.nextAttemptAt.getTime() - b.nextAttemptAt.getTime())[0] if (!due) return null due.status = "sending" - due.updatedAt = now + due.updatedAt = claimedAt return { ...due } }, finishClaimed: async ( diff --git a/test/flash/unit/app/payment-splits/record-split-executions.spec.ts b/test/flash/unit/app/payment-splits/record-split-executions.spec.ts index 8e956886c..14846ae1c 100644 --- a/test/flash/unit/app/payment-splits/record-split-executions.spec.ts +++ b/test/flash/unit/app/payment-splits/record-split-executions.spec.ts @@ -117,6 +117,34 @@ describe("recordSplitExecutions", () => { expect(await record()).toEqual({ recorded: 0, duplicates: 2, skippedMin: 0 }) }) + it("treats a replay after the rules were rewritten as a duplicate, not a second payout", async () => { + // walletSplitsSet replaces every rule with a fresh id. A webhook replay + // landing after the merchant re-saved the same split must not pay the + // same destination again under the new rule id. + expect(await record()).toEqual({ recorded: 2, duplicates: 0, skippedMin: 0 }) + + mockListRules.mockResolvedValue( + rules.map((rule) => ({ + ...rule, + id: `${rule.id}-resaved`, + createdAt: new Date("2026-10-02T00:00:00Z"), + })), + ) + + expect(await record()).toEqual({ recorded: 0, duplicates: 2, skippedMin: 0 }) + expect(mockStore.rows.size).toBe(2) + }) + + it("records a destination added by the rewrite for a receive it has not paid yet", async () => { + await record() + mockListRules.mockResolvedValue([ + rules[0], + { ...rules[1], id: "rule-3", destinationWalletId: "dest-3" }, + ]) + + expect(await record()).toEqual({ recorded: 1, duplicates: 1, skippedMin: 0 }) + }) + it("records nothing for a hash Flash minted for its own transfer", async () => { mockIsInternal.mockResolvedValue(true) diff --git a/test/flash/unit/app/payment-splits/split-loop.spec.ts b/test/flash/unit/app/payment-splits/split-loop.spec.ts index 65673188e..4ea5c2000 100644 --- a/test/flash/unit/app/payment-splits/split-loop.spec.ts +++ b/test/flash/unit/app/payment-splits/split-loop.spec.ts @@ -88,6 +88,29 @@ describe("runPaymentSplitsTick", () => { expect(mockExecute).toHaveBeenCalledTimes(1) }) + it("hands the executor and the sweep a clock read after a slow reconcile", async () => { + // A reconcile has no time bound. With a `now` captured at the start of the + // tick, the executor would stamp claims 11 minutes in the past and another + // replica's stale sweep would take a row that is still being paid. + const after = new Date(NOW.getTime() + 11 * 60_000) + jest.useFakeTimers({ now: NOW }) + mockReconcile.mockImplementation(async ({ now }: { now: Date }) => { + expect(now).toEqual(NOW) + jest.setSystemTime(after) + return { recorded: 0, errors: 0 } + }) + + try { + await runPaymentSplitsTick() + } finally { + jest.useRealTimers() + } + + expect(mockReconcile).toHaveBeenCalledTimes(1) + expect(mockExecute.mock.calls[0][0].now).toEqual(after) + expect(mockSweep.mock.calls[0][0].now).toEqual(after) + }) + it("does nothing while the feature flag is off", async () => { PaymentSplitsConfig.enabled = false diff --git a/test/flash/unit/app/payment-splits/sweep-stale-splits.spec.ts b/test/flash/unit/app/payment-splits/sweep-stale-splits.spec.ts index e1171fbbd..90b59b002 100644 --- a/test/flash/unit/app/payment-splits/sweep-stale-splits.spec.ts +++ b/test/flash/unit/app/payment-splits/sweep-stale-splits.spec.ts @@ -34,7 +34,7 @@ const seedSending = async (claimedAt: Date) => { status: "pending", now: claimedAt, }) - await mockStore.repo.claimNextDue(claimedAt) + await mockStore.repo.claimNextDue(claimedAt, claimedAt) } const originalEnabled = PaymentSplitsConfig.enabled diff --git a/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts b/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts index c84c4b109..7d50f2bb7 100644 --- a/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts +++ b/test/flash/unit/app/payments/mark-outgoing-invoice.spec.ts @@ -124,6 +124,7 @@ describe("isDefinitivePaymentRejection", () => { ["a FAILED status", PaymentSendStatus.Failure], ["an IBEX 400 (no route, expired)", httpError(400)], ["an IBEX 429", httpError(429)], + ["an already-paid invoice", new CompletedInvoice(new Error("400"))], ])("is true for %s", (_label, result) => { expect(isDefinitivePaymentRejection(result)).toBe(true) }) @@ -132,7 +133,6 @@ describe("isDefinitivePaymentRejection", () => { ["success", PaymentSendStatus.Success], ["pending", PaymentSendStatus.Pending], ["an unreadable response", new UnconfirmedIbexPayment("unreadable")], - ["an already-paid invoice", new CompletedInvoice(new Error("400"))], ["an IBEX 5xx", httpError(502)], ["a request timeout", httpError(408)], ["a transport error with no status", new IbexError(new Error("ECONNRESET"))], @@ -187,6 +187,20 @@ describe("payMarkedBolt11", () => { }) }) + it("withdraws its own marker when the invoice was already paid by someone else", async () => { + // Otherwise any Flash user could suppress a merchant's splits by trying to + // pay a bolt11 an external customer already paid: the 30-day marker would + // turn the pending split into skipped_internal. + const completed = new CompletedInvoice(new Error("400")) + + expect(await run(completed)).toBe(completed) + expect(mockDeleteInternalTransferInvoice).toHaveBeenCalledWith({ + paymentHash: HASH, + senderWalletId: SENDER, + origin: "ln-send", + }) + }) + it("leaves a marker it did not create, even on a rejection", async () => { mockInsertInternalTransferInvoice.mockResolvedValue({ created: false }) diff --git a/test/flash/unit/services/ibex/webhook-server/routes/on-receive-splits.spec.ts b/test/flash/unit/services/ibex/webhook-server/routes/on-receive-splits.spec.ts index 1fbac6031..93eb3a5c9 100644 --- a/test/flash/unit/services/ibex/webhook-server/routes/on-receive-splits.spec.ts +++ b/test/flash/unit/services/ibex/webhook-server/routes/on-receive-splits.spec.ts @@ -194,6 +194,65 @@ describe("on-receive webhook payment splits (ENG-653)", () => { expect(mockStore.rows.size).toBe(1) }) + it("a replay after the split was re-saved records nothing new", async () => { + await post(paths.invoice, lightningBody()) + // walletSplitsSet gave the same destination a fresh rule id. + mockListRules.mockResolvedValue([ + { + id: "rule-1-resaved", + walletId: WALLET, + destinationWalletId: "dest", + percent: 20, + enabled: true, + createdAt: new Date("2026-01-01T00:00:00Z"), + }, + ]) + await post(paths.invoice, lightningBody()) + + expect(mockStore.rows.size).toBe(1) + }) + + it("does not fire a rule created after the receive in the webhook body", async () => { + mockListRules.mockResolvedValue([ + { + id: "rule-new", + walletId: WALLET, + destinationWalletId: "dest", + percent: 20, + enabled: true, + createdAt: new Date("2026-10-11T12:01:00Z"), + }, + ]) + + const res = await post(paths.invoice, { + transaction: { + accountId: WALLET, + amount: 10, + invoice: { hash: HASH }, + createdAt: "2026-10-11T12:00:00Z", + }, + receivedMsat: 1000, + }) + + expect(res.statusCode).toBe(200) + expect(mockStore.rows.size).toBe(0) + }) + + it("fires a rule created before the receive in the webhook body", async () => { + const res = await post(paths.invoice, { + transaction: { + accountId: WALLET, + amount: 10, + invoice: { hash: HASH }, + settledAt: "2026-10-11T12:00:00Z", + }, + receivedMsat: 1000, + }) + + expect(res.statusCode).toBe(200) + expect(mockStore.rows.size).toBe(1) + }) + it.each([ ["onchain", paths.onchain], ["zap", paths.zap], From 9180eacbf309522fe3547ffad82bf26a2a210fd6 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 11 Oct 2026 23:48:04 -0700 Subject: [PATCH 13/13] fix(payment-splits): retry addInvoice failures, keep rule createdAt across re-saves - Thread a beforePay hook through the intraledger send. Only an IBEX error after payInvoice was reached is ambiguous (`unknown` + ops alert). An addInvoice failure or missing bolt11 moved nothing and is retried within maxAttempts (a definitive 4xx still fails). - replaceForWallet carries createdAt forward for destinations that survive the replace, so a re-save no longer drops a settled-but-unrecorded receive at reconcile time. - Integration spec: dedupe is on (triggerKey, walletId, destinationWalletId); a rule-rewrite replay is a no-op, another destination is not; index shape asserted; createdAt carry-forward and the reconcile-after-re-save case. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01G4SaLJF5SWq6sjBEaWJFow --- .../payment-splits/execute-pending-splits.ts | 62 ++++++--- src/app/payments/send-intraledger.ts | 4 + src/app/wallets/index.types.d.ts | 5 + src/services/mongoose/payment-split-rules.ts | 22 ++- .../payment-splits-repositories.spec.ts | 129 +++++++++++++++++- .../execute-pending-splits.spec.ts | 97 ++++++++++++- .../app/payments/send-intraledger.spec.ts | 69 ++++++++++ 7 files changed, 359 insertions(+), 29 deletions(-) diff --git a/src/app/payment-splits/execute-pending-splits.ts b/src/app/payment-splits/execute-pending-splits.ts index bd3185080..80cf99d2f 100644 --- a/src/app/payment-splits/execute-pending-splits.ts +++ b/src/app/payment-splits/execute-pending-splits.ts @@ -29,14 +29,29 @@ export const PAYMENT_SPLIT_MEMO = "Payment split" export const splitIdempotencyKey = (executionId: string): IdempotencyKey => `split:${executionId}` as IdempotencyKey +type SendProgress = { + // The ENG-573 guard (the `authorize` hook) passed. withPaymentIdempotency + // runs it inside the idempotency lock just before `execute`. + paymentAttempted: boolean + // The send reached IBEX payInvoice (the `beforePay` hook ran). Before this, + // only IBEX addInvoice (minting the recipient invoice) and the + // internal-transfer marker write ran, and neither moves money. + payInvoiceReached?: boolean +} + /** * Errors worth another attempt: raised before IBEX paid anything and expected * to clear on their own (rate limits, a send guard that could not reach Redis - * or a price, the internal-transfer marker write, an IBEX 429, and a busy - * idempotency lock that was never acquired). Everything else — limits, KYC - * level, insufficient balance, an inactive account, an unknown IBEX error — is - * terminal on the first attempt, so an error whose payment outcome is unknown - * is never re-sent. + * or a price, the internal-transfer marker write, an IBEX 429, a busy + * idempotency lock that was never acquired, and any IBEX error from minting + * the recipient invoice that is not a definitive 4xx). Everything else + * (limits, KYC level, insufficient balance, an inactive account, an IBEX error + * from payInvoice) is terminal on the first attempt, so an error whose payment + * outcome is unknown is never re-sent. + * + * An addInvoice failure is safe to retry: no payment was attempted and the + * error return is left uncached by withPaymentIdempotency, so the same key + * re-runs the whole send. * * Lock errors are the subtle case. redlock also returns a LockServiceError when * the locked routine throws or when releasing the lock fails, and both can @@ -47,16 +62,19 @@ export const splitIdempotencyKey = (executionId: string): IdempotencyKey => */ export const isRetryableSplitError = ( error: Error, - { paymentAttempted }: { paymentAttempted: boolean }, + { paymentAttempted, payInvoiceReached = false }: SendProgress, ): boolean => { if (error instanceof LockServiceError) { return error instanceof ResourceAttemptsLockServiceError && !paymentAttempted } + if (error instanceof IbexError) { + if (error.httpCode === 429) return true + return paymentAttempted && !payInvoiceReached && !isDefinitivePaymentRejection(error) + } return ( error instanceof RateLimitServiceError || error instanceof SendLimitsUnavailableError || - error instanceof InternalTransferMarkerError || - (error instanceof IbexError && error.httpCode === 429) + error instanceof InternalTransferMarkerError ) } @@ -67,21 +85,25 @@ export const isRetryableSplitError = ( * * - A lock error that is not a retryable pre-payment one (see * isRetryableSplitError). - * - Once the ENG-573 guard passed (`paymentAttempted`), any IBEX error that is - * not a definitive rejection: a transport error, a timeout or a 5xx from - * payInvoice, where IBEX may already have debited (the KNOWN GAP in + * - Once the send reached IBEX payInvoice (`payInvoiceReached`), any IBEX error + * that is not a definitive rejection: a transport error, a timeout or a 5xx + * from payInvoice, where IBEX may already have debited (the KNOWN GAP in * idempotency.ts). A definitive rejection (insufficient balance, a 4xx such - * as no route or a 429) moved no money and stays `failed` / retryable. + * as no route or a 429) moved no money and stays `failed` / retryable. An + * IBEX error from addInvoice (before payInvoice) moved nothing either and is + * retried. */ export const isAmbiguousSplitSendError = ( error: Error, - { paymentAttempted }: { paymentAttempted: boolean }, + { paymentAttempted, payInvoiceReached = false }: SendProgress, ): boolean => { if (error instanceof LockServiceError) { - return !isRetryableSplitError(error, { paymentAttempted }) + return !isRetryableSplitError(error, { paymentAttempted, payInvoiceReached }) } return ( - paymentAttempted && error instanceof IbexError && !isDefinitivePaymentRejection(error) + payInvoiceReached && + error instanceof IbexError && + !isDefinitivePaymentRejection(error) ) } @@ -204,6 +226,9 @@ const executeOne = async ( // inside the idempotency lock immediately before the payment, so this is the // last point at which no money can have moved yet. let paymentAttempted = false + // Set right before IBEX payInvoice. Between the two only IBEX addInvoice and + // the marker write run, and neither moves money. + let payInvoiceReached = false const result = await intraledgerPaymentSendWalletIdForUsdWallet({ senderWalletId: wallet.id, @@ -227,10 +252,13 @@ const executeOne = async ( if (!(authorized instanceof Error)) paymentAttempted = true return authorized }, + beforePay: () => { + payInvoiceReached = true + }, }) if (result instanceof Error) { - if (isAmbiguousSplitSendError(result, { paymentAttempted })) { + if (isAmbiguousSplitSendError(result, { paymentAttempted, payInvoiceReached })) { await finish({ status: "unknown", attempts, lastError: result.name }) logger.error( { executionId: execution.id, walletId: execution.walletId, error: result }, @@ -251,7 +279,7 @@ const executeOne = async ( return "unknown" } if ( - isRetryableSplitError(result, { paymentAttempted }) && + isRetryableSplitError(result, { paymentAttempted, payInvoiceReached }) && attempts < PaymentSplitsConfig.maxAttempts ) { return retry(result) diff --git a/src/app/payments/send-intraledger.ts b/src/app/payments/send-intraledger.ts index 1008dab0c..245b6376d 100644 --- a/src/app/payments/send-intraledger.ts +++ b/src/app/payments/send-intraledger.ts @@ -65,6 +65,7 @@ const intraledgerPaymentSendWalletId = async ({ amount: uncheckedAmount, memo, senderWalletId: uncheckedSenderWalletId, + beforePay, }: IntraLedgerPaymentSendWalletIdArgs): Promise => { const validatedPaymentInputs = await validateIntraledgerPaymentInputs({ uncheckedSenderWalletId, @@ -108,6 +109,9 @@ const intraledgerPaymentSendWalletId = async ({ }) if (marked instanceof Error) return marked + // Everything above moved no money: an error from addInvoice or the marker + // write is a pre-payment error. From here on IBEX may debit the sender. + beforePay?.() const payResp = await Ibex.payInvoice({ accountId: uncheckedSenderWalletId, invoice: invoiceResp.invoice.bolt11 as Bolt11, diff --git a/src/app/wallets/index.types.d.ts b/src/app/wallets/index.types.d.ts index 89c367da9..f7e8f323c 100644 --- a/src/app/wallets/index.types.d.ts +++ b/src/app/wallets/index.types.d.ts @@ -148,6 +148,11 @@ type IntraLedgerPaymentSendWalletIdArgs = PaymentSendArgs & { // `SendGuardHook` above. Required — pass `SEND_GUARD_NOT_APPLICABLE` to opt a // system credit out explicitly. authorize: SendGuardHook + // Called synchronously right before IBEX payInvoice, after the recipient + // invoice was minted and marked internal. Lets a caller tell an error that + // came before the payment (nothing moved, safe to retry) from one that came + // from the payment itself (outcome unknown). Optional; no other effect. + beforePay?: () => void } type PayOnChainByWalletIdResult = { diff --git a/src/services/mongoose/payment-split-rules.ts b/src/services/mongoose/payment-split-rules.ts index b56f57d85..50950d220 100644 --- a/src/services/mongoose/payment-split-rules.ts +++ b/src/services/mongoose/payment-split-rules.ts @@ -66,21 +66,39 @@ export const PaymentSplitRulesRepository = () => { // Replace semantics: the new set becomes the wallet's whole rule set. Callers // hold the wallet lock so two concurrent replaces cannot interleave. + // + // createdAt is carried forward for every destination that survives the + // replace; only a destination new to the set gets createdAt = now. The + // recorder skips receives that settled before a rule's createdAt, so + // re-stamping an unchanged destination on every save would drop a receive + // that settled before the save but was not yet recorded (a webhook whose + // record failed, waiting for the reconcile). const replaceForWallet = async ( walletId: string, rules: { destinationWalletId: string; percent: number }[], + { now = new Date() }: { now?: Date } = {}, ): Promise => { try { + const existing = await PaymentSplitRuleModel.find({ walletId: { $eq: walletId } }) + .lean() + .exec() + const createdAtByDestination = new Map() + for (const rule of existing) { + const known = createdAtByDestination.get(rule.destinationWalletId) + if (!known || rule.createdAt.getTime() < known.getTime()) { + createdAtByDestination.set(rule.destinationWalletId, rule.createdAt) + } + } + await PaymentSplitRuleModel.deleteMany({ walletId: { $eq: walletId } }) if (rules.length === 0) return [] - const now = new Date() const docs = await PaymentSplitRuleModel.insertMany( rules.map(({ destinationWalletId, percent }) => ({ walletId, destinationWalletId, percent, enabled: true, - createdAt: now, + createdAt: createdAtByDestination.get(destinationWalletId) ?? now, updatedAt: now, })), ) diff --git a/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts b/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts index 254ffb8a9..440e910e8 100644 --- a/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts +++ b/test/flash/integration/payment-splits/payment-splits-repositories.spec.ts @@ -1,3 +1,7 @@ +import { PaymentSplitsConfig } from "@config" + +import { recordSplitExecutions } from "@app/payment-splits/record-split-executions" + import { INTERNAL_TRANSFER_INVOICE_TTL_SECONDS, InternalTransferInvoice, @@ -13,11 +17,15 @@ import { import { PaymentSplitExecutionsRepository } from "@services/mongoose/payment-split-executions" import { PaymentSplitRulesRepository } from "@services/mongoose/payment-split-rules" -const executionFields = (triggerKey: string, ruleId: string) => ({ +const executionFields = ( + triggerKey: string, + ruleId: string, + { destinationWalletId = "dest-wallet" }: { destinationWalletId?: string } = {}, +) => ({ triggerKey, ruleId, walletId: "source-wallet", - destinationWalletId: "dest-wallet", + destinationWalletId, currency: "USDT", triggerAmountMinor: "1000000", shareCents: 10, @@ -68,18 +76,129 @@ describe("payment split repositories (ENG-653)", () => { expect(await rules.listWalletIdsWithEnabledRules()).toEqual([]) }) - it("rejects a duplicate (triggerKey, ruleId) as a typed no-op", async () => { + it("rejects a duplicate (triggerKey, walletId, destinationWalletId) as a typed no-op", async () => { const repo = PaymentSplitExecutionsRepository() const triggerKey = "d".repeat(64) const first = await repo.insert(executionFields(triggerKey, "rule-1")) const second = await repo.insert(executionFields(triggerKey, "rule-1")) - const otherRule = await repo.insert(executionFields(triggerKey, "rule-2")) + // A webhook replay after the rules were re-saved: same payout target under + // a new rule id. Still a duplicate, so the partner is not paid twice. + const rewrittenRule = await repo.insert(executionFields(triggerKey, "rule-2")) + // Another destination of the same receive is a payout of its own. + const otherDestination = await repo.insert( + executionFields(triggerKey, "rule-3", { destinationWalletId: "dest-wallet-2" }), + ) expect(first).toMatchObject({ inserted: true }) expect(second).toEqual({ inserted: false }) - expect(otherRule).toMatchObject({ inserted: true }) + expect(rewrittenRule).toEqual({ inserted: false }) + expect(otherDestination).toMatchObject({ inserted: true }) expect(await PaymentSplitExecution.countDocuments({ triggerKey })).toBe(2) + + const indexes = await PaymentSplitExecution.collection.indexes() + expect(indexes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: { triggerKey: 1, walletId: 1, destinationWalletId: 1 }, + unique: true, + }), + ]), + ) + expect( + indexes.find( + (index) => + index.unique && + JSON.stringify(index.key) === JSON.stringify({ triggerKey: 1, ruleId: 1 }), + ), + ).toBeUndefined() + }) + + it("carries createdAt forward for a destination that survives a re-save", async () => { + const rules = PaymentSplitRulesRepository() + const firstSave = new Date("2026-10-11T12:00:00Z") + const reSave = new Date("2026-10-11T12:10:00Z") + + const first = await rules.replaceForWallet( + "wallet-b", + [ + { destinationWalletId: "kept", percent: 10 }, + { destinationWalletId: "dropped", percent: 20 }, + ], + { now: firstSave }, + ) + if (first instanceof Error) throw first + + const second = await rules.replaceForWallet( + "wallet-b", + [ + { destinationWalletId: "kept", percent: 15 }, + { destinationWalletId: "new", percent: 5 }, + ], + { now: reSave }, + ) + if (second instanceof Error) throw second + + const byDestination = new Map(second.map((rule) => [rule.destinationWalletId, rule])) + expect(byDestination.get("kept")).toMatchObject({ + percent: 15, + createdAt: firstSave, + updatedAt: reSave, + }) + expect(byDestination.get("new")).toMatchObject({ createdAt: reSave }) + expect(byDestination.has("dropped")).toBe(false) + }) + + it("records an unrecorded receive that settled before a re-save of the same destination", async () => { + const enabled = PaymentSplitsConfig.enabled + PaymentSplitsConfig.enabled = true + try { + const rules = PaymentSplitRulesRepository() + const firstSave = new Date("2026-10-11T11:00:00Z") + const receivedAt = new Date("2026-10-11T12:00:00Z") + const reSave = new Date("2026-10-11T12:02:00Z") + const reconcileAt = new Date("2026-10-11T12:15:00Z") + + // The partner has been on the wallet since 11:00. + const saved = await rules.replaceForWallet( + "wallet-c", + [{ destinationWalletId: "partner", percent: 10 }], + { now: firstSave }, + ) + if (saved instanceof Error) throw saved + + // A receive settles at 12:00 and the webhook's record fails. At 12:02 the + // merchant re-saves, adding a destination and keeping the partner. + const reSaved = await rules.replaceForWallet( + "wallet-c", + [ + { destinationWalletId: "partner", percent: 10 }, + { destinationWalletId: "late-partner", percent: 5 }, + ], + { now: reSave }, + ) + if (reSaved instanceof Error) throw reSaved + + // The reconcile at 12:15 records the partner's share. The destination + // added after the receive is still skipped. + const triggerKey = "c".repeat(64) + const result = await recordSplitExecutions({ + walletId: "wallet-c", + triggerKey, + paymentHash: triggerKey, + amount: 100, + currency: "USDT", + receivedAt, + now: reconcileAt, + }) + if (result instanceof Error) throw result + + expect(result).toMatchObject({ recorded: 1 }) + const rows = await PaymentSplitExecution.find({ triggerKey }).lean() + expect(rows.map((row) => row.destinationWalletId)).toEqual(["partner"]) + } finally { + PaymentSplitsConfig.enabled = enabled + } }) it("lets exactly one of several concurrent claimers claim a pending row", async () => { diff --git a/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts index 0842e684a..ccfe6196f 100644 --- a/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts +++ b/test/flash/unit/app/payment-splits/execute-pending-splits.spec.ts @@ -73,7 +73,11 @@ import { InternalTransferMarkerError, splitRetryBackoffMs } from "@domain/paymen import { SendLimitsUnavailableError } from "@domain/payments/errors" import { PaymentSendRateLimiterExceededError } from "@domain/rate-limit/errors" import { WalletCurrency } from "@domain/shared" -import { IbexError, InsufficientIbexBalance } from "@services/ibex/errors" +import { + IbexError, + InsufficientIbexBalance, + UnexpectedIbexResponse, +} from "@services/ibex/errors" import { createFakeExecutionsStore } from "./fake-executions-store" @@ -422,13 +426,18 @@ describe("executePendingSplits", () => { }) }) - describe("IBEX errors after the send guard passed", () => { - // authorize passed, so Ibex.payInvoice was called: anything but a definite - // rejection may have debited (the KNOWN GAP in idempotency.ts). + describe("IBEX errors from payInvoice", () => { + // authorize passed and the beforePay hook ran, so Ibex.payInvoice was + // called: anything but a definite rejection may have debited (the KNOWN + // GAP in idempotency.ts). const afterAuthorize = (error: Error) => - async (args: { authorize: () => Promise }): Promise => { + async (args: { + authorize: () => Promise + beforePay?: () => void + }): Promise => { await args.authorize() + args.beforePay?.() return error } @@ -490,6 +499,84 @@ describe("executePendingSplits", () => { }) }) + describe("IBEX errors from addInvoice (after the guard, before payInvoice)", () => { + // authorize passed but the send never reached Ibex.payInvoice: only the + // recipient invoice was being minted, so nothing moved. + const beforePayInvoice = + (error: Error) => + async (args: { authorize: () => Promise }): Promise => { + await args.authorize() + return error + } + + const httpError = (httpCode?: number) => { + const err = new IbexError(new Error(`IBEX ${httpCode ?? "ECONNRESET"}`)) + if (httpCode !== undefined) { + Object.defineProperty(err, "httpCode", { value: httpCode }) + } + return err + } + + it("passes a beforePay hook to the send", async () => { + await seed() + mockSend.mockResolvedValue(PaymentSendStatus.Success) + + await executePendingSplits({ now: T0 }) + + expect(mockSend.mock.calls[0][0]).toEqual( + expect.objectContaining({ beforePay: expect.any(Function) }), + ) + }) + + it.each([ + ["a 500", httpError(500)], + ["a transport error with no status", httpError()], + ["a missing bolt11", new UnexpectedIbexResponse("Bolt11 field not found.")], + ])( + "retries %s as pending with no ops alert and no owner push", + async (_label, error) => { + await seed() + mockSend.mockImplementationOnce(beforePayInvoice(error)) + + const summary = await executePendingSplits({ now: T0 }) + + expect(summary).toMatchObject({ claimed: 1, retried: 1, unknown: 0, failed: 0 }) + expect(onlyRow()).toMatchObject({ status: "pending", attempts: 1 }) + expect(mockNotifyOps).not.toHaveBeenCalled() + expect(mockNotify).not.toHaveBeenCalled() + + // The retry re-runs the send and pays. + mockSend.mockResolvedValue(PaymentSendStatus.Success) + await executePendingSplits({ now: minutes(1) }) + expect(onlyRow()).toMatchObject({ status: "sent", attempts: 2 }) + }, + ) + + it("fails once the attempt budget is spent", async () => { + await seed() + mockSend.mockImplementation(beforePayInvoice(httpError(500))) + + let now = T0 + for (const backoff of [1, 2, 4, 8]) { + await executePendingSplits({ now }) + now = new Date(now.getTime() + backoff * 60_000) + } + await executePendingSplits({ now }) + + expect(onlyRow()).toMatchObject({ status: "failed", attempts: 5 }) + expect(mockNotifyOps).not.toHaveBeenCalled() + }) + + it("fails a definitive 400 from addInvoice at once", async () => { + await seed() + mockSend.mockImplementation(beforePayInvoice(httpError(400))) + + expect(await executePendingSplits({ now: T0 })).toMatchObject({ failed: 1 }) + expect(onlyRow()).toMatchObject({ status: "failed", attempts: 1 }) + expect(mockNotifyOps).not.toHaveBeenCalled() + }) + }) + describe("repository errors before the send", () => { it.each([ ["the wallet lookup", "wallet"], diff --git a/test/flash/unit/app/payments/send-intraledger.spec.ts b/test/flash/unit/app/payments/send-intraledger.spec.ts index f8b27da9d..73256aee8 100644 --- a/test/flash/unit/app/payments/send-intraledger.spec.ts +++ b/test/flash/unit/app/payments/send-intraledger.spec.ts @@ -938,3 +938,72 @@ describe("internal-transfer marker (ENG-653)", () => { expect(mockPayInvoice).not.toHaveBeenCalled() }) }) + +describe("intraledger beforePay hook (ENG-653)", () => { + // The payment-split executor uses this hook to tell an error from minting the + // recipient invoice (nothing moved, safe to retry) from one from payInvoice. + beforeEach(() => { + jest.clearAllMocks() + + mockFindAccountById.mockImplementation(async (accountId: AccountId) => + activeAccount(accountId as string), + ) + mockAddInvoice.mockResolvedValue(ibexAddInvoiceResponse) + mockPayInvoice.mockResolvedValue(ibexPayResponse(2)) + mockRecordInternalTransferInvoice.mockResolvedValue(true) + mockFindWalletById.mockImplementation(async (walletId: WalletId) => + wallet({ + id: walletId, + accountId: + walletId === senderUsdWalletId ? "sender-account" : "recipient-account", + currency: WalletCurrency.Usd, + }), + ) + }) + + const sendArgs = { + senderWalletId: senderUsdWalletId, + recipientWalletId: recipientUsdWalletId, + amount: 100, + memo: "before pay", + authorize: allowSend, + } + + it("runs right before payInvoice, after the invoice was minted and marked", async () => { + const order: string[] = [] + mockAddInvoice.mockImplementation(async () => { + order.push("addInvoice") + return ibexAddInvoiceResponse + }) + mockRecordInternalTransferInvoice.mockImplementation(async () => { + order.push("marker") + return true + }) + mockPayInvoice.mockImplementation(async () => { + order.push("payInvoice") + return ibexPayResponse(2) + }) + + const result = await intraledgerPaymentSendWalletIdForUsdWallet({ + ...sendArgs, + beforePay: () => order.push("beforePay"), + }) + + expect(result).toEqual({ value: "success" }) + expect(order).toEqual(["addInvoice", "marker", "beforePay", "payInvoice"]) + }) + + it("does not run when addInvoice fails", async () => { + const beforePay = jest.fn() + mockAddInvoice.mockResolvedValue(new Error("IBEX 500")) + + const result = await intraledgerPaymentSendWalletIdForUsdWallet({ + ...sendArgs, + beforePay, + }) + + expect(result).toBeInstanceOf(Error) + expect(beforePay).not.toHaveBeenCalled() + expect(mockPayInvoice).not.toHaveBeenCalled() + }) +})