From 311433c2d2ee040d657881df356f003745bd2b35 Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 08:58:52 -0400 Subject: [PATCH 01/11] feat: record inventory with shared reservations and concurrent swap reselection Port of Veil PR #172 (head 2e324d5) into the Python SDK. aleo.inventory: get/plan/rebalance/reconcile record inventory, join_records, split_record, auto_join over credits.aleo and ARC-20/ARC-22 tokens. One sans-IO engine drives both RecordInventory and AsyncRecordInventory. The planner, join-helper registry, and output resolution follow Veil; its planner tests are the known answers. RecordInventoryStore is a three-method protocol with an in-memory implementation; scopes match Veil's so journals can be shared. Facade: Aleo(provider, record_management=...) and aleo.inventory. On a managed client every transact and delegate reserves its record inputs before proving, saves the proved transaction before broadcast, journals submission, and enforces the fee ceiling. Private fees pick and reserve the credits record once the exact fee is known (fee_record may be a callable). A node rejected broadcast releases its reservation; uncertain outcomes keep it. transact_with_result returns the built transaction beside the id. Shield Swap: plan_inventory and rebalance_inventory on the token's underlying program, reserved records excluded from selection, and swaps that lose a record to another writer move to the next eligible record before proving. Concurrent-swaps example with a file-backed store and recovery steps. AGENTS.md regenerated. --- README.md | 58 ++ sdk/Readme.md | 17 + sdk/python/aleo/__init__.py | 19 + sdk/python/aleo/__init__.pyi | 19 + sdk/python/aleo/facade/async_client.py | 100 +- sdk/python/aleo/facade/call.py | 107 +- sdk/python/aleo/facade/client.py | 36 +- sdk/python/aleo/inventory/__init__.py | 88 ++ sdk/python/aleo/inventory/_engine.py | 793 ++++++++++++++ sdk/python/aleo/inventory/assets.py | 251 +++++ sdk/python/aleo/inventory/client.py | 705 +++++++++++++ sdk/python/aleo/inventory/interface.py | 179 ++++ sdk/python/aleo/inventory/join_programs.py | 209 ++++ sdk/python/aleo/inventory/planner.py | 204 ++++ sdk/python/aleo/inventory/store.py | 119 +++ sdk/python/aleo/inventory/types.py | 584 +++++++++++ .../tests/fixtures/main_aj_arc20_2_15.aleo | 389 +++++++ sdk/python/tests/test_facade_inventory.py | 253 +++++ sdk/python/tests/test_inventory_assets.py | 247 +++++ sdk/python/tests/test_inventory_engine.py | 968 ++++++++++++++++++ sdk/python/tests/test_inventory_planner.py | 152 +++ sdk/python/tests/test_inventory_store.py | 103 ++ shield-swap-sdk/AGENTS.md | 42 + shield-swap-sdk/README.md | 48 + shield-swap-sdk/codegen/gen_context.py | 1 + .../examples/concurrent-swaps/README.md | 143 +++ .../concurrent-swaps/concurrent_swaps.py | 264 +++++ .../python/aleo_shield_swap/AGENTS.md | 42 + .../python/aleo_shield_swap/_calls.py | 102 +- .../python/aleo_shield_swap/_core.py | 144 ++- .../python/aleo_shield_swap/async_client.py | 156 ++- .../python/aleo_shield_swap/client.py | 161 ++- .../tests/test_concurrent_swaps_example.py | 121 +++ shield-swap-sdk/tests/test_gen_context.py | 4 +- shield-swap-sdk/tests/test_inventory.py | 437 ++++++++ 35 files changed, 7151 insertions(+), 114 deletions(-) create mode 100644 sdk/python/aleo/inventory/__init__.py create mode 100644 sdk/python/aleo/inventory/_engine.py create mode 100644 sdk/python/aleo/inventory/assets.py create mode 100644 sdk/python/aleo/inventory/client.py create mode 100644 sdk/python/aleo/inventory/interface.py create mode 100644 sdk/python/aleo/inventory/join_programs.py create mode 100644 sdk/python/aleo/inventory/planner.py create mode 100644 sdk/python/aleo/inventory/store.py create mode 100644 sdk/python/aleo/inventory/types.py create mode 100644 sdk/python/tests/fixtures/main_aj_arc20_2_15.aleo create mode 100644 sdk/python/tests/test_facade_inventory.py create mode 100644 sdk/python/tests/test_inventory_assets.py create mode 100644 sdk/python/tests/test_inventory_engine.py create mode 100644 sdk/python/tests/test_inventory_planner.py create mode 100644 sdk/python/tests/test_inventory_store.py create mode 100644 shield-swap-sdk/examples/concurrent-swaps/README.md create mode 100644 shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py create mode 100644 shield-swap-sdk/tests/test_concurrent_swaps_example.py create mode 100644 shield-swap-sdk/tests/test_inventory.py diff --git a/README.md b/README.md index 52085ac4..b7fdca93 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,64 @@ already held by the application, or assign a custom `RecordProvider` to `aleo.record_provider`. Choose local proving separately if transaction contents must also stay out of the proving service. +## Manage private records + +A private balance is a set of records, and one transaction spends whole +records. A large payment needs one record that covers it; several payments +submitted at the same time need several records, one each. `aleo.inventory` +reads an asset's spendable records, plans the joins and splits that reach a +chosen layout, executes that plan one confirmed transaction at a time, and +keeps writers that share a journal from spending the same record. + +Build the client with record management enabled so every `transact` and +`delegate` reserves its record inputs before proving and saves the proved +transaction before broadcast. The example below reshapes the account's private +credits into four equal records of at least one credit each: + +```python +from aleo import Aleo, HTTPProvider +from aleo.inventory import InventoryTarget, RecordAsset, RecordManagementConfig + +aleo = Aleo(HTTPProvider("https://edge.provable.com/api", network="testnet"), + record_management=RecordManagementConfig(enabled=True)) +aleo.default_account = account +aleo.records.register(account) # The scanner can decrypt the account's records. + +credits = RecordAsset("credits.aleo") +snapshot = aleo.inventory.get_record_inventory(credits) +print(snapshot.balance, len(snapshot.available), len(snapshot.reserved)) + +plan = aleo.inventory.plan_record_inventory( + credits, InventoryTarget(records=4, min_record_amount=1_000_000, distribution="balanced")) +print(plan.deduction) # Microcredits the native splits remove; network fees are separate. +result = aleo.inventory.rebalance_record_inventory(plan, max_fee_microcredits=1_000_000) +``` + +Planning reads the scanner and the deployed program and submits nothing. +Execution requires record management to be enabled. Each step reserves its +inputs, proves, broadcasts, waits for confirmation, and waits until the scanner +shows the exact outputs before the next step spends them. No accepted step is +rolled back. An `interrupted` result names the reason and every submitted +transaction. After an interruption, reconcile, rescan, and plan again; the old +plan describes records that the completed steps have already spent. + +More than two records join through deployed helper programs in one +transaction where one exists (up to 16 credits or dedicated stablecoin records, +15 for other ARC-20 and ARC-22 tokens). `join_records`, `split_record`, and +`auto_join` perform single operations without a plan. + +The default journal is in memory and lasts for one process. Supply any object +with `list`, `acquire`, and `update` methods as `store` to share reservations +across processes or to resume after a restart; `acquire` must reserve all of an +entry's records or none. After an interruption, `reconcile_record_inventory()` +checks each pending entry against the chain and, with `rebroadcast=True`, +resubmits a saved proved transaction with its original id. A write that finds +its record already reserved raises `RecordReservationConflictError` before any +proof is built, so the caller can choose another record. + +The [Shield Swap guide](shield-swap-sdk/README.md#concurrent-swaps) shows the +same mechanism preparing records for several trades submitted at once. + ## Bridge assets Bring assets from Ethereum or Solana to Aleo for use in Aleo applications, or diff --git a/sdk/Readme.md b/sdk/Readme.md index f67ed448..33c95418 100644 --- a/sdk/Readme.md +++ b/sdk/Readme.md @@ -164,6 +164,23 @@ credits.functions.transfer_private(record, str(recipient.address), 1) \ `aleo.record_provider` is swappable: set it to your own object implementing the `RecordProvider` protocol (`get_unspent_credits_record` / `find`), and the whole facade (including private-fee auto-sourcing) uses it, with no view-key sharing. +### Record inventory — join, split, plan, reserve, recover + +One transaction spends whole records, so a large payment needs one record that covers it and several payments submitted together need one record each. `aleo.inventory` reads an asset's spendable records, plans the joins and splits that reach a target layout, and executes that plan one confirmed transaction at a time. Build the client with `record_management` enabled to make every `transact` and `delegate` reserve its record inputs before proving and save the proved transaction before broadcast; writers sharing the same store never spend the same record. + +```python +from aleo.inventory import InventoryTarget, RecordAsset, RecordManagementConfig + +aleo = Aleo(HTTPProvider(ENDPOINT, network="testnet"), + record_management=RecordManagementConfig(enabled=True)) +credits = RecordAsset("credits.aleo") +plan = aleo.inventory.plan_record_inventory( + credits, InventoryTarget(records=4, min_record_amount=1_000_000, distribution="balanced")) +result = aleo.inventory.rebalance_record_inventory(plan, max_fee_microcredits=1_000_000) +``` + +`get_record_inventory` reads; `plan_record_inventory` plans without signing; `join_records`, `split_record`, and `auto_join` run single operations; `reconcile_record_inventory` settles pending journal entries against the chain and can rebroadcast a saved proved transaction. The journal is in memory by default; pass any object with `list`, `acquire`, and `update` as `store` for a database or file. The top-level [README](../README.md#manage-private-records) explains the choices and the recovery steps. + ## Async (`AsyncAleo`) The async facade mirrors the sync surface. Construction and the local, CPU-bound steps stay synchronous; everything that touches the network is awaitable. diff --git a/sdk/python/aleo/__init__.py b/sdk/python/aleo/__init__.py index c1b0aa02..8850e4df 100644 --- a/sdk/python/aleo/__init__.py +++ b/sdk/python/aleo/__init__.py @@ -47,6 +47,25 @@ ExecutionError as ExecutionError, TransactionConfirmationTimeout as TransactionConfirmationTimeout, ) +from .inventory import ( + AsyncRecordInventory as AsyncRecordInventory, + InventoryError as InventoryError, + InventoryPlan as InventoryPlan, + InventoryPlanError as InventoryPlanError, + InventoryRecord as InventoryRecord, + InventoryResult as InventoryResult, + InventoryTarget as InventoryTarget, + MemoryRecordInventoryStore as MemoryRecordInventoryStore, + RecordAsset as RecordAsset, + RecordInventory as RecordInventory, + RecordInventorySnapshot as RecordInventorySnapshot, + RecordInventoryStore as RecordInventoryStore, + RecordJoinProgram as RecordJoinProgram, + RecordManagementConfig as RecordManagementConfig, + RecordReservationConflictError as RecordReservationConflictError, + RecordSpend as RecordSpend, + TokenJoinRouter as TokenJoinRouter, +) def __getattr__(name: str) -> object: diff --git a/sdk/python/aleo/__init__.pyi b/sdk/python/aleo/__init__.pyi index 9a275022..a624bb9c 100644 --- a/sdk/python/aleo/__init__.pyi +++ b/sdk/python/aleo/__init__.pyi @@ -28,6 +28,25 @@ from .facade.errors import ( ExecutionError as ExecutionError, TransactionConfirmationTimeout as TransactionConfirmationTimeout, ) +from .inventory import ( + AsyncRecordInventory as AsyncRecordInventory, + InventoryError as InventoryError, + InventoryPlan as InventoryPlan, + InventoryPlanError as InventoryPlanError, + InventoryRecord as InventoryRecord, + InventoryResult as InventoryResult, + InventoryTarget as InventoryTarget, + MemoryRecordInventoryStore as MemoryRecordInventoryStore, + RecordAsset as RecordAsset, + RecordInventory as RecordInventory, + RecordInventorySnapshot as RecordInventorySnapshot, + RecordInventoryStore as RecordInventoryStore, + RecordJoinProgram as RecordJoinProgram, + RecordManagementConfig as RecordManagementConfig, + RecordReservationConflictError as RecordReservationConflictError, + RecordSpend as RecordSpend, + TokenJoinRouter as TokenJoinRouter, +) def consensus_version(block_height: Optional[int] = None) -> int: ... diff --git a/sdk/python/aleo/facade/async_client.py b/sdk/python/aleo/facade/async_client.py index d5a6c6df..ec63bc0a 100644 --- a/sdk/python/aleo/facade/async_client.py +++ b/sdk/python/aleo/facade/async_client.py @@ -899,15 +899,51 @@ async def transact( private_fee: bool = False, base_fee: int | None = None, ) -> str: - """Build and broadcast; return the transaction id (async).""" - result = await self.build_transaction( - account, - priority_fee=priority_fee, - fee_record=fee_record, - private_fee=private_fee, - base_fee=base_fee, + """Build and broadcast; return the transaction id (async). + + On a client built with record management enabled, record inputs are + reserved before proving and the proved transaction is saved before + broadcast; see :meth:`~aleo.facade.call.BoundCall.transact`. + """ + tx_id, _ = await self.transact_with_result( + account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, + ) + return tx_id + + async def transact_with_result( + self, + account: Any = None, + *, + priority_fee: int = 0, + fee_record: Any = None, + private_fee: bool = False, + base_fee: int | None = None, + ) -> tuple[str, TransactionResult]: + """Build, broadcast, and also return the built transaction (async). + + See :meth:`~aleo.facade.call.BoundCall.transact_with_result`. + """ + from .call import _managed_inventory, _fee_record_text, _prepared_from_result # pyright: ignore[reportPrivateUsage] + managed = _managed_inventory(self._client) + if managed is None: + result = await self.build_transaction( + account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, + ) + return await self._client.network.submit_transaction(result.raw), result + built: list[TransactionResult] = [] + + async def build(request: Any) -> Any: + result = await self.build_transaction( + account, priority_fee=priority_fee, fee_record=request.fee_record, + private_fee=request.private_fee, base_fee=base_fee, + ) + built.append(result) + return _prepared_from_result(result) + tx_id = await managed.managed_write( + self.program_id, self.function_name, self.args, account=account, + private_fee=private_fee or fee_record is not None, fee_record=_fee_record_text(fee_record), builder=build, ) - return await self._client.network.submit_transaction(result.raw) + return tx_id, built[0] async def delegate( self, @@ -923,8 +959,27 @@ async def delegate( By default ``fee_authorization=None`` — the prover's fee master pays. Self-paid fees require proving locally (blocking, in ``asyncio.to_thread``). The only self-paid fee knob is *priority_fee*; delegate never overrides - the base fee. + the base fee. On a client built with record management enabled, a + broadcasting call reserves its record inputs, has the prover return + the transaction, saves it, and submits from this client. """ + from .call import _managed_inventory, _fee_record_text, _prepared_from_payload # pyright: ignore[reportPrivateUsage] + managed = _managed_inventory(self._client) if broadcast else None + if managed is not None: + payloads: list[Any] = [] + + async def build(request: Any) -> Any: + payload = await self.delegate( + account, broadcast=False, pay_own_fee=pay_own_fee, fee_record=request.fee_record, + priority_fee=priority_fee, + ) + payloads.append(payload) + return _prepared_from_payload(payload) + await managed.managed_write( + self.program_id, self.function_name, self.args, account=account, + private_fee=fee_record is not None, fee_record=_fee_record_text(fee_record), builder=build, + ) + return payloads[0] net = self._net() auth = self._build_authorization(account) @@ -1021,6 +1076,10 @@ async def _resolve_fee_record_async( self, fee_record: Any, *, min_microcredits: int | None = None ) -> Any: net = self._net() + if callable(fee_record): + # Record management chooses the fee record once the exact fee is + # known; the callable reserves it and returns its plaintext. + fee_record = fee_record(int(min_microcredits or 0)) if fee_record is not None: if isinstance(fee_record, str): return net.RecordPlaintext.from_string(fee_record) @@ -1124,17 +1183,26 @@ class AsyncAleo: ---------- provider: An :class:`~aleo.facade.provider.HTTPProvider`. + record_management: + Optional :class:`~aleo.inventory.RecordManagementConfig`; see + :class:`~aleo.facade.client.Aleo`. With ``enabled=True`` every + awaited ``transact`` and ``delegate`` reserves its record inputs and + journals its progress. """ # Expose HTTPProvider as a nested class attribute for ``AsyncAleo.HTTPProvider(...)`` HTTPProvider = HTTPProvider - def __init__(self, provider: object) -> None: + def __init__(self, provider: object, *, record_management: Any = None) -> None: if not isinstance(provider, HTTPProvider): raise TypeError( f"provider must be an HTTPProvider, got {type(provider).__name__}" ) self._provider: HTTPProvider = provider + self._record_management: Any = record_management + # Built eagerly so every task sharing the client sees one journal. + from ..inventory.client import AsyncRecordInventory + self._inventory: Any = AsyncRecordInventory(self, record_management) # Async network client (httpx-based) self._async_client: Any = provider._build_async_client() # pyright: ignore[reportPrivateUsage] self._process: Any = None # lazy @@ -1213,6 +1281,18 @@ def record_provider(self, provider: Any) -> None: """ self._record_provider = provider + # ── Record inventory ───────────────────────────────────────────────────── + + @property + def record_management(self) -> Any: + """The :class:`~aleo.inventory.RecordManagementConfig` this client was built with, or ``None``.""" + return self._record_management + + @property + def inventory(self) -> Any: + """The :class:`~aleo.inventory.AsyncRecordInventory` bound to this client.""" + return self._inventory + # ── Network identity (sync — local) ──────────────────────────────────── @property diff --git a/sdk/python/aleo/facade/call.py b/sdk/python/aleo/facade/call.py index a0729f68..61b97f00 100644 --- a/sdk/python/aleo/facade/call.py +++ b/sdk/python/aleo/facade/call.py @@ -337,6 +337,10 @@ def _resolve_fee_record( private fee is requested but no record can be sourced. """ net = self._net() + if callable(fee_record): + # Record management chooses the fee record once the exact fee is + # known; the callable reserves it and returns its plaintext. + fee_record = fee_record(int(min_microcredits or 0)) if fee_record is not None: if isinstance(fee_record, str): return net.RecordPlaintext.from_string(fee_record) @@ -457,19 +461,63 @@ def transact( ) -> str: """Build the transaction (:meth:`build_transaction`) and broadcast it. + On a client built with record management enabled, the call's record + inputs are reserved before proving, the proved transaction is saved + before broadcast, and a private fee record is chosen and reserved + once the exact fee is known. A record another writer holds raises + :class:`~aleo.inventory.RecordReservationConflictError` before any + proof is built. + Returns ------- str The transaction id returned by the node. """ - result = self.build_transaction( - account, - priority_fee=priority_fee, - fee_record=fee_record, - private_fee=private_fee, - base_fee=base_fee, + return self.transact_with_result( + account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, + )[0] + + def transact_with_result( + self, + account: Any = None, + *, + priority_fee: int = 0, + fee_record: Any = None, + private_fee: bool = False, + base_fee: int | None = None, + ) -> tuple[str, TransactionResult]: + """Build, broadcast, and also return the built transaction. + + The same as :meth:`transact`, for callers that read the transaction's + outputs without waiting for confirmation. The returned result is the + transaction as proved locally; on a managed client it is also the + transaction saved to the journal before broadcast. + + Returns + ------- + tuple[str, TransactionResult] + The transaction id returned by the node and the built transaction. + """ + managed = _managed_inventory(self._client) + if managed is None: + result = self.build_transaction( + account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, + ) + return self._client.network.submit_transaction(result.raw), result + built: list[TransactionResult] = [] + + def build(request: Any) -> Any: + result = self.build_transaction( + account, priority_fee=priority_fee, fee_record=request.fee_record, + private_fee=request.private_fee, base_fee=base_fee, + ) + built.append(result) + return _prepared_from_result(result) + tx_id = managed.managed_write( + self.program_id, self.function_name, self.args, account=account, + private_fee=private_fee or fee_record is not None, fee_record=_fee_record_text(fee_record), builder=build, ) - return self._client.network.submit_transaction(result.raw) + return tx_id, built[0] # ── Verb: delegate (the flagship DPS path) ────────────────────────────── @@ -511,7 +559,28 @@ def delegate( ------- Any The prover's result payload (the data dict) from the DPS. + + On a client built with record management enabled, a broadcasting + call asks the prover not to broadcast, reserves the record inputs + before proving, saves the returned transaction, and submits it from + this client; the payload then carries the proved ``transaction``. """ + managed = _managed_inventory(self._client) if broadcast else None + if managed is not None: + payloads: list[Any] = [] + + def build(request: Any) -> Any: + payload = self.delegate( + account, broadcast=False, pay_own_fee=pay_own_fee, fee_record=request.fee_record, + priority_fee=priority_fee, + ) + payloads.append(payload) + return _prepared_from_payload(payload) + managed.managed_write( + self.program_id, self.function_name, self.args, account=account, + private_fee=fee_record is not None, fee_record=_fee_record_text(fee_record), builder=build, + ) + return payloads[0] net = self._net() auth = self._build_authorization(account) @@ -546,6 +615,30 @@ def delegate( return self._client.network_client.submit_proving_request(request) +# ── Record management hook helpers (shared with the async client) ──────────── + + +def _managed_inventory(client: Any) -> Any: + """The client's inventory when record management is enabled, else ``None``.""" + from ..inventory.client import managed_inventory + return managed_inventory(client) + + +def _fee_record_text(fee_record: Any) -> str | None: + """A caller's preferred fee record as plaintext, for the journal's selector.""" + return None if fee_record is None else str(fee_record) + + +def _prepared_from_result(result: Any) -> Any: + from ..inventory.client import prepared_from_result + return prepared_from_result(result) + + +def _prepared_from_payload(payload: Any) -> Any: + from ..inventory.client import prepared_from_payload + return prepared_from_payload(payload) + + __all__ = [ "BoundCall", "AuthorizationResult", diff --git a/sdk/python/aleo/facade/client.py b/sdk/python/aleo/facade/client.py index e626c03b..5db7bc03 100644 --- a/sdk/python/aleo/facade/client.py +++ b/sdk/python/aleo/facade/client.py @@ -35,18 +35,31 @@ class Aleo: provider: A :class:`~aleo.facade.provider.HTTPProvider` instance that configures the underlying network connection. + record_management: + Optional :class:`~aleo.inventory.RecordManagementConfig`. With + ``enabled=True`` every ``transact`` and ``delegate`` on this client + reserves its record inputs before proving, saves the proved + transaction before broadcast, and journals the submission, so + writers sharing the store never spend the same record. ``None`` + (the default) leaves writes unchanged; ``aleo.inventory`` still + reads, plans, and performs single joins or splits. """ # Expose HTTPProvider as a nested class attribute so callers can write # ``Aleo.HTTPProvider(...)`` without a separate import. HTTPProvider = HTTPProvider - def __init__(self, provider: object) -> None: + def __init__(self, provider: object, *, record_management: Any = None) -> None: if not isinstance(provider, HTTPProvider): raise TypeError( f"provider must be an HTTPProvider, got {type(provider).__name__}" ) self._provider: HTTPProvider = provider + self._record_management: Any = record_management + # Built eagerly so every thread sharing the client sees one journal; + # construction reads nothing from the network. + from ..inventory.client import RecordInventory + self._inventory: Any = RecordInventory(self, record_management) # Access the build helper via name to avoid pyright's private-access error; # _build_client is intentionally package-private (single leading underscore). self._client: Any = provider._build_client() # pyright: ignore[reportPrivateUsage] @@ -140,6 +153,27 @@ def record_provider(self, provider: Any) -> None: """ self._record_provider = provider + # ── Record inventory ───────────────────────────────────────────────────── + + @property + def record_management(self) -> Any: + """The :class:`~aleo.inventory.RecordManagementConfig` this client was built with, or ``None``. + + Fixed at construction so every writer sharing the client uses one + journal and scope. Build a new client to change it. + """ + return self._record_management + + @property + def inventory(self) -> Any: + """The :class:`~aleo.inventory.RecordInventory` bound to this client. + + Reads, plans, joins, splits, reservations, and recovery for private + records, configured by ``record_management``; with no configuration it + reads and plans but reserves nothing. + """ + return self._inventory + # ── Network identity ─────────────────────────────────────────────────── @property diff --git a/sdk/python/aleo/inventory/__init__.py b/sdk/python/aleo/inventory/__init__.py new file mode 100644 index 00000000..f4a2808d --- /dev/null +++ b/sdk/python/aleo/inventory/__init__.py @@ -0,0 +1,88 @@ +"""Private record inventory: inspect, join, split, plan, rebalance, reserve, recover. + +A private balance is a set of encrypted records. One large spend needs a +record that covers it; several concurrent spends need several records. This +package reads one asset's spendable records, plans the joins and splits that +reach a target layout, executes that plan one confirmed transaction at a +time, keeps participating writers from spending the same record, and +recovers an interrupted run without rebuilding a proof. + +Bind it to a client with ``Aleo(provider, record_management=...)`` and use +``aleo.inventory``, or construct :class:`RecordInventory` directly. +""" +from __future__ import annotations + +from ._engine import ( + PreparedTransaction as PreparedTransaction, + inventory_output_commitments as inventory_output_commitments, + record_input_ids as record_input_ids, + transaction_fee as transaction_fee, +) +from .assets import ( + CREDITS_SPLIT_DEDUCTION as CREDITS_SPLIT_DEDUCTION, + AssetCapabilities as AssetCapabilities, + decode_record as decode_record, + inventory_record_id as inventory_record_id, + scope_for as scope_for, + validate_asset as validate_asset, +) +from .interface import ( + ProgramFunction as ProgramFunction, + ProgramInterface as ProgramInterface, + ProgramParam as ProgramParam, + ProgramRecord as ProgramRecord, + RecordField as RecordField, + parse_program_interface as parse_program_interface, +) +from .join_programs import ( + RECORD_JOIN_PROGRAMS as RECORD_JOIN_PROGRAMS, + validate_record_join_program as validate_record_join_program, + validate_token_join_router as validate_token_join_router, +) +from .client import ( + AsyncRecordInventory as AsyncRecordInventory, + RecordInventory as RecordInventory, + SpendCheckpoint as SpendCheckpoint, + managed_inventory as managed_inventory, + prepared_from_payload as prepared_from_payload, + prepared_from_result as prepared_from_result, +) +from .planner import build_inventory_plan as build_inventory_plan +from .store import ( + MemoryRecordInventoryStore as MemoryRecordInventoryStore, + held_record_ids as held_record_ids, + holds_records as holds_records, +) +from .types import ( + InventoryError as InventoryError, + InventoryPlan as InventoryPlan, + InventoryPlanError as InventoryPlanError, + InventoryRecord as InventoryRecord, + InventoryResult as InventoryResult, + InventoryStep as InventoryStep, + InventoryTarget as InventoryTarget, + PlannedRecord as PlannedRecord, + RecordAsset as RecordAsset, + RecordInventorySnapshot as RecordInventorySnapshot, + RecordInventoryStore as RecordInventoryStore, + RecordJoinProgram as RecordJoinProgram, + RecordManagementConfig as RecordManagementConfig, + RecordReservationConflictError as RecordReservationConflictError, + RecordSpend as RecordSpend, + RecordSpendPatch as RecordSpendPatch, + ResolvedRecordAsset as ResolvedRecordAsset, + TokenJoinRouter as TokenJoinRouter, +) + +__all__ = [ + "AssetCapabilities", "AsyncRecordInventory", "CREDITS_SPLIT_DEDUCTION", "InventoryError", "InventoryPlan", + "InventoryPlanError", "InventoryRecord", "InventoryResult", "InventoryStep", "InventoryTarget", + "MemoryRecordInventoryStore", "PlannedRecord", "PreparedTransaction", "ProgramFunction", "ProgramInterface", + "ProgramParam", "ProgramRecord", "RECORD_JOIN_PROGRAMS", "RecordAsset", "RecordField", "RecordInventory", + "RecordInventorySnapshot", "RecordInventoryStore", "RecordJoinProgram", "RecordManagementConfig", + "RecordReservationConflictError", "RecordSpend", "RecordSpendPatch", "ResolvedRecordAsset", "SpendCheckpoint", + "TokenJoinRouter", "build_inventory_plan", "decode_record", "held_record_ids", "holds_records", + "inventory_output_commitments", "inventory_record_id", "managed_inventory", "parse_program_interface", + "prepared_from_payload", "prepared_from_result", "record_input_ids", "scope_for", "transaction_fee", + "validate_asset", "validate_record_join_program", "validate_token_join_router", +] diff --git a/sdk/python/aleo/inventory/_engine.py b/sdk/python/aleo/inventory/_engine.py new file mode 100644 index 00000000..86be18e3 --- /dev/null +++ b/sdk/python/aleo/inventory/_engine.py @@ -0,0 +1,793 @@ +"""Inventory orchestration, written once for the sync and async clients. + +Every action here is a generator. Instead of calling the facade, it ``yield``s +a small request object (fetch a program, scan records, prove, broadcast, +look up a confirmation, sleep) and receives the answer from whichever client +is driving it. :class:`~aleo.inventory.client.RecordInventory` performs the +requests synchronously; :class:`~aleo.inventory.client.AsyncRecordInventory` +awaits them. The rules that protect funds (reserve before proving, save the +proved transaction before broadcast, keep a reservation when an outcome is +uncertain, feed only confirmed outputs into later steps) therefore exist in +one place and behave identically on both clients. + +Reading the generators: ``value = yield Request(...)`` is a facade call whose +result is ``value``; ``yield from self.other(...)`` composes actions like an +ordinary call. Errors raised by the facade are thrown into the generator at +the ``yield`` that requested the work, so a ``try`` around a ``yield`` catches +them where they matter. +""" +from __future__ import annotations + +import re +import time +from dataclasses import dataclass, replace +from typing import Any, Callable, Generator, Literal, Mapping, Sequence, TypeVar, Union, cast + +from .._client_common import AleoNetworkError +from .assets import ( + CREDITS_PROGRAM, AssetCapabilities, amount_width, decode_record, inventory_record_id, max_amount, scope_for, + split_deduction, validate_asset, +) +from .interface import parse_program_interface +from .join_programs import ( + join_program_candidates, router_as_helper, supported_join_program, validate_record_join_function, + validate_token_join_function, validate_token_join_router, +) +from .planner import build_inventory_plan, positive_integer +from .store import reserved_record_ids +from .types import ( + InventoryError, InventoryPlan, InventoryRecord, InventoryResult, InventoryStep, InventoryTarget, PlannedRecord, + RecordAsset, RecordInventorySnapshot, RecordInventoryStore, RecordJoinProgram, RecordManagementConfig, + RecordReservationConflictError, RecordSpend, ResolvedRecordAsset, TokenJoinRouter, +) + +T = TypeVar("T") + +# ── Requests the engine yields and the clients perform ─────────────────────── + + +@dataclass(frozen=True) +class ProgramSource: + """Fetch a deployed program's source; the answer is the text, or ``None`` when it is not deployed.""" + + program_id: str + + +@dataclass(frozen=True) +class FindRecords: + """Scan the account's unspent records of one program; the answer is the scanner's record list.""" + + program: str + + +@dataclass(frozen=True) +class PreparedTransaction: + """A proved transaction that has not been broadcast. + + Attributes: + id: The transaction id. + transaction: The transaction as JSON data, saved to the journal before + broadcast so recovery can resubmit it. + """ + + id: str + transaction: dict[str, Any] + + +FeeRecordSelector = Callable[[int], str] + + +@dataclass(frozen=True) +class BuildTransaction: + """Prove a call without broadcasting; the answer is a :class:`PreparedTransaction`. + + Attributes: + program: Program to call. + function: Transition name. + inputs: Aleo-encoded inputs, record plaintext included. + imports: Programs reached by dynamic dispatch that the prover must + register beside the called program. + private_fee: Pay the fee from a credits record instead of the public + balance. + fee_record: Under a private fee, a callable that receives the exact + fee in microcredits once the execution is proved and returns the + plaintext of the credits record to spend. The engine reserves that + record inside the callable. + """ + + program: str + function: str + inputs: tuple[str, ...] + imports: tuple[str, ...] = () + private_fee: bool = False + fee_record: FeeRecordSelector | None = None + + +@dataclass(frozen=True) +class Broadcast: + """Submit a proved transaction; the answer is the transaction id the node returned.""" + + transaction: Mapping[str, Any] + + +@dataclass(frozen=True) +class ConfirmedTransaction: + """Look up a confirmed transaction; the answer is its JSON, or ``None`` while the node has none.""" + + tx_id: str + + +@dataclass(frozen=True) +class Sleep: + """Pause between polls.""" + + seconds: float + + +@dataclass(frozen=True) +class WaitForScannerSync: + """Wait until the record scanner reports itself synced, or raise after *timeout* seconds.""" + + timeout: float + + +Request = Union[ProgramSource, FindRecords, BuildTransaction, Broadcast, ConfirmedTransaction, Sleep, WaitForScannerSync] +Step = Generator[Request, Any, T] +JsonDict = dict[str, Any] + + +class _Default: + """Marker for "use the engine's configured value" where ``None`` is itself a valid setting.""" + + +DEFAULT = _Default() + + +def _dict(value: Any) -> JsonDict: + """A JSON object from untyped data, or an empty one.""" + return cast(JsonDict, value) if isinstance(value, dict) else {} + + +def _list(value: Any) -> list[Any]: + """A JSON array from untyped data, or an empty one.""" + return cast(list[Any], value) if isinstance(value, list) else [] + + +_RECORD_LITERAL = re.compile(r"^\s*\{.*_nonce:\s*\d+group", re.DOTALL) +_U64 = re.compile(r"^\d+u64$") +_OWNER = re.compile(r"(? "Management": + if config.token_join: + validate_token_join_router(config.token_join) + if config.max_fee_microcredits is not None and config.max_fee_microcredits < 0: + raise InventoryError("Fee limit cannot be negative") + return Management( + enabled=config.enabled, store=default_store if config.enabled else None, + chain_id=config.chain_id or network, max_fee_microcredits=config.max_fee_microcredits, + token_join=config.token_join, wait_for_sync=config.wait_for_sync, sync_timeout=config.sync_timeout, + ) + + +# ── Pure helpers used by the engine and the facade hook ────────────────────── + + +def transaction_fee(transaction: Mapping[str, Any]) -> int: + """Base plus priority fee in microcredits, read from the proved transaction. + + Both fee functions publish the amounts; ``fee_private`` puts its consumed + credits record before them. + + Raises: + InventoryError: The fee transition is missing or malformed, so it + cannot be compared with a spending limit. + """ + fee = _dict(transaction.get("fee")) + if not fee: + return 0 + transition = _dict(fee.get("transition")) + offset = 1 if transition.get("function") == "fee_private" else 0 + inputs = _list(transition.get("inputs")) + values = [_dict(item).get("value") for item in inputs[offset:offset + 2]] + amounts = [value for value in values if isinstance(value, str) and _U64.match(value)] + if len(values) != 2 or len(amounts) != 2: + raise InventoryError("Cannot verify transaction fee before broadcast") + return sum(int(value[:-3]) for value in amounts) + + +def inventory_output_commitments( + transaction: Mapping[str, Any], asset: RecordAsset, step: InventoryStep, router: TokenJoinRouter | None, +) -> list[str]: + """Record commitments a confirmed step produced, in planned output order. + + Native joins and splits expose their commitments directly. A static helper + returns an external record and a dynamic router returns a dynamic id, so + for both the commitment the scanner indexes comes from the final nested + native ``join``, the second-to-last transition. + + Raises: + InventoryError: The outer transition does not match the step, or the + nested output cannot be matched unambiguously. + """ + transitions = [_dict(item) for item in _list(_dict(transaction.get("execution")).get("transitions"))] + outer = transitions[-1] if transitions else {} + helper = step.join_program if step.kind == "join" else None + routed = step.kind == "join" and asset.program != CREDITS_PROGRAM and router is not None + target = helper.program if helper else (router.program if routed and router else asset.program) + expected = f"join_{len(step.inputs)}" if (helper or routed) else step.kind + if not outer or outer.get("program") != target or outer.get("function") != expected: + raise InventoryError("Confirmed transition does not match the inventory plan") + outputs = [_dict(item) for item in _list(outer.get("outputs"))] + final = transitions[-2] if len(transitions) >= 2 else {} + joined = [_dict(item) for item in _list(final.get("outputs"))] + nested_join = final.get("program") == asset.program and final.get("function") == "join" and len(joined) == 1 + if helper is not None and helper.input_mode == "records": + # A static helper returns an external record; the final nested native + # join produces the encrypted record the scanner indexes. + if ( + len(outputs) != 1 or outputs[0].get("type") != "external_record" + or not nested_join or joined[0].get("type") not in ("record", "record_with_dynamic_id") + ): + raise InventoryError("Cannot resolve confirmed helper output to the underlying record commitment") + return [str(joined[0]["id"])] + if helper is None and not routed: + return [str(item["id"]) for item in outputs if item.get("type") in ("record", "record_with_dynamic_id")] + if len(outputs) != 1 or outputs[0].get("type") != "record_dynamic": + raise InventoryError("Unexpected confirmed router output") + # The router's own return slot carries a dynamic id, not a scanner + # commitment; the final native join holds the encrypted record. + if not nested_join or joined[0].get("type") != "record_with_dynamic_id": + raise InventoryError("Cannot resolve confirmed router output to the underlying token commitment") + return [str(joined[0]["id"])] + + +def record_input_ids(inputs: Sequence[str], address: str) -> list[str]: + """Nonces of the record plaintext inputs the account owns. + + Raises: + InventoryError: An input is a record ciphertext (no nonce to reserve), + belongs to another account, or repeats. + """ + ids: list[str] = [] + for item in inputs: + if item.startswith("record1"): + raise InventoryError("Managed spending requires a record nonce; ciphertext alone is insufficient") + if not _RECORD_LITERAL.match(item): + continue + owner = _OWNER.search(item) + if owner is None or owner.group(1) != address: + raise InventoryError("Cannot reserve another account's record") + ids.append(inventory_record_id({"record_plaintext": item})) + if len(set(ids)) != len(ids): + raise InventoryError("A transaction cannot spend the same record twice") + return ids + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +# ── The engine ─────────────────────────────────────────────────────────────── + + +class InventoryEngine: + """Sans-IO record inventory actions bound to one network, account, and management setting. + + Args: + network: Network name, used for helper lookup and the default scope. + address: Callable returning the signer's current address, so a + changed default account is noticed mid-run. + management: Resolved management settings. + clock: Monotonic clock in seconds, injectable for tests. + """ + + def __init__( + self, network: str, address: Callable[[], str], management: Management, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.network = network + self._address = address + self.management = management + self.clock = clock + + # ── Scope and reservations ─────────────────────────────────────────────── + + @property + def address(self) -> str: + return self._address() + + @property + def scope(self) -> str: + return scope_for(self.management.chain_id, self.address) + + def reserved_record_ids(self) -> set[str]: + """Nonces excluded by the journal; empty while management is disabled.""" + return reserved_record_ids(self.management.store, self.scope) + + def _store(self, action: str) -> RecordInventoryStore: + if not self.management.enabled or self.management.store is None: + raise InventoryError(f"{action} requires record management to be enabled with a shared store") + return self.management.store + + # ── Reads ──────────────────────────────────────────────────────────────── + + def _validate_asset(self, asset: RecordAsset) -> Step[AssetCapabilities]: + source = None if asset.program == CREDITS_PROGRAM else (yield ProgramSource(asset.program)) + return validate_asset(asset, source) + + def _find_records(self, program: str) -> Step[list[Mapping[str, Any]]]: + if self.management.wait_for_sync: + yield WaitForScannerSync(self.management.sync_timeout) + records: list[Mapping[str, Any]] = yield FindRecords(program) + return records + + def get_record_inventory(self, asset: RecordAsset) -> Step[RecordInventorySnapshot]: + capabilities = yield from self._validate_asset(asset) + excluded = self.reserved_record_ids() + records = yield from self._find_records(asset.program) + seen: dict[str, InventoryRecord] = {} + address = self.address + for record in records: + decoded = decode_record(address, asset, record, capabilities) + if decoded is not None and decoded.id not in seen: + seen[decoded.id] = decoded + available = tuple(item for item in seen.values() if item.id not in excluded) + reserved = tuple(item for item in seen.values() if item.id in excluded) + return RecordInventorySnapshot(capabilities.asset, available, reserved, sum(item.amount for item in available)) + + def _join_programs(self, asset: ResolvedRecordAsset, token_join: TokenJoinRouter | Literal[False] | None) -> Step[tuple[RecordJoinProgram, ...]]: + available: list[RecordJoinProgram] = [] + for helper in join_program_candidates(self.network, asset, token_join): + # A missing deployment permits native joins; a transport outage + # propagates instead of silently changing the transaction count. + source = yield ProgramSource(helper.program) + if source is None: + continue + supported = supported_join_program(helper, parse_program_interface(source), asset) + if supported is not None: + available.append(supported) + return tuple(available) + + def plan_record_inventory( + self, asset: RecordAsset, target: InventoryTarget, max_transactions: int | None = None, + ) -> Step[InventoryPlan]: + inventory = yield from self.get_record_inventory(asset) + helpers: tuple[RecordJoinProgram, ...] = () + if len(inventory.available) > 2: + helpers = yield from self._join_programs(inventory.asset, self.management.token_join) + # Keep only nonces and amounts in the plan; plaintext stays in the + # scanner result and is fetched again when each step executes. + return build_inventory_plan( + scope=self.scope, asset=inventory.asset, target=target, + inputs=[PlannedRecord(item.id, item.amount) for item in inventory.available], + max_transactions=max_transactions, token_join=self.management.token_join, join_programs=helpers, + ) + + # ── Single writes ──────────────────────────────────────────────────────── + + def join_records( + self, asset: RecordAsset, records: Sequence[InventoryRecord], join_program: RecordJoinProgram | Literal[False] | None = None, + *, token_join: TokenJoinRouter | Literal[False] | None | _Default = DEFAULT, + fee_limit: int | None | _Default = DEFAULT, observed: list[str] | None = None, + ) -> Step[str]: + router_policy = self.management.token_join if isinstance(token_join, _Default) else token_join + capabilities = yield from self._validate_asset(asset) + resolved = capabilities.asset + router = router_policy if (resolved.program != CREDITS_PROGRAM and router_policy) else None + helper: RecordJoinProgram | None + if join_program is None and router: + validate_token_join_router(router) + source = yield ProgramSource(router.program) + if source is None: + raise InventoryError(f"{router.program} is not deployed") + validate_token_join_function(router, parse_program_interface(source), len(records)) + helper = router_as_helper(router) + elif join_program is None and len(records) > 2: + helpers = yield from self._join_programs(resolved, router_policy) + helper = next((item for item in helpers if len(records) in item.counts), None) + elif join_program is False: + helper = None + else: + helper = join_program + if helper is not None: + source = yield ProgramSource(helper.program) + if source is None: + raise InventoryError(f"{helper.program} is not deployed") + validate_record_join_function(helper, parse_program_interface(source), resolved, len(records)) + elif len(records) != 2: + raise InventoryError( + "join_records requires two records without an available join helper; use auto_join for several transactions" + ) + # Re-decode before proving; cached amounts do not establish ownership, + # eligibility, or distinct inputs. + address = self.address + decoded = [decode_record(address, asset, item.record, capabilities) for item in records] + eligible = [item for item in decoded if item is not None] + if len(eligible) != len(records) or len({item.id for item in eligible}) != len(eligible): + raise InventoryError("Join requires distinct eligible records owned by this account") + if sum(item.amount for item in eligible) > max_amount(resolved): + raise InventoryError("Joined amount overflows the token width") + dynamic = helper is not None and helper.input_mode == "program_identifier_then_records" + inputs = [f"'{resolved.program[:-5]}'"] if dynamic else [] + inputs.extend(item.plaintext for item in eligible) + return (yield from self._spend( + helper.program if helper else resolved.program, f"join_{len(eligible)}" if helper else "join", inputs, + imports=(resolved.program,) if dynamic else (), + asset_program=resolved.program if helper else None, fee_limit=fee_limit, observed=observed, + )) + + def split_record( + self, asset: RecordAsset, record: InventoryRecord, amount: int, + *, fee_limit: int | None | _Default = DEFAULT, observed: list[str] | None = None, + ) -> Step[str]: + capabilities = yield from self._validate_asset(asset) + resolved = capabilities.asset + decoded = decode_record(self.address, asset, record.record, capabilities) + # Both outputs must stay positive after the native split's own deduction. + if decoded is None or isinstance(amount, bool) or amount <= 0 or amount + split_deduction(resolved) >= decoded.amount: + raise InventoryError("Split must produce two positive eligible records") + return (yield from self._spend( + resolved.program, "split", [decoded.plaintext, f"{amount}{amount_width(resolved)}"], + fee_limit=fee_limit, observed=observed, + )) + + # ── Managed spending ───────────────────────────────────────────────────── + + def _spend( + self, program: str, function: str, inputs: Sequence[str], *, imports: Sequence[str] = (), + asset_program: str | None = None, private_fee: bool = False, preferred_fee_record: str | None = None, + fee_limit: int | None | _Default = DEFAULT, observed: list[str] | None = None, + ) -> Step[str]: + """Prove and broadcast one call, reserving its inputs first when management is enabled. + + Returns the transaction id. With management enabled the journal + records the reservation, the proved transaction, and the submission; + a failure before the transaction is proved releases the reservation, + a failure afterwards keeps it for reconciliation. + """ + address = self.address + ids = record_input_ids(inputs, address) + limit = self.management.max_fee_microcredits if isinstance(fee_limit, _Default) else fee_limit + if not self.management.enabled or self.management.store is None: + prepared = yield BuildTransaction(program, function, tuple(inputs), tuple(imports), private_fee, None) + if observed is not None: + observed.append(prepared.id) + return (yield Broadcast(prepared.transaction)) + store = self.management.store + scope = self.scope + base = RecordSpend( + id="", scope=scope, records=tuple(ids), program=program, function=function, status="reserved", + created_at=_now_ms(), asset_program=asset_program, + ) + # Acquire every input nonce together before any proving starts. + entry = store.acquire(base) + if entry is None: + raise RecordReservationConflictError(ids) + entries = [entry] + fee_choice: list[InventoryRecord] = [] + selector: FeeRecordSelector | None = None + if private_fee: + credits = yield from self.get_record_inventory(RecordAsset(CREDITS_PROGRAM)) + preferred = inventory_record_id({"record_plaintext": preferred_fee_record}) if preferred_fee_record else None + candidates = sorted( + (item for item in credits.available if item.id not in ids and item.plaintext and item.tag), + key=lambda item: (item.id != preferred, item.amount, item.id), + ) + + def select(required: int) -> str: + # Called once the execution is proved and the exact fee is + # known. The chosen record joins the journal as its own entry, + # so a rejected execution can keep it held while releasing the + # inputs. + if limit is not None and required > limit: + raise InventoryError("Estimated transaction exceeds max_fee_microcredits") + if self.scope != scope: + raise InventoryError("Account or chain changed during private fee selection") + for candidate in candidates: + if candidate.amount < required: + continue + fee_entry = store.acquire(replace(base, records=(candidate.id,), fee_record_id=candidate.id)) + if fee_entry is not None: + entries.append(fee_entry) + fee_choice.append(candidate) + return candidate.plaintext + raise InventoryError( + "No available credits record covers the private fee separately from the transaction inputs; " + "refresh records or fund private credits" + ) + + selector = select + prepared_flag = False + submitted = False + try: + prepared = yield BuildTransaction(program, function, tuple(inputs), tuple(imports), private_fee, selector) + if self.scope != scope: + raise InventoryError("Account or chain changed during record spending") + transaction = prepared.transaction + if fee_choice: + # Check the prover honoured the pinned record before saving a + # transaction that recovery could broadcast. + chosen = fee_choice[0] + fee_transition = _dict(_dict(transaction.get("fee")).get("transition")) + fee_inputs = _list(fee_transition.get("inputs")) + first = _dict(fee_inputs[0]) if fee_inputs else {} + if ( + fee_transition.get("program") != CREDITS_PROGRAM or fee_transition.get("function") != "fee_private" + or first.get("type") != "record" or first.get("tag") != chosen.tag + ): + raise InventoryError("Prover did not spend the reserved private fee record") + fee = transaction_fee(transaction) + if fee_choice and fee > fee_choice[0].amount: + raise InventoryError("Private fee exceeds the reserved credits record balance") + if limit is not None and fee > limit: + raise InventoryError("Proved transaction exceeds max_fee_microcredits") + for item in entries: + store.update(scope, item.id, { + "status": "prepared", "transaction_id": prepared.id, "transaction": transaction, "fee_microcredits": str(fee), + }) + prepared_flag = True + if observed is not None: + observed.append(prepared.id) + try: + tx_id = yield Broadcast(transaction) + except AleoNetworkError as exc: + # A 4xx from the broadcast endpoint means the node refused the + # transaction before admitting it, so nothing can reach the + # chain and the inputs can be released. Any other failure + # (timeout, lost response, 5xx) is uncertain and keeps them. + if exc.status is not None and 400 <= exc.status < 500: + prepared_flag = False + raise + # Mark submission before touching the store: a storage failure must + # not release inputs the network may already have accepted. + submitted = True + for item in entries: + store.update(scope, item.id, {"status": "submitted", "transaction_id": tx_id}) + return tx_id + except BaseException as exc: + # Nothing was broadcast before the prepared checkpoint; afterwards + # the outcome is uncertain and the reservations stay. + if not prepared_flag and not submitted: + for item in entries: + store.update(scope, item.id, {"status": "cancelled"}) + if isinstance(exc, RecordReservationConflictError): + # A nested write or a callback raised its own conflict after this + # spend acquired its inputs; it must not authorise a retry. + raise InventoryError("Record spending failed after acquiring inputs") from exc + raise + + def managed_write( + self, program: str, function: str, inputs: Sequence[str], *, imports: Sequence[str] = (), + private_fee: bool = False, fee_record: str | None = None, + ) -> Step[str]: + """Run one caller-built write through the journal; used by the facade's managed verbs.""" + return (yield from self._spend( + program, function, inputs, imports=imports, private_fee=private_fee or fee_record is not None, + preferred_fee_record=fee_record, + )) + + # ── Plan execution and recovery ────────────────────────────────────────── + + def rebalance_record_inventory( + self, plan: InventoryPlan, *, timeout: float = 120.0, poll_interval: float = 1.0, + max_fee_microcredits: int | None = None, cancelled: Callable[[], bool] | None = None, + ) -> Step[InventoryResult]: + store = self._store("Inventory execution") + # Rebuild instead of trusting caller-supplied amounts or dependencies. + rebuilt = build_inventory_plan( + scope=plan.scope, asset=plan.asset, target=plan.target, inputs=plan.inputs, + max_transactions=plan.max_transactions, token_join=plan.token_join, join_programs=plan.join_programs, + ) + if rebuilt != plan: + raise InventoryError("Inventory plan was modified; plan again") + if plan.scope != self.scope: + raise InventoryError("Inventory plan belongs to another account or chain") + if timeout <= 0 or poll_interval <= 0: + raise InventoryError("timeout and poll_interval must be positive") + if max_fee_microcredits is not None and max_fee_microcredits < 0: + raise InventoryError("Fee budget cannot be negative") + transaction_ids: list[str] = [] + completed = 0 + # Plan inputs already use nonces; symbolic step outputs acquire real + # nonces only after confirmation and scanner visibility. + references = {item.id: item.id for item in plan.inputs} + planned = {item.id: item for item in plan.inputs} + for step in plan.steps: + planned.update({item.id: item for item in step.outputs}) + fees = 0 + + def check() -> None: + if cancelled is not None and cancelled(): + raise InventoryError("Inventory execution cancelled; unresolved reservations retained") + if self.scope != plan.scope: + raise InventoryError("Account or chain changed during inventory execution") + + try: + capabilities = (yield from self._validate_asset(plan.asset)) if plan.steps else None + for step in plan.steps: + check() + # Rescan before every step so a competing spend or a changed + # amount stops execution before a dependent proof is built. + inventory = yield from self.get_record_inventory(plan.asset) + inputs: list[InventoryRecord] = [] + for ref in step.inputs: + record = next((item for item in inventory.available if item.id == references.get(ref)), None) + expected = planned.get(ref) + if record is None or expected is None or record.amount != expected.amount: + raise InventoryError("Inventory changed; reconcile and plan again") + inputs.append(record) + check() + # The stricter of the remaining run budget and the per-transaction ceiling. + remaining = None if max_fee_microcredits is None else max_fee_microcredits - fees + if remaining is not None and remaining <= 0: + # Stop before reserving or proving a step the budget cannot pay for. + raise InventoryError("Fee budget exhausted; remaining steps were not submitted") + per_transaction = self.management.max_fee_microcredits + if remaining is None: + limit = per_transaction + elif per_transaction is None or remaining < per_transaction: + limit = remaining + else: + limit = per_transaction + if step.kind == "join": + tx_id = yield from self.join_records( + plan.asset, inputs, step.join_program if step.join_program else (None if plan.token_join else False), + token_join=plan.token_join, fee_limit=limit, observed=transaction_ids, + ) + else: + assert step.amount is not None + tx_id = yield from self.split_record(plan.asset, inputs[0], step.amount, fee_limit=limit, observed=transaction_ids) + if tx_id not in transaction_ids: + transaction_ids.append(tx_id) + entry = next((item for item in store.list(plan.scope) if item.transaction_id == tx_id and item.fee_record_id is None), None) + if entry is None: + raise InventoryError("Submitted transaction is missing its reservation") + fees += int(entry.fee_microcredits or "0") + deadline = self.clock() + timeout + confirmed: Mapping[str, Any] | None = None + # A missing transaction may still be propagating; keep its + # reservation while polling and stop if acceptance cannot be + # established within the deadline. + while confirmed is None: + check() + if self.clock() >= deadline: + raise InventoryError(f"Timed out confirming {tx_id}; reservation retained") + confirmed = yield ConfirmedTransaction(tx_id) + if confirmed is None: + yield Sleep(poll_interval) + if confirmed.get("status") != "accepted": + if confirmed.get("status") == "rejected": + store.update(plan.scope, entry.id, {"status": "rejected", "transaction": None}) + raise InventoryError(f"Transaction {tx_id} was not accepted") + store.update(plan.scope, entry.id, {"status": "confirmed", "transaction": None}) + commitments = inventory_output_commitments(_dict(confirmed.get("transaction")), plan.asset, step, plan.token_join) + if len(commitments) != len(step.outputs): + raise InventoryError("Unexpected confirmed inventory output count") + # The scanner must return these exact outputs, not an unrelated deposit. + outputs: list[InventoryRecord] = [] + while len(outputs) != len(commitments): + check() + if self.clock() >= deadline: + raise InventoryError(f"Scanner has not indexed {tx_id}; reconcile before continuing") + records = yield from self._find_records(plan.asset.program) + outputs = [] + for commitment in commitments: + record = next((item for item in records if str(item.get("commitment")) == commitment), None) + decoded = decode_record(self.address, plan.asset, record, capabilities) if record is not None and capabilities else None + if decoded is not None: + outputs.append(decoded) + if len(outputs) != len(commitments): + yield Sleep(poll_interval) + # Bind each planned output to the scanned nonce and amount before + # a dependent step may use it. + for output, expected in zip(outputs, step.outputs): + if output.amount != expected.amount: + raise InventoryError("Confirmed output amount differs from the plan") + references[expected.id] = output.id + completed += 1 + except Exception as exc: # noqa: BLE001 - every interruption is reported, never swallowed + # Earlier accepted steps stay on chain; report progress and let + # reconciliation settle any pending spend before a new plan. + return InventoryResult("interrupted", tuple(transaction_ids), completed, str(exc)) + return InventoryResult("complete", tuple(transaction_ids), completed) + + def reconcile_record_inventory(self, *, rebroadcast: bool = False, release_reserved: bool = False) -> Step[list[RecordSpend]]: + store = self._store("Reconciliation") + scope = self.scope + for entry in store.list(scope): + if entry.status in ("confirmed", "rejected", "cancelled"): + continue + if entry.transaction_id is None: + # A reserved entry never reached the prepared checkpoint, so + # nothing was broadcast. It is released only on request, because + # another process may still be proving against it. + if release_reserved and entry.status == "reserved": + store.update(scope, entry.id, {"status": "cancelled"}) + continue + # Only a definitive chain result changes the reservation; a missing + # transaction does not prove a prepared or submitted one failed. + confirmed = yield ConfirmedTransaction(entry.transaction_id) + if confirmed is not None: + status = confirmed.get("status") + if status not in ("accepted", "rejected"): + raise InventoryError("Unexpected transaction confirmation response") + store.update(scope, entry.id, {"status": "confirmed" if status == "accepted" else "rejected", "transaction": None}) + continue + if rebroadcast and entry.transaction is not None: + # Reuse the saved proof and id; rebuilding could create a second + # transaction while the first is still unresolved. + tx_id = yield Broadcast(entry.transaction) + if tx_id != entry.transaction_id: + raise InventoryError("Rebroadcast returned a different transaction id") + store.update(scope, entry.id, {"status": "submitted"}) + return store.list(scope) + + def auto_join( + self, asset: RecordAsset, *, min_amount: int | None = None, max_transactions: int | None = None, + timeout: float = 120.0, poll_interval: float = 1.0, max_fee_microcredits: int | None = None, + cancelled: Callable[[], bool] | None = None, + ) -> Step[InventoryResult]: + inventory = yield from self.get_record_inventory(asset) + if min_amount is not None and min_amount <= 0: + raise InventoryError("min_amount must be positive") + records = list(inventory.available) + if min_amount is not None: + # Keep a sufficient existing record; otherwise take the largest + # records until they cover the amount and leave the rest untouched. + if any(item.amount >= min_amount for item in records): + return InventoryResult("complete") + records.sort(key=lambda item: (-item.amount, item.id)) + selected: list[InventoryRecord] = [] + total = 0 + for item in records: + if total >= min_amount: + break + selected.append(item) + total += item.amount + if total < min_amount: + raise InventoryError("Insufficient available balance for auto_join") + records = selected + if len(records) <= 1: + return InventoryResult("complete") + helpers: tuple[RecordJoinProgram, ...] = () + if len(records) > 2: + helpers = yield from self._join_programs(inventory.asset, self.management.token_join) + plan = build_inventory_plan( + scope=self.scope, asset=inventory.asset, target=InventoryTarget(records=1), + inputs=[PlannedRecord(item.id, item.amount) for item in records], max_transactions=max_transactions, + token_join=self.management.token_join, join_programs=helpers, + ) + return (yield from self.rebalance_record_inventory( + plan, timeout=timeout, poll_interval=poll_interval, max_fee_microcredits=max_fee_microcredits, cancelled=cancelled, + )) + + +__all__ = [ + "DEFAULT", "Broadcast", "BuildTransaction", "ConfirmedTransaction", "FeeRecordSelector", "FindRecords", + "InventoryEngine", "Management", "PreparedTransaction", "ProgramSource", "Request", "Sleep", "Step", + "WaitForScannerSync", "inventory_output_commitments", "positive_integer", "record_input_ids", "transaction_fee", +] diff --git a/sdk/python/aleo/inventory/assets.py b/sdk/python/aleo/inventory/assets.py new file mode 100644 index 00000000..1a48c51f --- /dev/null +++ b/sdk/python/aleo/inventory/assets.py @@ -0,0 +1,251 @@ +"""Resolve an asset against its deployed program and decode scanner records. + +Before any record is spent, the asset's program is checked for the native +``join``/``split`` interface and the balance record and amount field are +fixed. Scanner records are then decoded against that resolution: a record +from another program, another owner, a compliance or registry record, or a +recipient-bound wrapper record is never treated as spendable inventory. +""" +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Any, Mapping + +from .interface import ProgramFunction, ProgramInterface, ProgramParam, parse_program_interface +from .types import ( + AmountType, InventoryError, InventoryRecord, RecordAsset, ResolvedRecordAsset, Standard, +) + +CREDITS_PROGRAM = "credits.aleo" +#: Microcredits a native ``credits.aleo/split`` removes from its outputs, separate from the fee. +CREDITS_SPLIT_DEDUCTION = 10_000 +#: Record entries that mark a compliance, registry, or non-fungible record. +EXCLUDED_RECORD_FIELDS = ("token_id", "sender", "recipient") + +_PROGRAM_ID = re.compile(r"^[a-z][a-z0-9_]*\.aleo$") +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_UNSIGNED = re.compile(r"^u(8|16|32|64|128)$") +_NONCE = re.compile(r"(? int: + """Credits removed by one split of this asset, independent of network fees.""" + return CREDITS_SPLIT_DEDUCTION if asset.program == CREDITS_PROGRAM else 0 + + +def amount_width(asset: RecordAsset) -> AmountType: + """The declared width of the amount field, with the native defaults for unresolved assets.""" + if asset.amount_type is not None: + return asset.amount_type + return "u64" if asset.program == CREDITS_PROGRAM or asset.amount_field == "microcredits" else "u128" + + +def record_name(asset: RecordAsset) -> str: + """The configured record name, or the native default (``credits`` / ``Token``).""" + return asset.record_name or ("credits" if asset.program == CREDITS_PROGRAM else "Token") + + +def max_amount(asset: RecordAsset) -> int: + """The largest amount the asset's declared width can hold.""" + return (1 << int(amount_width(asset)[1:])) - 1 + + +def scope_for(chain_id: str, address: str) -> str: + """The reservation namespace for one chain and account. + + Serialised exactly as Veil's ``JSON.stringify([chainId, address])`` so a + journal can be shared with other Veil clients. + """ + return json.dumps([chain_id, address], separators=(",", ":")) + + +@dataclass(frozen=True) +class AssetCapabilities: + """What was learned about an asset's record when it was resolved. + + Attributes: + asset: The resolved record name, amount field, and width. + recipient_bound: The record declares ``recipient_bound``; bound + records are excluded from inventory. + excluded: The record declares a compliance or registry field, so no + record of this type is spendable inventory. + """ + + asset: ResolvedRecordAsset + recipient_bound: bool + excluded: bool + + +def _is_record(param: ProgramParam, name: str) -> bool: + return param.kind == "record" and param.type == name + + +def _detect_standard(interface: ProgramInterface) -> Standard | None: + """Detect ARC-20 or ARC-22 from the token interface, ignoring missing view functions.""" + required = {"join", "split", "transfer_private", "transfer_public"} + if not required.issubset(interface.functions): + return None + return "arc22" if "ComplianceRecord" in interface.records else "arc20" + + +def validate_asset(asset: RecordAsset, source: str | None) -> AssetCapabilities: + """Resolve the asset's record defaults and check the program's join/split interface. + + Reads nothing from the network: pass the deployed program source (or + ``None`` for ``credits.aleo``, whose interface is fixed). + + Raises: + InventoryError: The asset names are malformed, the hint contradicts + the program, the program is missing, or its ``join``/``split`` + does not take and return the selected record. + """ + if not _PROGRAM_ID.match(asset.program): + raise InventoryError("Invalid asset program") + for name in (asset.record_name, asset.amount_field): + if name is not None and not _IDENTIFIER.match(name): + raise InventoryError("Invalid record or amount field name") + if asset.standard is not None and asset.standard not in ("credits", "arc20", "arc22"): + raise InventoryError("Unsupported token standard") + if asset.standard == "credits" and asset.program != CREDITS_PROGRAM: + raise InventoryError("Native credits require credits.aleo") + if asset.program == CREDITS_PROGRAM and asset.standard not in (None, "credits"): + raise InventoryError("credits.aleo requires the credits standard") + if asset.program == CREDITS_PROGRAM: + # Native credits have a fixed record interface; an override must match it. + if (asset.record_name not in (None, "credits")) or (asset.amount_field not in (None, "microcredits")): + raise InventoryError("credits.aleo requires the credits record and microcredits amount field") + resolved = ResolvedRecordAsset( + CREDITS_PROGRAM, standard="credits", record_name="credits", amount_field="microcredits", amount_type="u64", + ) + return AssetCapabilities(resolved, recipient_bound=False, excluded=False) + if source is None: + raise InventoryError(f"{asset.program} is not deployed; its record interface cannot be resolved") + interface = parse_program_interface(source) + join = interface.functions.get("join") + split = interface.functions.get("split") + join_input = join.inputs[0] if join and join.inputs else None + # An explicit record wins. Otherwise follow the native join input, which is + # Token for ARC programs and can be another name for a custom program. + name = asset.record_name or (join_input.type if join_input and join_input.kind == "record" else "Token") + record = interface.records.get(name) + if record is None or interface.id != asset.program: + raise InventoryError(f"{asset.program} has an incompatible record join/split interface") + field_name = asset.amount_field or next( + (candidate for candidate in ("amount", "microcredits") if record.field(candidate) is not None), None, + ) + entry = record.field(field_name) if field_name else None + if entry is None or not _UNSIGNED.match(entry.type): + raise InventoryError("Record amount field must be a declared unsigned integer") + owner = record.field("owner") + if ( + owner is None or owner.type != "address" + or not _join_split_match(join, split, name, entry.type) + ): + raise InventoryError(f"{asset.program} has an incompatible token join/split interface") + standard = asset.standard + if standard is None and name == "Token" and field_name == "amount" and entry.type == "u128": + # Detect a standard only for the default Token balance; a custom field + # must never inherit eligibility for a helper that sums Token.amount. + standard = _detect_standard(interface) + resolved = ResolvedRecordAsset( + asset.program, standard=standard, record_name=name, amount_field=str(field_name), + amount_type=entry.type, # type: ignore[arg-type] # matched by _UNSIGNED above + ) + return AssetCapabilities( + resolved, + recipient_bound=record.field("recipient_bound") is not None, + excluded=any(record.field(item) is not None for item in EXCLUDED_RECORD_FIELDS), + ) + + +def _join_split_match(join: ProgramFunction | None, split: ProgramFunction | None, name: str, amount_type: str) -> bool: + if join is None or split is None: + return False + return ( + len(join.inputs) == 2 and all(_is_record(param, name) for param in join.inputs) + and len(join.outputs) == 1 and _is_record(join.outputs[0], name) + and len(split.inputs) == 2 and _is_record(split.inputs[0], name) + and split.inputs[1].kind == "plaintext" and split.inputs[1].type == amount_type + and len(split.outputs) == 2 and all(_is_record(param, name) for param in split.outputs) + ) + + +def record_plaintext_of(record: Mapping[str, Any]) -> str | None: + """The decrypted literal carried by a scanner record, or ``None``.""" + value = record.get("record_plaintext") + return str(value) if value else None + + +def inventory_record_id(record: Mapping[str, Any]) -> str: + """The record nonce (``…group``) that identifies a record in plans and journals. + + Raises: + InventoryError: The record carries no decrypted plaintext. + """ + plaintext = record_plaintext_of(record) + match = _NONCE.search(plaintext) if plaintext else None + if match is None: + raise InventoryError("Inventory requires decrypted record plaintext; register the account for scanning") + return match.group(1) + + +def plaintext_field(plaintext: str, name: str) -> str | None: + """The literal value of entry *name* in a record plaintext, without its visibility suffix.""" + match = re.search(rf"(? InventoryRecord | None: + """Decode one scanner record as spendable inventory, or ``None`` when it is not. + + A record is skipped when it belongs to another program or owner, is + already spent, is a compliance or registry record, is recipient-bound, + or holds an amount outside the asset's declared width. + + Raises: + InventoryError: The record has no decrypted plaintext, so eligibility + cannot be established. + """ + resolved = capabilities.asset + if capabilities.excluded or record.get("program_name") != asset.program or record.get("spent") is True: + return None + declared_name = record.get("record_name") + if declared_name and declared_name != resolved.record_name: + return None + plaintext = record_plaintext_of(record) + if plaintext is None: + raise InventoryError("Inventory requires decrypted record plaintext; register the account for scanning") + owner = _OWNER.search(plaintext) + if owner is None or owner.group(1) != address: + return None + if any(plaintext_field(plaintext, name) is not None for name in EXCLUDED_RECORD_FIELDS): + return None + if plaintext_field(plaintext, "recipient_bound") == "true": + return None + raw = plaintext_field(plaintext, resolved.amount_field) + match = _AMOUNT.match(raw) if raw else None + if match is None or match.group(2) != resolved.amount_type: + return None + amount = int(match.group(1)) + if amount <= 0 or amount > max_amount(resolved): + return None + return InventoryRecord(inventory_record_id(record), amount, record) + + +__all__ = [ + "AssetCapabilities", "CREDITS_PROGRAM", "CREDITS_SPLIT_DEDUCTION", "EXCLUDED_RECORD_FIELDS", "amount_width", + "decode_record", "inventory_record_id", "max_amount", "plaintext_field", "record_name", "record_plaintext_of", + "scope_for", "split_deduction", "validate_asset", +] diff --git a/sdk/python/aleo/inventory/client.py b/sdk/python/aleo/inventory/client.py new file mode 100644 index 00000000..bcd0b5b5 --- /dev/null +++ b/sdk/python/aleo/inventory/client.py @@ -0,0 +1,705 @@ +"""Record inventory over an ``Aleo`` or ``AsyncAleo`` facade. + +:class:`RecordInventory` and :class:`AsyncRecordInventory` expose the same +seven actions. Both drive the generators in :mod:`aleo.inventory._engine`; +the only difference is whether a request to the facade is called or awaited. +""" +from __future__ import annotations + +import asyncio +import inspect +import json +import threading +import time +from contextlib import contextmanager +from typing import Any, Awaitable, Callable, Generator, Literal, Mapping, Sequence, cast + +from .._scanner_common import net_module +from ..facade.errors import ProgramNotFound, TransactionNotFound +from ._engine import ( + Broadcast, BuildTransaction, ConfirmedTransaction, FindRecords, InventoryEngine, Management, PreparedTransaction, + ProgramSource, Request, Sleep, Step, transaction_fee, +) +from .assets import CREDITS_PROGRAM +from .interface import parse_program_interface +from .store import MemoryRecordInventoryStore +from .types import ( + InventoryError, InventoryPlan, InventoryRecord, InventoryResult, InventoryTarget, RecordAsset, + RecordInventorySnapshot, RecordInventoryStore, RecordJoinProgram, RecordManagementConfig, + RecordReservationConflictError, RecordSpend, +) + +Proving = Literal["delegated", "local"] +Builder = Callable[[BuildTransaction], PreparedTransaction] +AsyncBuilder = Callable[[BuildTransaction], "PreparedTransaction | Awaitable[PreparedTransaction]"] + + +class SpendCheckpoint: + """Report a self-managed write's progress to the journal. + + Yielded by :meth:`RecordInventory.reserve`. Call :meth:`prepared` as soon + as the proved transaction exists and before it is broadcast, then + :meth:`submitted`; recovery can then resubmit the saved transaction + without building a second spend. + + Attributes: + entry: The journal entry holding the reservation. + """ + + def __init__(self, store: RecordInventoryStore, scope: str, entry: RecordSpend, fee_limit: int | None) -> None: + self._store = store + self._scope = scope + self._fee_limit = fee_limit + self.entry = entry + self.is_prepared = False + self.is_submitted = False + + def prepared(self, tx_id: str, transaction: Mapping[str, Any], fee_microcredits: int | None = None) -> None: + """Save the proved transaction before broadcast and enforce the fee ceiling. + + Raises: + InventoryError: The fee exceeds ``max_fee_microcredits`` or cannot + be read from the transaction. The reservation is released on + exit because nothing was broadcast. + """ + fee = transaction_fee(transaction) if fee_microcredits is None else fee_microcredits + if self._fee_limit is not None and fee > self._fee_limit: + raise InventoryError("Proved transaction exceeds max_fee_microcredits") + self._store.update(self._scope, self.entry.id, { + "status": "prepared", "transaction_id": tx_id, "transaction": dict(transaction), "fee_microcredits": str(fee), + }) + self.is_prepared = True + + def submitted(self, tx_id: str) -> None: + """Record that the network accepted the broadcast; the reservation now outlives any error.""" + self.is_submitted = True + self._store.update(self._scope, self.entry.id, {"status": "submitted", "transaction_id": tx_id}) + + def confirmed(self, tx_id: str) -> None: + """Record acceptance and drop the saved transaction.""" + self._store.update(self._scope, self.entry.id, {"status": "confirmed", "transaction_id": tx_id, "transaction": None}) + + +class _InventoryBase: + """Shared construction, scope, and journal access for both clients.""" + + def __init__( + self, aleo: Any, management: RecordManagementConfig | None = None, *, proving: Proving = "delegated", + priority_fee: int = 0, pay_own_fee: bool = False, + ) -> None: + if proving not in ("delegated", "local"): + raise ValueError("proving must be 'delegated' or 'local'") + self._aleo = aleo + self.config: RecordManagementConfig = management or RecordManagementConfig() + self._store: RecordInventoryStore | None = self.config.store or ( + MemoryRecordInventoryStore() if self.config.enabled else None + ) + self._management = Management.resolve(self.config, str(aleo.network_name), self._store) + self.proving: Proving = proving + self.priority_fee = priority_fee + self.pay_own_fee = pay_own_fee + + @property + def enabled(self) -> bool: + """Whether writes reserve records and journal their progress.""" + return self._management.enabled + + @property + def store(self) -> RecordInventoryStore | None: + """The journal in use, or ``None`` while management is disabled.""" + return self._management.store + + @property + def network(self) -> str: + """The bound client's network name.""" + return str(self._aleo.network_name) + + def _account(self, account: Any) -> Any: + resolved = account if account is not None else self._aleo.default_account + if resolved is None: + raise InventoryError("Inventory requires an account; pass account= or set aleo.default_account") + return resolved + + def _engine(self, account: Any) -> InventoryEngine: + if account is None: + # Re-read the default account on every use so a change mid-run is noticed. + address: Callable[[], str] = lambda: str(self._account(None).address) + else: + fixed = str(account.address) + address = lambda: fixed + return InventoryEngine(self.network, address, self._management) + + def scope(self, account: Any = None) -> str: + """The reservation namespace (chain id and address) for *account* or the default account.""" + return self._engine(account).scope + + def reserved_record_ids(self, account: Any = None) -> set[str]: + """Record nonces the journal currently excludes from selection. + + Reads the store only. Empty while management is disabled. Pass the + result as ``exclude_nonces`` to any selector that chooses records + outside this module. + """ + return self._engine(account).reserved_record_ids() + + @contextmanager + def reserve( + self, program: str, function: str, record_ids: Sequence[str], *, asset_program: str | None = None, + account: Any = None, + ) -> Generator[SpendCheckpoint, None, None]: + """Reserve records for a write this code does not build, and journal its progress. + + Use it around a write that proves and broadcasts through another + path. Inside the block, call ``checkpoint.prepared`` once the proved + transaction exists and ``checkpoint.submitted`` after broadcast. + Leaving the block before ``prepared``, by return or by exception, + releases the reservation because nothing can have reached the + network; afterwards it is kept for reconciliation. + + Raises: + InventoryError: Management is disabled. + RecordReservationConflictError: A record is already held. + """ + store = self._management.store + if not self._management.enabled or store is None: + raise InventoryError("reserve requires record management to be enabled with a shared store") + scope = self.scope(account) + entry = store.acquire(RecordSpend( + id="", scope=scope, records=tuple(record_ids), program=program, function=function, status="reserved", + created_at=int(time.time() * 1000), asset_program=asset_program, + )) + if entry is None: + raise RecordReservationConflictError(tuple(record_ids)) + checkpoint = SpendCheckpoint(store, scope, entry, self._management.max_fee_microcredits) + try: + yield checkpoint + finally: + if not checkpoint.is_prepared and not checkpoint.is_submitted: + store.update(scope, entry.id, {"status": "cancelled"}) + + # ── Program registration for local authorization ───────────────────────── + + def _register_sources(self, sources: Mapping[str, str]) -> None: + """Add fetched programs to the client's process, dependencies first.""" + process = getattr(self._aleo, "process", None) + if process is None: + return + net = net_module(self.network) + added: set[str] = set() + + def add(program_id: str) -> None: + if program_id == CREDITS_PROGRAM or program_id in added or program_id not in sources: + return + added.add(program_id) + if process.contains_program(net.ProgramID.from_string(program_id)): + return + for dependency in parse_program_interface(sources[program_id]).imports: + add(dependency) + process.add_program(net.Program.from_source(sources[program_id])) + + for program_id in sources: + add(program_id) + + +def prepared_from_result(result: Any) -> PreparedTransaction: + """The unbroadcast transaction behind a facade ``TransactionResult``, ready for the journal.""" + return PreparedTransaction(str(result.id), json.loads(str(result.raw.to_json()))) + + +def _json_dict(value: Any) -> dict[str, Any]: + return cast(dict[str, Any], value) if isinstance(value, dict) else {} + + +def prepared_from_payload(payload: Any) -> PreparedTransaction: + """The unbroadcast transaction inside a delegated prover payload. + + Raises: + InventoryError: The payload carries no transaction, so there is + nothing to save before broadcast. + """ + transaction = _json_dict(_json_dict(payload).get("transaction")) + if not transaction.get("id"): + raise InventoryError("The prover returned no transaction to save before broadcast") + return PreparedTransaction(str(transaction["id"]), transaction) + + +def managed_inventory(aleo: Any) -> Any: + """The client's inventory when record management is enabled on it, else ``None``. + + Writers use this to decide whether a write must go through the journal. + A client without an ``inventory`` attribute, such as a test stub, counts + as unmanaged. + """ + inventory = getattr(aleo, "inventory", None) + return inventory if inventory is not None and getattr(inventory, "enabled", False) else None + + +_prepared_from_transaction = prepared_from_result +_prepared_from_payload = prepared_from_payload + + +def _scanner_synced(status: Any) -> bool: + report = _json_dict(status) + return bool(report.get("ok")) and bool(_json_dict(report.get("data")).get("synced")) + + +class RecordInventory(_InventoryBase): + """Inspect, reshape, and protect one account's private records. + + Reads go through the client's record provider, writes through its proving + path (delegated by default, with the fee master paying; ``proving="local"`` + proves in-process and pays a public fee). With management enabled, every + write reserves its inputs before proving and saves the proved transaction + before broadcast; the same journal is shared by every writer given the + same store. + + Args: + aleo: The bound ``Aleo`` facade. + management: Reservation and recovery settings. ``None`` disables + both; reads, plans, and single joins or splits still work. + proving: ``"delegated"`` (default) or ``"local"``. + priority_fee: Extra fee in microcredits on every write. Defaults to 0. + pay_own_fee: Under delegated proving, pay the public fee from the + account instead of the prover's fee master. Defaults to False. + """ + + def _run(self, step: Step[Any], account: Any, builder: Builder | None = None) -> Any: + try: + request = next(step) + while True: + try: + result = self._perform(request, account, builder) + except BaseException as exc: # the engine decides whether a 404 or outage is fatal + request = step.throw(exc) + else: + request = step.send(result) + except StopIteration as done: + return done.value + + def _perform(self, request: Request, account: Any, builder: Builder | None) -> Any: + if isinstance(request, ProgramSource): + try: + return str(self._aleo.programs.get(request.program_id).source) + except ProgramNotFound: + return None + if isinstance(request, FindRecords): + provider = self._aleo.record_provider + if provider is None: + raise InventoryError("Inventory requires a record provider; register the account with aleo.records or set aleo.record_provider") + return list(provider.find(account, program=request.program, unspent=True)) + if isinstance(request, BuildTransaction): + return builder(request) if builder is not None else self._build(request, account) + if isinstance(request, Broadcast): + return str(self._aleo.network.submit_transaction(json.dumps(request.transaction))) + if isinstance(request, ConfirmedTransaction): + try: + return self._aleo.network.get_confirmed_transaction(request.tx_id) + except TransactionNotFound: + return None + if isinstance(request, Sleep): + time.sleep(request.seconds) + return None + deadline = time.monotonic() + request.timeout + while True: + if _scanner_synced(self._aleo.records.status()): + return None + if time.monotonic() >= deadline: + raise InventoryError("Record scanner synchronization timed out") + time.sleep(1.0) + + def _build(self, request: BuildTransaction, account: Any) -> PreparedTransaction: + self._register_programs([request.program, *request.imports]) + bound = self._aleo.programs.get(request.program).functions[request.function](*request.inputs) + if self.proving == "local": + result = bound.build_transaction( + account, priority_fee=self.priority_fee, fee_record=request.fee_record, private_fee=request.private_fee, + ) + return _prepared_from_transaction(result) + payload = bound.delegate( + account, broadcast=False, pay_own_fee=self.pay_own_fee, fee_record=request.fee_record, priority_fee=self.priority_fee, + ) + return _prepared_from_payload(payload) + + def _register_programs(self, program_ids: Sequence[str]) -> None: + if getattr(self._aleo, "process", None) is None: + return + sources: dict[str, str] = {} + stack = [pid for pid in program_ids if pid != CREDITS_PROGRAM] + while stack: + program_id = stack.pop() + if program_id in sources or program_id == CREDITS_PROGRAM: + continue + sources[program_id] = str(self._aleo.programs.get(program_id).source) + stack.extend(parse_program_interface(sources[program_id]).imports) + self._register_sources(sources) + + # ── Actions ────────────────────────────────────────────────────────────── + + def get_record_inventory(self, asset: RecordAsset, *, account: Any = None) -> RecordInventorySnapshot: + """Read the account's spendable records of one asset. + + Checks the deployed program's record interface, scans the record + provider, and separates records held by other participating + operations from available ones. Nothing is signed or submitted. + + Args: + asset: Program with optional record and amount-field overrides. + account: Signer whose records to read. Defaults to ``aleo.default_account``. + + Raises: + InventoryError: The program's interface is incompatible, a record + arrived without decrypted plaintext, or no record provider is + configured. + + Example:: + + snapshot = aleo.inventory.get_record_inventory(RecordAsset("credits.aleo")) + print(snapshot.balance, len(snapshot.available), len(snapshot.reserved)) + """ + return self._run(self._engine(account).get_record_inventory(asset), account) + + def plan_record_inventory( + self, asset: RecordAsset, target: InventoryTarget, *, max_transactions: int | None = None, account: Any = None, + ) -> InventoryPlan: + """Plan the joins and splits that reach *target* from the current inventory. + + Reads the scanner and the deployed program, including any join + helpers deployed on this network. Nothing is reserved, signed, or + submitted. Review ``plan.steps`` and ``plan.deduction`` (credits a + native split removes, separate from network fees), then pass the + plan unchanged to :meth:`rebalance_record_inventory`. + + Args: + asset: Program with optional record and amount-field overrides. + target: Desired count, minimum amount, and distribution. + max_transactions: Most operations the plan may take, 1 through + 10000. Defaults to 100. + account: Signer; defaults to ``aleo.default_account``. + + Raises: + InventoryPlanError: The target cannot be reached within the bounds. + + Example:: + + plan = aleo.inventory.plan_record_inventory( + RecordAsset("credits.aleo"), + InventoryTarget(records=4, min_record_amount=1_000_000, distribution="balanced"), + ) + for step in plan.steps: + print(step.kind, len(step.inputs), [o.amount for o in step.outputs]) + """ + return self._run(self._engine(account).plan_record_inventory(asset, target, max_transactions), account) + + def join_records( + self, asset: RecordAsset, records: Sequence[InventoryRecord], *, join_program: RecordJoinProgram | Literal[False] | None = None, + account: Any = None, + ) -> str: + """Join records into one transaction and return its id. + + Two records use the program's native ``join``. More than two use a + deployed helper when one accepts that count (up to 16 for credits and + dedicated stablecoins, 15 for other ARC tokens); pass + ``join_program=False`` to force the native join. Signs, proves, and + broadcasts; acceptance is asynchronous. + + Args: + asset: Program with optional overrides. + records: Distinct records from :meth:`get_record_inventory`. + join_program: Helper override, ``False`` for native, ``None`` to select. + account: Signer; defaults to ``aleo.default_account``. + + Raises: + InventoryError: Records are not distinct and eligible, the joined + amount overflows the width, or no helper accepts the count. + RecordReservationConflictError: An input is already reserved. + + Example:: + + snapshot = aleo.inventory.get_record_inventory(asset) + tx_id = aleo.inventory.join_records(asset, snapshot.available[:2]) + aleo.network.wait_for_transaction(tx_id, timeout=120) + """ + return self._run(self._engine(account).join_records(asset, records, join_program), account) + + def split_record(self, asset: RecordAsset, record: InventoryRecord, amount: int, *, account: Any = None) -> str: + """Split one record into two and return the transaction id. + + The first output holds *amount*; the second holds the remainder, less + 10,000 microcredits for ``credits.aleo``. Both outputs must stay + positive. Signs, proves, and broadcasts; rescan after confirmation + before spending either output. + + Example:: + + tx_id = aleo.inventory.split_record(asset, snapshot.available[0], 500_000) + """ + return self._run(self._engine(account).split_record(asset, record, amount), account) + + def rebalance_record_inventory( + self, plan: InventoryPlan, *, timeout: float = 120.0, poll_interval: float = 1.0, + max_fee_microcredits: int | None = None, cancel: threading.Event | None = None, account: Any = None, + ) -> InventoryResult: + """Execute a reviewed plan one confirmed transaction at a time. + + Requires management to be enabled. Each step rescans, reserves its + inputs, proves, broadcasts, waits for confirmation and for the scanner + to show the exact outputs, and only then feeds them to the next step. + No accepted step is rolled back. + + Args: + plan: The plan from :meth:`plan_record_inventory`, unchanged. + timeout: Seconds allowed per step for confirmation and scanner + visibility. Defaults to 120. + poll_interval: Seconds between polls. Defaults to 1. + max_fee_microcredits: Total network-fee budget for this run, + excluding split deductions. ``None`` means no budget. + cancel: Set the event to stop before the next step; unresolved + reservations are kept. + account: Signer; defaults to ``aleo.default_account``. + + Returns: + ``complete`` when every step confirmed, otherwise ``interrupted`` + with the reason and every known transaction id. After an + interruption, reconcile, rescan, and plan again. + + Raises: + InventoryError: Before any submission, when the plan was edited, + belongs to another scope, or management is disabled. + + Example:: + + result = aleo.inventory.rebalance_record_inventory(plan, max_fee_microcredits=1_000_000) + if result.status == "interrupted": + aleo.inventory.reconcile_record_inventory() + """ + return self._run(self._engine(account).rebalance_record_inventory( + plan, timeout=timeout, poll_interval=poll_interval, max_fee_microcredits=max_fee_microcredits, + cancelled=cancel.is_set if cancel is not None else None, + ), account) + + def reconcile_record_inventory( + self, *, rebroadcast: bool = False, release_reserved: bool = False, account: Any = None, + ) -> list[RecordSpend]: + """Settle pending journal entries against the chain after an interruption. + + Reads each unresolved entry's transaction: an accepted one becomes + ``confirmed``, a rejected one ``rejected``. A transaction the node has + not seen keeps its reservation. With ``rebroadcast=True`` the saved + proved transaction is resubmitted with its original id; nothing is + rebuilt. Requires management to be enabled. + + Args: + rebroadcast: Resubmit saved proved transactions the node has not seen. + release_reserved: Cancel entries still in ``reserved`` status. Such + an entry never reached the prepared checkpoint, so nothing was + broadcast; set this after a crash, once no other process is + proving against this journal. + account: Signer whose scope to reconcile; defaults to ``aleo.default_account``. + + Returns: + The journal for this scope after reconciliation. + + Example:: + + journal = aleo.inventory.reconcile_record_inventory() + pending = [e for e in journal if e.status not in ("confirmed", "rejected", "cancelled")] + """ + return self._run(self._engine(account).reconcile_record_inventory( + rebroadcast=rebroadcast, release_reserved=release_reserved), account) + + def auto_join( + self, asset: RecordAsset, *, min_amount: int | None = None, max_transactions: int | None = None, + timeout: float = 120.0, poll_interval: float = 1.0, max_fee_microcredits: int | None = None, + cancel: threading.Event | None = None, account: Any = None, + ) -> InventoryResult: + """Join records until one covers *min_amount*, or consolidate everything when it is omitted. + + A record that already covers the amount means nothing is submitted. + Otherwise the largest records are joined, through helpers where + available, and the rest are left untouched. Requires management to be + enabled; executes like :meth:`rebalance_record_inventory`. + + Raises: + InventoryError: Before any submission when the available balance + cannot cover *min_amount*. + + Example:: + + result = aleo.inventory.auto_join(RecordAsset("credits.aleo"), min_amount=1_000_000) + """ + return self._run(self._engine(account).auto_join( + asset, min_amount=min_amount, max_transactions=max_transactions, timeout=timeout, + poll_interval=poll_interval, max_fee_microcredits=max_fee_microcredits, + cancelled=cancel.is_set if cancel is not None else None, + ), account) + + def managed_write( + self, program: str, function: str, inputs: Sequence[str], *, imports: Sequence[str] = (), account: Any = None, + private_fee: bool = False, fee_record: str | None = None, builder: Builder | None = None, + ) -> str: + """Run one write through the journal: reserve, prove, checkpoint, broadcast. + + The facade's ``transact`` and ``delegate`` call this on a managed + client. *builder* proves the call and returns the unbroadcast + transaction; without one the inventory's own proving settings apply. + Under a private fee the credits record is chosen once the exact fee is + known and reserved beside the inputs; *fee_record* is preferred when + it is still available and sufficient. + + Returns: + The transaction id the node returned. + """ + return self._run(self._engine(account).managed_write( + program, function, inputs, imports=imports, private_fee=private_fee, fee_record=fee_record, + ), account, builder) + + +class AsyncRecordInventory(_InventoryBase): + """Async twin of :class:`RecordInventory` over an ``AsyncAleo`` facade. + + Every action is awaited. Store calls are synchronous and brief; a custom + store that blocks should wrap its own I/O. + """ + + async def _run(self, step: Step[Any], account: Any, builder: AsyncBuilder | None = None) -> Any: + try: + request = next(step) + while True: + try: + result = await self._perform(request, account, builder) + except BaseException as exc: + request = step.throw(exc) + else: + request = step.send(result) + except StopIteration as done: + return done.value + + async def _perform(self, request: Request, account: Any, builder: AsyncBuilder | None) -> Any: + if isinstance(request, ProgramSource): + try: + return str((await self._aleo.programs.get(request.program_id)).source) + except ProgramNotFound: + return None + if isinstance(request, FindRecords): + provider = self._aleo.record_provider + if provider is None: + raise InventoryError("Inventory requires a record provider; register the account with aleo.records or set aleo.record_provider") + return list(await provider.find(account, program=request.program, unspent=True)) + if isinstance(request, BuildTransaction): + if builder is None: + return await self._build(request, account) + built = builder(request) + return await built if inspect.isawaitable(built) else built + if isinstance(request, Broadcast): + return str(await self._aleo.network.submit_transaction(json.dumps(request.transaction))) + if isinstance(request, ConfirmedTransaction): + try: + return await self._aleo.network.get_confirmed_transaction(request.tx_id) + except TransactionNotFound: + return None + if isinstance(request, Sleep): + await asyncio.sleep(request.seconds) + return None + deadline = time.monotonic() + request.timeout + while True: + if _scanner_synced(await self._aleo.records.status()): + return None + if time.monotonic() >= deadline: + raise InventoryError("Record scanner synchronization timed out") + await asyncio.sleep(1.0) + + async def _build(self, request: BuildTransaction, account: Any) -> PreparedTransaction: + await self._register_programs([request.program, *request.imports]) + program = await self._aleo.programs.get(request.program) + bound = program.functions[request.function](*request.inputs) + if self.proving == "local": + result = await bound.build_transaction( + account, priority_fee=self.priority_fee, fee_record=request.fee_record, private_fee=request.private_fee, + ) + return _prepared_from_transaction(result) + payload = await bound.delegate( + account, broadcast=False, pay_own_fee=self.pay_own_fee, fee_record=request.fee_record, priority_fee=self.priority_fee, + ) + return _prepared_from_payload(payload) + + async def _register_programs(self, program_ids: Sequence[str]) -> None: + if getattr(self._aleo, "process", None) is None: + return + sources: dict[str, str] = {} + stack = [pid for pid in program_ids if pid != CREDITS_PROGRAM] + while stack: + program_id = stack.pop() + if program_id in sources or program_id == CREDITS_PROGRAM: + continue + sources[program_id] = str((await self._aleo.programs.get(program_id)).source) + stack.extend(parse_program_interface(sources[program_id]).imports) + self._register_sources(sources) + + async def get_record_inventory(self, asset: RecordAsset, *, account: Any = None) -> RecordInventorySnapshot: + """Read the account's spendable records of one asset (awaited). See :meth:`RecordInventory.get_record_inventory`.""" + return await self._run(self._engine(account).get_record_inventory(asset), account) + + async def plan_record_inventory( + self, asset: RecordAsset, target: InventoryTarget, *, max_transactions: int | None = None, account: Any = None, + ) -> InventoryPlan: + """Plan joins and splits without submitting (awaited). See :meth:`RecordInventory.plan_record_inventory`.""" + return await self._run(self._engine(account).plan_record_inventory(asset, target, max_transactions), account) + + async def join_records( + self, asset: RecordAsset, records: Sequence[InventoryRecord], *, join_program: RecordJoinProgram | Literal[False] | None = None, + account: Any = None, + ) -> str: + """Join records in one transaction and return its id (awaited). See :meth:`RecordInventory.join_records`.""" + return await self._run(self._engine(account).join_records(asset, records, join_program), account) + + async def split_record(self, asset: RecordAsset, record: InventoryRecord, amount: int, *, account: Any = None) -> str: + """Split one record into two and return the transaction id (awaited). See :meth:`RecordInventory.split_record`.""" + return await self._run(self._engine(account).split_record(asset, record, amount), account) + + async def rebalance_record_inventory( + self, plan: InventoryPlan, *, timeout: float = 120.0, poll_interval: float = 1.0, + max_fee_microcredits: int | None = None, cancel: threading.Event | asyncio.Event | None = None, account: Any = None, + ) -> InventoryResult: + """Execute a reviewed plan step by step (awaited). See :meth:`RecordInventory.rebalance_record_inventory`.""" + return await self._run(self._engine(account).rebalance_record_inventory( + plan, timeout=timeout, poll_interval=poll_interval, max_fee_microcredits=max_fee_microcredits, + cancelled=cancel.is_set if cancel is not None else None, + ), account) + + async def reconcile_record_inventory( + self, *, rebroadcast: bool = False, release_reserved: bool = False, account: Any = None, + ) -> list[RecordSpend]: + """Settle pending journal entries against the chain (awaited). See :meth:`RecordInventory.reconcile_record_inventory`.""" + return await self._run(self._engine(account).reconcile_record_inventory( + rebroadcast=rebroadcast, release_reserved=release_reserved), account) + + async def auto_join( + self, asset: RecordAsset, *, min_amount: int | None = None, max_transactions: int | None = None, + timeout: float = 120.0, poll_interval: float = 1.0, max_fee_microcredits: int | None = None, + cancel: threading.Event | asyncio.Event | None = None, account: Any = None, + ) -> InventoryResult: + """Join enough records to cover *min_amount*, or all of them (awaited). See :meth:`RecordInventory.auto_join`.""" + return await self._run(self._engine(account).auto_join( + asset, min_amount=min_amount, max_transactions=max_transactions, timeout=timeout, + poll_interval=poll_interval, max_fee_microcredits=max_fee_microcredits, + cancelled=cancel.is_set if cancel is not None else None, + ), account) + + async def managed_write( + self, program: str, function: str, inputs: Sequence[str], *, imports: Sequence[str] = (), account: Any = None, + private_fee: bool = False, fee_record: str | None = None, builder: AsyncBuilder | None = None, + ) -> str: + """Run one write through the journal (awaited). See :meth:`RecordInventory.managed_write`. + + *builder* may be a plain function or a coroutine function; an + awaitable result is awaited. + """ + return await self._run(self._engine(account).managed_write( + program, function, inputs, imports=imports, private_fee=private_fee, fee_record=fee_record, + ), account, builder) + + +__all__ = [ + "AsyncBuilder", "AsyncRecordInventory", "Builder", "Proving", "RecordInventory", "SpendCheckpoint", + "managed_inventory", "prepared_from_payload", "prepared_from_result", +] diff --git a/sdk/python/aleo/inventory/interface.py b/sdk/python/aleo/inventory/interface.py new file mode 100644 index 00000000..335a423f --- /dev/null +++ b/sdk/python/aleo/inventory/interface.py @@ -0,0 +1,179 @@ +"""Read a program's record and function interface from its Aleo bytecode. + +Inventory actions check a deployed program before spending anything: the +balance record exists, ``join`` takes two of it and returns one, ``split`` +takes one plus an amount and returns two, and a join helper's ``join_N`` +accepts the asset's record. The compiled :class:`~aleo.mainnet.Program` lists +function inputs but not outputs or record names, so this module reads the +declarations directly from the source text. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Literal + +ParamKind = Literal["plaintext", "record", "external_record", "dynamic_record", "future", "dynamic_future"] + +_PROGRAM = re.compile(r"^program\s+([a-z][a-z0-9_]*\.aleo);\s*$") +_IMPORT = re.compile(r"^import\s+([a-z][a-z0-9_]*\.aleo);\s*$") +_BLOCK = re.compile(r"^(function|closure|finalize|record|struct|mapping|constructor)\s+([A-Za-z_][A-Za-z0-9_]*):\s*$") +_PARAM = re.compile(r"^\s+(input|output)\s+r\d+\s+as\s+(.+?);\s*$") +_FIELD = re.compile(r"^\s+([A-Za-z_][A-Za-z0-9_]*)\s+as\s+(.+?)\.(constant|public|private);\s*$") + + +@dataclass(frozen=True) +class ProgramParam: + """One function input or output. + + Attributes: + kind: ``"record"`` for a local record, ``"external_record"`` for + ``other.aleo/Name.record``, ``"dynamic_record"`` for + ``dynamic.record``, ``"future"`` or ``"dynamic_future"`` for + futures, otherwise ``"plaintext"``. + type: The declared type without its visibility: a record name, + ``program.aleo/Record``, ``identifier``, ``u64``, a struct name, + or an array literal type. + visibility: ``constant``, ``public``, or ``private`` for plaintext + parameters; ``None`` for records and futures. + """ + + kind: ParamKind + type: str + visibility: str | None = None + + +@dataclass(frozen=True) +class ProgramFunction: + """A function's declared inputs and outputs, in order.""" + + name: str + inputs: tuple[ProgramParam, ...] = () + outputs: tuple[ProgramParam, ...] = () + + +@dataclass(frozen=True) +class RecordField: + """One record entry with its declared type and visibility.""" + + name: str + type: str + visibility: str + + +@dataclass(frozen=True) +class ProgramRecord: + """A record declaration, including its ``owner`` entry.""" + + name: str + fields: tuple[RecordField, ...] = () + + def field(self, name: str) -> RecordField | None: + """Return the entry named *name*, or ``None``.""" + return next((item for item in self.fields if item.name == name), None) + + +@dataclass(frozen=True) +class ProgramInterface: + """The parts of a program that inventory actions check. + + Attributes: + id: The declared program id. + imports: Program ids named by ``import`` lines, in order. + functions: Functions by name with their inputs and outputs. Closures + and finalize blocks are not included. + records: Record declarations by name. + """ + + id: str + imports: tuple[str, ...] + functions: dict[str, ProgramFunction] + records: dict[str, ProgramRecord] + + +def parse_param_type(declared: str) -> ProgramParam: + """Classify a declared parameter type such as ``credits.record`` or ``u64.private``.""" + if declared.endswith(".record"): + base = declared[: -len(".record")] + if base == "dynamic": + return ProgramParam("dynamic_record", base) + return ProgramParam("external_record" if "/" in base else "record", base) + if declared.endswith(".future"): + base = declared[: -len(".future")] + return ProgramParam("dynamic_future" if base == "dynamic" else "future", base) + type_name, _, visibility = declared.rpartition(".") + if not type_name or visibility not in ("constant", "public", "private"): + return ProgramParam("plaintext", declared) + return ProgramParam("plaintext", type_name, visibility) + + +def parse_program_interface(source: str) -> ProgramInterface: + """Read the program id, imports, records, and function signatures from Aleo bytecode. + + Only declarations are read; instruction bodies are skipped. ``input`` and + ``output`` lines inside closures and finalize blocks belong to those + blocks and are not attributed to functions. + + Raises: + ValueError: The text declares no ``program`` line. + """ + program_id: str | None = None + imports: list[str] = [] + functions: dict[str, ProgramFunction] = {} + records: dict[str, ProgramRecord] = {} + block: tuple[str, str] | None = None + inputs: list[ProgramParam] = [] + outputs: list[ProgramParam] = [] + fields: list[RecordField] = [] + + def close() -> None: + if block is None: + return + kind, name = block + if kind == "function": + functions[name] = ProgramFunction(name, tuple(inputs), tuple(outputs)) + elif kind == "record": + records[name] = ProgramRecord(name, tuple(fields)) + + for raw in source.splitlines(): + line = raw.rstrip() + if not line.strip(): + continue + header = _BLOCK.match(line) + if header: + close() + block = (header.group(1), header.group(2)) + inputs, outputs, fields = [], [], [] + continue + if not line[0].isspace(): + close() + block = None + program = _PROGRAM.match(line) + if program: + program_id = program.group(1) + continue + imported = _IMPORT.match(line) + if imported: + imports.append(imported.group(1)) + continue + if block is None: + continue + kind, _ = block + if kind == "function": + param = _PARAM.match(line) + if param: + (inputs if param.group(1) == "input" else outputs).append(parse_param_type(param.group(2))) + elif kind == "record": + entry = _FIELD.match(line) + if entry: + fields.append(RecordField(entry.group(1), entry.group(2), entry.group(3))) + close() + if program_id is None: + raise ValueError("Program source declares no program id") + return ProgramInterface(program_id, tuple(imports), functions, records) + + +__all__ = [ + "ParamKind", "ProgramFunction", "ProgramInterface", "ProgramParam", "ProgramRecord", "RecordField", + "parse_param_type", "parse_program_interface", +] diff --git a/sdk/python/aleo/inventory/join_programs.py b/sdk/python/aleo/inventory/join_programs.py new file mode 100644 index 00000000..23b80f78 --- /dev/null +++ b/sdk/python/aleo/inventory/join_programs.py @@ -0,0 +1,209 @@ +"""Deployed helpers that join several records in one transaction. + +A native ``join`` consumes two records. Consolidating many records pairwise +costs one transaction per record. The deployed helpers below join up to 16 +credits or dedicated stablecoin records, or up to 15 records of any ARC-20 or +ARC-22 token, in a single transaction. The planner prefers them; execution +re-checks each helper's deployed signature before sending records to it. +""" +from __future__ import annotations + +import re +from typing import Literal, Mapping + +from .assets import CREDITS_PROGRAM +from .interface import ProgramFunction, ProgramInterface +from .types import ( + MAX_DYNAMIC_JOIN, MAX_STATIC_JOIN, InventoryError, RecordJoinProgram, ResolvedRecordAsset, TokenJoinRouter, +) + +_PROGRAM_ID = re.compile(r"^[a-z][a-z0-9_]*\.aleo$") +Network = Literal["mainnet", "testnet"] + + +def _counts(low: int, high: int) -> tuple[int, ...]: + return tuple(range(low, high + 1)) + + +def _static_helpers(prefix: str) -> tuple[RecordJoinProgram, ...]: + # Static helper deployments split their join functions into three size ranges. + return tuple( + RecordJoinProgram(f"{prefix}_{low}_{high}.aleo", "records", _counts(low, high)) + for low, high in ((2, 10), (11, 14), (15, 16)) + ) + + +def _dynamic_helpers(network: Network) -> tuple[RecordJoinProgram, ...]: + prefix = "main" if network == "mainnet" else "test" + return (RecordJoinProgram(f"{prefix}_aj_arc20_2_15.aleo", "program_identifier_then_records", _counts(2, 15)),) + + +#: Known join helpers by network and source program, with ARC defaults. +#: An exact program entry takes precedence over the ``arc20`` / ``arc22`` +#: entries. Selection still checks each helper's deployed signature; this +#: table alone does not establish that a helper exists on a node. +RECORD_JOIN_PROGRAMS: Mapping[str, Mapping[str, tuple[RecordJoinProgram, ...]]] = { + "mainnet": { + "credits.aleo": _static_helpers("autojoin_credits"), + "usad_stablecoin.aleo": _static_helpers("aj_usad_stablecoin"), + "usdcx_stablecoin.aleo": _static_helpers("aj_usdcx_stablecoin"), + "arc20": _dynamic_helpers("mainnet"), + "arc22": _dynamic_helpers("mainnet"), + }, + "testnet": { + "credits.aleo": _static_helpers("autojoin_credits"), + "test_usad_stablecoin.aleo": _static_helpers("aj_test_usad_stablecoin"), + "test_usdcx_stablecoin.aleo": _static_helpers("aj_test_usdcx_stablecoin"), + "arc20": _dynamic_helpers("testnet"), + "arc22": _dynamic_helpers("testnet"), + }, +} + + +def validate_record_join_program(helper: RecordJoinProgram) -> None: + """Check helper metadata without contacting a node. + + Raises: + InventoryError: The program id is malformed, a count repeats, or a + count lies outside 2 through 16 (15 for dynamic routers). + """ + limit = MAX_STATIC_JOIN if helper.input_mode == "records" else MAX_DYNAMIC_JOIN + if ( + not _PROGRAM_ID.match(helper.program) + or helper.input_mode not in ("records", "program_identifier_then_records") + or not helper.counts or len(set(helper.counts)) != len(helper.counts) + or any(count < 2 or count > limit for count in helper.counts) + ): + raise InventoryError("Invalid record join helper configuration") + + +def validate_token_join_router(router: TokenJoinRouter) -> int: + """Check a configured router without contacting a node and return its input ceiling. + + Raises: + InventoryError: The program id is malformed or ``max_records`` is + outside 2 through 15. + """ + if not _PROGRAM_ID.match(router.program): + raise InventoryError("Invalid token join router program") + limit = router.max_records + if limit < 2 or limit > MAX_DYNAMIC_JOIN: + raise InventoryError("Token join router max_records must be from 2 to 15") + return limit + + +def router_as_helper(router: TokenJoinRouter) -> RecordJoinProgram: + """Describe a configured dynamic router as a join helper.""" + return RecordJoinProgram(router.program, "program_identifier_then_records", _counts(2, validate_token_join_router(router))) + + +def join_program_candidates( + network: str, asset: ResolvedRecordAsset, token_join: TokenJoinRouter | Literal[False] | None, +) -> tuple[RecordJoinProgram, ...]: + """Helpers worth checking for *asset* on *network*, before reading any program. + + Returns nothing when helpers are disabled, when an explicit router covers + the asset, when the network has no known helpers, or when the asset uses a + custom record or amount field that no helper understands. + """ + if token_join is False or (token_join is not None and asset.program != CREDITS_PROGRAM): + return () + registry = RECORD_JOIN_PROGRAMS.get(network) + if registry is None: + return () + if asset.program == CREDITS_PROGRAM: + defaults_match = asset.record_name == "credits" and asset.amount_field == "microcredits" + else: + defaults_match = asset.record_name == "Token" and asset.amount_field == "amount" and asset.amount_type == "u128" + if not defaults_match: + return () + candidates = registry.get(asset.program) + if candidates is None and asset.standard is not None: + candidates = registry.get(asset.standard) + return candidates or () + + +def join_function_compatible( + function: ProgramFunction | None, asset: ResolvedRecordAsset, helper: RecordJoinProgram, count: int, +) -> bool: + """Whether a deployed ``join_N`` accepts *count* of the asset's records and returns one.""" + if function is None or len(function.outputs) != 1: + return False + if helper.input_mode == "program_identifier_then_records": + # Dynamic routers call Token.join and sum ``amount``; a custom record or + # field must not be sent through that interface. + if not (asset.record_name == "Token" and asset.amount_field == "amount" and asset.amount_type == "u128"): + return False + first = function.inputs[0] if function.inputs else None + return ( + len(function.inputs) == count + 1 + and first is not None and first.kind == "plaintext" and first.type == "identifier" and first.visibility == "public" + and all(param.kind == "dynamic_record" for param in function.inputs[1:]) + and function.outputs[0].kind == "dynamic_record" + ) + external = f"{asset.program}/{asset.record_name}" + return ( + len(function.inputs) == count + and all(param.kind == "external_record" and param.type == external for param in function.inputs) + and function.outputs[0].kind == "external_record" and function.outputs[0].type == external + ) + + +def supported_join_program( + helper: RecordJoinProgram, interface: ProgramInterface, asset: ResolvedRecordAsset, +) -> RecordJoinProgram | None: + """Restrict a helper to the counts its deployed code supports for *asset*; ``None`` when none do.""" + if interface.id != helper.program: + return None + counts = tuple( + count for count in helper.counts + if join_function_compatible(interface.functions.get(f"join_{count}"), asset, helper, count) + ) + return RecordJoinProgram(helper.program, helper.input_mode, counts) if counts else None + + +def validate_record_join_function( + helper: RecordJoinProgram, interface: ProgramInterface, asset: ResolvedRecordAsset, count: int, +) -> None: + """Re-check a helper's deployed signature immediately before spending through it. + + Raises: + InventoryError: The count is unsupported or the deployed function + does not accept and return the asset's record. + """ + validate_record_join_program(helper) + if count not in helper.counts: + raise InventoryError("Record count is outside the join helper limits") + if interface.id != helper.program or not join_function_compatible( + interface.functions.get(f"join_{count}"), asset, helper, count, + ): + raise InventoryError("Record join helper has an incompatible function signature") + + +def validate_token_join_function(router: TokenJoinRouter, interface: ProgramInterface, count: int) -> None: + """Check a configured router's deployed ``join_N`` before sending it records. + + Raises: + InventoryError: The count is outside the router's ceiling or the + deployed function lacks the public identifier and dynamic record + signature. + """ + limit = validate_token_join_router(router) + if count < 2 or count > limit: + raise InventoryError(f"Token join requires 2 to {limit} records") + function = interface.functions.get(f"join_{count}") + first = function.inputs[0] if function and function.inputs else None + if ( + interface.id != router.program or function is None or len(function.inputs) != count + 1 + or first is None or first.kind != "plaintext" or first.type != "identifier" or first.visibility != "public" + or any(param.kind != "dynamic_record" for param in function.inputs[1:]) + or len(function.outputs) != 1 or function.outputs[0].kind != "dynamic_record" + ): + raise InventoryError("Token join router has an incompatible function signature") + + +__all__ = [ + "RECORD_JOIN_PROGRAMS", "join_function_compatible", "join_program_candidates", "router_as_helper", + "supported_join_program", "validate_record_join_function", "validate_record_join_program", + "validate_token_join_function", "validate_token_join_router", +] diff --git a/sdk/python/aleo/inventory/planner.py b/sdk/python/aleo/inventory/planner.py new file mode 100644 index 00000000..3c665a90 --- /dev/null +++ b/sdk/python/aleo/inventory/planner.py @@ -0,0 +1,204 @@ +"""Plan the joins and splits that reshape an inventory, without any network access. + +The planner is deterministic: the same records, target, and helpers always +produce the same steps, so a plan built for review can be rebuilt at +execution time and compared. It follows one bounded strategy: keep records +that already fit, split large records directly into the missing lots when +the amounts divide exactly, and otherwise consolidate and then split. Every +amount stays within the asset's declared width. +""" +from __future__ import annotations + +from typing import Literal, Sequence + +from .assets import max_amount, split_deduction +from .join_programs import validate_record_join_program, validate_token_join_router +from .types import ( + MAX_PLAN_TRANSACTIONS, MAX_TARGET_RECORDS, InventoryPlan, InventoryPlanError, InventoryStep, InventoryTarget, + PlannedRecord, RecordAsset, RecordJoinProgram, StepKind, TokenJoinRouter, +) + + +def positive_integer(value: int, name: str, maximum: int = MAX_TARGET_RECORDS) -> None: + """Require an integer from 1 through *maximum*. + + Raises: + InventoryPlanError: The value is not an integer in range. + """ + if isinstance(value, bool) or value < 1 or value > maximum: + raise InventoryPlanError(f"{name} must be an integer from 1 to {maximum}") + + +def build_inventory_plan( + *, + scope: str, + asset: RecordAsset, + target: InventoryTarget, + inputs: Sequence[PlannedRecord], + max_transactions: int | None = None, + token_join: TokenJoinRouter | Literal[False] | None = None, + join_programs: Sequence[RecordJoinProgram] | None = None, +) -> InventoryPlan: + """Compute the joins and splits that take *inputs* to *target*. + + Reads nothing from the network and reserves nothing. The result lists + each step's inputs, predicted outputs, and credits deduction so it can be + reviewed before execution. + + Args: + scope: Chain and account the plan belongs to. + asset: The resolved asset; its width bounds every amount. + target: Desired count, minimum amount, and distribution. + inputs: Eligible records as nonces and amounts. + max_transactions: Most steps allowed, 1 through 10000. Defaults to 100. + token_join: Explicit dynamic router for ARC tokens; ``False`` or + ``None`` means none was configured. + join_programs: Deployed helpers available for batch joins. + + Raises: + InventoryPlanError: A bound is invalid, the target cannot be reached + from the inputs, an intermediate join would overflow the width, or + the plan needs more steps than allowed. + """ + if token_join: + validate_token_join_router(token_join) + router = token_join if (asset.program != "credits.aleo" and token_join) else None + helpers: tuple[RecordJoinProgram, ...] = () + if join_programs: + for helper in join_programs: + validate_record_join_program(helper) + helpers = tuple(join_programs) + max_join = validate_token_join_router(router) if router else 2 + positive_integer(target.records, "records") + limit = 100 if max_transactions is None else max_transactions + positive_integer(limit, "max_transactions", MAX_PLAN_TRANSACTIONS) + minimum = target.min_record_amount + tolerance = target.tolerance_bps + ceiling = max_amount(asset) + if isinstance(minimum, bool) or minimum < 1 or minimum > ceiling: + raise InventoryPlanError("Invalid minimum record amount") + if isinstance(tolerance, bool) or tolerance < 0 or tolerance > 10_000: + raise InventoryPlanError("Invalid balance tolerance") + if target.distribution not in ("preserve", "balanced"): + raise InventoryPlanError("Invalid inventory distribution") + records = tuple(PlannedRecord(item.id, item.amount) for item in inputs) + if len({item.id for item in records}) != len(records): + raise InventoryPlanError("Duplicate inventory records") + if any(item.amount <= 0 or item.amount > ceiling for item in records): + raise InventoryPlanError("Invalid record amount") + + working: list[PlannedRecord] = list(records) + steps: list[InventoryStep] = [] + total = sum(item.amount for item in working) + deduction = split_deduction(asset) + count = target.records + + def balanced() -> bool: + # Compare against the ideal equal share in integer arithmetic so + # fractional base units cannot introduce rounding drift. + for item in working: + difference = abs(item.amount * count - total) + # One base unit of rounding never warrants another transaction. + if difference > count and difference * 10_000 > total * tolerance: + return False + return True + + satisfied = ( + len(working) == count and all(item.amount >= minimum for item in working) + and (target.distribution != "balanced" or balanced()) + ) + + def finish() -> InventoryPlan: + return InventoryPlan( + scope=scope, asset=asset, target=target, inputs=records, steps=tuple(steps), + deduction=sum(step.deduction for step in steps), max_transactions=limit, + token_join=router, join_programs=helpers or None, + ) + + def push(kind: StepKind, consumed: Sequence[PlannedRecord], amounts: Sequence[int], amount: int | None = None) -> None: + # Replace consumed records with symbolic outputs that later steps can + # reference; execution resolves them to confirmed records. + nonlocal working + if len(steps) >= limit: + raise InventoryPlanError("Inventory plan exceeds max_transactions") + if any(value < minimum or value > ceiling for value in amounts): + raise InventoryPlanError("Inventory target is infeasible within token amount bounds") + outputs = tuple(PlannedRecord(f"step:{len(steps)}:{index}", value) for index, value in enumerate(amounts)) + steps.append(InventoryStep( + kind=kind, inputs=tuple(item.id for item in consumed), outputs=outputs, amount=amount, + deduction=deduction if kind == "split" else 0, + )) + consumed_ids = {item.id for item in consumed} + working = [item for item in working if item.id not in consumed_ids] + list(outputs) + + # Preserve exact lots when larger records split directly into the missing + # lots; this avoids a consolidate-and-split round trip for [25, 25, 50]. + if not satisfied and target.distribution == "balanced" and len(working) < count: + # Each split adds one record and may deduct credits; compute the final + # lot size after every deduction before deciding whether lots can stay. + final_total = total - deduction * (count - len(working)) + unit = final_total // count + if ( + unit >= minimum and final_total % count == 0 + and all((item.amount + deduction) % (unit + deduction) == 0 for item in working) + ): + for original in list(working): + record = original + while record.amount > unit: + push("split", [record], [unit, record.amount - unit - deduction], unit) + record = steps[-1].outputs[1] + return finish() + + if not satisfied: + if not working or total < count * minimum: + raise InventoryPlanError("Insufficient available inventory for target") + # Repack only when existing records cannot be preserved: balanced + # layouts or undersized records. + capacity = sum((item.amount + deduction) // (minimum + deduction) for item in working) + repack = ( + target.distribution == "balanced" or any(item.amount < minimum for item in working) or capacity < count + ) + reduce_to = 1 if repack else count + # Join the smallest records first, with ids breaking ties so rebuilding + # the same plan produces the same dependency sequence. + while len(working) > reduce_to: + working.sort(key=lambda item: (item.amount, item.id)) + needed = len(working) - reduce_to + 1 + # Prefer the largest helper that preserves the requested count. A + # final pair uses the native join unless a router was configured. + helper_count = max([2, *(c for helper in helpers for c in helper.counts if c <= needed)]) + batch = min(max_join if router else helper_count, needed) + amount = sum(item.amount for item in working[:batch]) + # Shrink the batch if its amount would overflow or its exact arity is unavailable. + while batch > 2 and ( + amount > ceiling or (not router and not any(batch in helper.counts for helper in helpers)) + ): + batch -= 1 + amount -= working[batch].amount + helper = next((h for h in helpers if batch in h.counts), None) if (not router and batch > 2) else None + consumed = working[:batch] + # Intermediate joins can be below the output minimum while accumulating dust. + if amount > ceiling: + raise InventoryPlanError("Join would overflow the asset amount width") + if len(steps) >= limit: + raise InventoryPlanError("Inventory plan exceeds max_transactions") + output = PlannedRecord(f"step:{len(steps)}:0", amount) + steps.append(InventoryStep( + kind="join", inputs=tuple(item.id for item in consumed), outputs=(output,), join_program=helper, + )) + working = working[batch:] + [output] + # Split the largest remaining record and leave enough in its remainder + # for any dependent splits, including their credits deductions. + while len(working) < count: + working.sort(key=lambda item: (-item.amount, item.id)) + record = working[0] + outputs_needed = count - len(working) + 1 if repack else 2 + usable = record.amount - deduction * (outputs_needed - 1) + amount = usable // outputs_needed + push("split", [record], [amount, record.amount - amount - deduction], amount) + if any(item.amount < minimum for item in working): + raise InventoryPlanError("Insufficient balance after split deductions") + return finish() + + +__all__ = ["build_inventory_plan", "positive_integer"] diff --git a/sdk/python/aleo/inventory/store.py b/sdk/python/aleo/inventory/store.py new file mode 100644 index 00000000..6f2bca89 --- /dev/null +++ b/sdk/python/aleo/inventory/store.py @@ -0,0 +1,119 @@ +"""Journal stores for record reservations. + +The SDK ships one store, :class:`MemoryRecordInventoryStore`, for a single +process. Anything that satisfies :class:`~aleo.inventory.types.RecordInventoryStore` +can replace it: a database shared by several processes, a key-value service, +or a file. A journal holds record nonces, program and function names, and +proved transactions; it never holds keys or record plaintext. +""" +from __future__ import annotations + +import copy +import threading + +from .types import RecordInventoryStore, RecordSpend, RecordSpendPatch + + +def holds_records(entry: RecordSpend) -> bool: + """Return whether a journal entry still excludes any record from selection. + + Confirmed entries keep excluding their inputs because a scanner can report + a spent record as unspent for a while. A rejected entry releases its + inputs but keeps a pinned fee record, since the chain can still charge + the fee of a rejected execution. + """ + return entry.status != "cancelled" and (entry.status != "rejected" or entry.fee_record_id is not None) + + +def held_record_ids(entry: RecordSpend) -> tuple[str, ...]: + """Return the nonces an entry currently excludes from selection.""" + if not holds_records(entry): + return () + if entry.status == "rejected": + return (entry.fee_record_id,) if entry.fee_record_id is not None else () + return entry.records + + +class MemoryRecordInventoryStore: + """Reserve records for writers in one process. + + Entries disappear when the process exits, so a restart cannot resume a + prepared transaction; supply a durable implementation for a long-running + service. Each method holds a lock only for the duration of the call. A + reservation is the entry's stored status, which is why it can outlive a + crash in a durable backend. + """ + + def __init__(self) -> None: + self._entries: list[RecordSpend] = [] + self._lock = threading.Lock() + + def list(self, scope: str) -> list[RecordSpend]: + """Return copies of every entry under *scope*, oldest first. + + Entries are copied so a caller cannot change the journal by mutating + a result. Saved transactions are shared, not copied: they are written + once at the prepared checkpoint and read back only for rebroadcast, + and copying them on every reservation lookup would cost megabytes + for a busy journal. + """ + with self._lock: + return [copy.copy(entry) for entry in self._entries if entry.scope == scope] + + def acquire(self, entry: RecordSpend) -> RecordSpend | None: + """Reserve all of the entry's records or none of them. + + The supplied ``id`` is ignored; the store assigns one. Returns + ``None`` when any record is already held under the same scope, so the + caller can stop before proving. + + Raises: + ValueError: The entry names the same record twice. + """ + if len(set(entry.records)) != len(entry.records): + raise ValueError("Duplicate record inputs") + with self._lock: + # Check contention and insert under one lock so another thread + # cannot reserve an input between the check and the insertion. + wanted = set(entry.records) + for old in self._entries: + if old.scope == entry.scope and wanted.intersection(held_record_ids(old)): + return None + saved = copy.deepcopy(entry) + saved.id = str(len(self._entries) + 1) + self._entries.append(saved) + return copy.copy(saved) + + def update(self, scope: str, entry_id: str, patch: RecordSpendPatch) -> None: + """Apply a checkpoint to an existing entry. + + Raises: + KeyError: No entry with that id exists under *scope*. + """ + with self._lock: + for entry in self._entries: + if entry.scope == scope and entry.id == entry_id: + if "status" in patch: + entry.status = patch["status"] + if "transaction_id" in patch: + entry.transaction_id = patch["transaction_id"] + if "transaction" in patch: + # ``None`` removes a transaction that recovery no longer needs. + entry.transaction = copy.deepcopy(patch["transaction"]) + if "fee_microcredits" in patch: + entry.fee_microcredits = patch["fee_microcredits"] + return + raise KeyError("Unknown record reservation") + + +def reserved_record_ids(store: RecordInventoryStore | None, scope: str) -> set[str]: + """Return every nonce the journal excludes under *scope*; empty without a store.""" + if store is None: + return set() + out: set[str] = set() + for entry in store.list(scope): + out.update(held_record_ids(entry)) + return out + + +__all__ = ["MemoryRecordInventoryStore", "RecordInventoryStore", "held_record_ids", "holds_records", "reserved_record_ids"] diff --git a/sdk/python/aleo/inventory/types.py b/sdk/python/aleo/inventory/types.py new file mode 100644 index 00000000..beaf2202 --- /dev/null +++ b/sdk/python/aleo/inventory/types.py @@ -0,0 +1,584 @@ +"""Core types for private record inventory management. + +A private balance is held in encrypted records. These types describe one +asset's spendable records, the layout a caller wants, the joins and splits +that reach it, and the journal entries that keep participating writers from +spending the same record. They hold no keys. A plan or journal entry carries +nonces and amounts, never record plaintext, so either can be stored or shared +without exposing a balance. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal, Mapping, Protocol, Sequence, TypedDict + +from .._client_common import AleoError + +Standard = Literal["credits", "arc20", "arc22"] +AmountType = Literal["u8", "u16", "u32", "u64", "u128"] +Distribution = Literal["preserve", "balanced"] +InputMode = Literal["records", "program_identifier_then_records"] +StepKind = Literal["join", "split"] +RecordSpendStatus = Literal["reserved", "prepared", "submitted", "confirmed", "rejected", "cancelled"] +ResultStatus = Literal["complete", "interrupted"] + +#: Most records a target may ask for. +MAX_TARGET_RECORDS = 1000 +#: Most operations a plan may request. +MAX_PLAN_TRANSACTIONS = 10_000 +#: Largest input count a static join helper exposes (``join_16``). +MAX_STATIC_JOIN = 16 +#: Largest input count a dynamic ARC join router exposes (``join_15``). +MAX_DYNAMIC_JOIN = 15 + + +class InventoryError(AleoError): + """A record inventory operation could not proceed; the message says why.""" + + +class InventoryPlanError(InventoryError): + """A target cannot be reached from the available records within the configured bounds.""" + + +class RecordReservationConflictError(InventoryError): + """An execution input was already reserved or spent when the write tried to acquire it. + + Raised before signing or proving. The failed attempt holds no reservation + and has submitted nothing, so the caller may select different inputs and + try again. Proving, broadcast, and confirmation failures never raise this + error, and a conflict raised by a nested write after the outer write + acquired its inputs is reported as a plain :class:`InventoryError`. + + Attributes: + record_ids: Execution input nonces involved in the failed acquisition. + Fee-record contention is not reported here. + """ + + def __init__(self, record_ids: Sequence[str]) -> None: + super().__init__("Record inventory changed: an input is already reserved or spent") + self.record_ids: tuple[str, ...] = tuple(record_ids) + + +@dataclass(frozen=True) +class RecordAsset: + """Select the program and fungible record balance to manage. + + Only ``program`` is required. The deployed program's ``join``/``split`` + interface supplies the record name and amount field; pass overrides for a + program whose balance record is not named ``Token`` or whose amount field + is not ``amount``. ``credits.aleo`` always uses ``credits.microcredits``. + + Attributes: + program: Deployed program id, such as ``"credits.aleo"``. + standard: Optional interface hint (``"credits"``, ``"arc20"``, + ``"arc22"``), checked against the deployed code. Omit it to let the + SDK detect the standard from the program. + record_name: Record whose balance is managed. Defaults to ``credits`` + for ``credits.aleo``, otherwise to the record the program's ``join`` + accepts. + amount_field: Unsigned integer field read as the balance. Defaults to + ``amount``, then ``microcredits``. A named field must exist on the + record; no other field is substituted. + amount_type: Declared width of the amount field, set once the asset + has been resolved against the program. Bounds every amount the + planner accepts. + """ + + program: str + standard: Standard | None = None + record_name: str | None = None + amount_field: str | None = None + amount_type: AmountType | None = None + + +@dataclass(frozen=True, kw_only=True) +class ResolvedRecordAsset(RecordAsset): + """A :class:`RecordAsset` whose record, amount field, and width were read from the program. + + Returned in :attr:`RecordInventorySnapshot.asset` and carried by plans so + execution spends the same record and field the caller reviewed. + """ + + # The base class makes these optional; a resolved asset requires them. The + # keyword-only override is what pyright flags, and it is intended. + record_name: str # pyright: ignore[reportGeneralTypeIssues] + amount_field: str # pyright: ignore[reportGeneralTypeIssues] + amount_type: AmountType # pyright: ignore[reportGeneralTypeIssues] + + +@dataclass(frozen=True) +class TokenJoinRouter: + """Select a dynamic ARC join router explicitly. + + A router exposes ``join_2`` through ``join_15`` and joins records from any + ARC-20 or ARC-22 token whose identifier is passed as the first input. + Omit the router to let the SDK pick the deployed helper for the asset and + network; set it to point at a custom deployment. + + Attributes: + program: Router program id on the client's network. + max_records: Most inputs one transaction may join, from 2 through 15. + Defaults to 15. + """ + + program: str + max_records: int = 15 + + +@dataclass(frozen=True) +class RecordJoinProgram: + """Describe a deployed helper that joins several records in one transaction. + + Attributes: + program: Helper program id on the selected network. + input_mode: ``"records"`` when ``join_N`` takes only records (credits + and dedicated stablecoin helpers); ``"program_identifier_then_records"`` + when the token program identifier comes first (dynamic ARC routers). + counts: Input counts the helper supports, each from 2 through 16 + (15 for dynamic routers). + """ + + program: str + input_mode: InputMode + counts: tuple[int, ...] + + +@dataclass(frozen=True) +class InventoryRecord: + """One spendable record with its decoded amount. + + Attributes: + id: The record nonce (``…group``), stable across scanners and the + identity used in plans and journal entries. + amount: Balance in integer base units, bounded by the asset's declared + width (u64 for credits, u128 for ARC tokens). + record: The scanner record this amount was decoded from. Its + ``record_plaintext`` is what a join or split spends; it stays in + memory and is never written to a plan or journal. + """ + + id: str + amount: int + record: Mapping[str, Any] + + @property + def plaintext(self) -> str: + """The decrypted record literal a transaction input needs.""" + return str(self.record.get("record_plaintext", "")) + + @property + def commitment(self) -> str | None: + """The commitment the scanner indexes this record under, when reported.""" + value = self.record.get("commitment") + return None if value is None else str(value) + + @property + def tag(self) -> str | None: + """The record tag, used to recognise a consumed fee record in a proved transaction.""" + value = self.record.get("tag") + return None if value is None else str(value) + + +@dataclass(frozen=True) +class InventoryTarget: + """Describe the record layout a caller wants. + + Choose ``preserve`` to keep records that already fit and only add or + consolidate as needed; choose ``balanced`` to reshape the inventory into + roughly equal amounts, which costs more transactions. + + Attributes: + records: Desired record count, from 1 through 1000. + min_record_amount: Smallest acceptable output in base units. Defaults + to 1. + distribution: ``"preserve"`` (default) or ``"balanced"``. + tolerance_bps: Allowed deviation from equal amounts under + ``balanced``, in basis points from 0 through 10000. Defaults to + 1000 (10%). + """ + + records: int + min_record_amount: int = 1 + distribution: Distribution = "preserve" + tolerance_bps: int = 1000 + + +@dataclass(frozen=True) +class PlannedRecord: + """A record identity and amount as a plan sees it. + + Attributes: + id: A scanned record nonce for plan inputs, or a symbolic + ``step::`` reference for an output a later step consumes. + amount: Predicted amount in base units. + """ + + id: str + amount: int + + +@dataclass(frozen=True) +class InventoryStep: + """One join or split in a plan, with references to earlier planned outputs. + + Attributes: + kind: ``"join"`` or ``"split"``. + inputs: Record nonces or earlier step output references. A native + join consumes two; a helper consumes up to 16 (15 for dynamic + routers). + outputs: Predicted outputs. None is spendable until the step is + confirmed and its records appear in the scanner. + amount: The split's first output amount; ``None`` for joins. + deduction: Credits the split itself removes from its outputs + (10,000 microcredits for ``credits.aleo``), separate from the + transaction fee. + join_program: Helper captured for this join. ``None`` means the + native join, or the plan's router for ARC tokens. + """ + + kind: StepKind + inputs: tuple[str, ...] + outputs: tuple[PlannedRecord, ...] + amount: int | None = None + deduction: int = 0 + join_program: RecordJoinProgram | None = None + + +@dataclass(frozen=True) +class InventoryPlan: + """A reviewable sequence of joins and splits bound to one account and chain. + + Review ``steps`` and ``deduction`` before passing the plan to + ``rebalance_record_inventory``. Execution rebuilds the plan from its + inputs and refuses one whose steps were edited. + + Attributes: + scope: The chain and account the plan belongs to. + asset: Program with its resolved record name, amount field, and width. + target: Requested count, minimum amount, and distribution. + inputs: Eligible records at planning time, as nonces and amounts. + steps: Ordered operations; later steps may consume earlier outputs. + deduction: Total credits removed by splits, excluding network fees. + max_transactions: Operation ceiling the plan was built under. + token_join: Dynamic router captured for ARC joins, when configured. + join_programs: Deployed helpers available at planning time; ``None`` + or empty means native pairwise joins. + """ + + scope: str + asset: RecordAsset + target: InventoryTarget + inputs: tuple[PlannedRecord, ...] + steps: tuple[InventoryStep, ...] + deduction: int + max_transactions: int + token_join: TokenJoinRouter | None = None + join_programs: tuple[RecordJoinProgram, ...] | None = None + + def to_dict(self) -> dict[str, Any]: + """Serialise the plan to JSON-compatible data for review or storage. + + Amounts are rendered as decimal strings so a u128 balance survives + JSON consumers that cannot represent large integers. + """ + def record(item: PlannedRecord) -> dict[str, str]: + return {"id": item.id, "amount": str(item.amount)} + + def helper(item: RecordJoinProgram) -> dict[str, Any]: + return {"program": item.program, "input_mode": item.input_mode, "counts": list(item.counts)} + + asset: dict[str, Any] = {"program": self.asset.program} + for name in ("standard", "record_name", "amount_field", "amount_type"): + value = getattr(self.asset, name) + if value is not None: + asset[name] = value + out: dict[str, Any] = { + "scope": self.scope, + "asset": asset, + "target": { + "records": self.target.records, + "min_record_amount": str(self.target.min_record_amount), + "distribution": self.target.distribution, + "tolerance_bps": self.target.tolerance_bps, + }, + "inputs": [record(item) for item in self.inputs], + "steps": [ + { + "kind": step.kind, + "inputs": list(step.inputs), + "outputs": [record(item) for item in step.outputs], + **({"amount": str(step.amount)} if step.amount is not None else {}), + "deduction": str(step.deduction), + **({"join_program": helper(step.join_program)} if step.join_program else {}), + } + for step in self.steps + ], + "deduction": str(self.deduction), + "max_transactions": self.max_transactions, + } + if self.token_join is not None: + out["token_join"] = {"program": self.token_join.program, "max_records": self.token_join.max_records} + if self.join_programs: + out["join_programs"] = [helper(item) for item in self.join_programs] + return out + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "InventoryPlan": + """Rebuild a plan saved with :meth:`to_dict`. + + The result carries a plain :class:`RecordAsset`; execution re-resolves + it against the deployed program before spending anything. + """ + def record(item: Mapping[str, Any]) -> PlannedRecord: + return PlannedRecord(str(item["id"]), int(item["amount"])) + + def helper(item: Mapping[str, Any]) -> RecordJoinProgram: + return RecordJoinProgram(str(item["program"]), item["input_mode"], tuple(int(c) for c in item["counts"])) + + asset_data = data["asset"] + target_data = data["target"] + router = data.get("token_join") + helpers = data.get("join_programs") + return InventoryPlan( + scope=str(data["scope"]), + asset=RecordAsset( + program=str(asset_data["program"]), + standard=asset_data.get("standard"), + record_name=asset_data.get("record_name"), + amount_field=asset_data.get("amount_field"), + amount_type=asset_data.get("amount_type"), + ), + target=InventoryTarget( + records=int(target_data["records"]), + min_record_amount=int(target_data.get("min_record_amount", 1)), + distribution=target_data.get("distribution", "preserve"), + tolerance_bps=int(target_data.get("tolerance_bps", 1000)), + ), + inputs=tuple(record(item) for item in data["inputs"]), + steps=tuple( + InventoryStep( + kind=step["kind"], + inputs=tuple(str(item) for item in step["inputs"]), + outputs=tuple(record(item) for item in step["outputs"]), + amount=None if step.get("amount") is None else int(step["amount"]), + deduction=int(step.get("deduction", 0)), + join_program=helper(step["join_program"]) if step.get("join_program") else None, + ) + for step in data["steps"] + ), + deduction=int(data["deduction"]), + max_transactions=int(data["max_transactions"]), + token_join=None if router is None else TokenJoinRouter(str(router["program"]), int(router.get("max_records", 15))), + join_programs=None if not helpers else tuple(helper(item) for item in helpers), + ) + + +@dataclass +class RecordSpend: + """A journal entry: one operation's reserved records and its progress. + + Entries hold nonces, program and function names, and the proved + transaction; never keys or record plaintext. A store may persist them + anywhere. + + Attributes: + id: Store-assigned identifier, unique within the store. + scope: Chain and account the entry belongs to. + records: Reserved record nonces, including the private fee record + when one was pinned. + program: Program the transaction calls. + function: Transition name. + status: Last persisted lifecycle boundary. + created_at: Unix time in milliseconds. + account_type: ``"local"`` for keys held by the SDK. Kept for journal + compatibility with other Veil clients. + asset_program: Underlying token for a routed join; ``None`` for + native operations. + fee_record_id: Nonce of the private fee record. It stays excluded + after a rejected execution because the chain can still charge it. + transaction_id: Known transaction id, retained on an uncertain + submission. + transaction: The proved transaction as JSON data, kept from + preparation until confirmation so recovery can resubmit it + without building a second spend. + fee_microcredits: Total network fee as a decimal string. + """ + + id: str + scope: str + records: tuple[str, ...] + program: str + function: str + status: RecordSpendStatus + created_at: int + account_type: str = "local" + asset_program: str | None = None + fee_record_id: str | None = None + transaction_id: str | None = None + transaction: dict[str, Any] | None = None + fee_microcredits: str | None = None + + def to_dict(self) -> dict[str, Any]: + """Serialise the entry with Veil's field names, for a journal shared with other clients.""" + out: dict[str, Any] = { + "id": self.id, "scope": self.scope, "records": list(self.records), "program": self.program, + "function": self.function, "status": self.status, "createdAt": self.created_at, + "accountType": self.account_type, + } + if self.asset_program is not None: + out["assetProgram"] = self.asset_program + if self.fee_record_id is not None: + out["feeRecordId"] = self.fee_record_id + if self.transaction_id is not None: + out["transactionId"] = self.transaction_id + if self.transaction is not None: + out["transaction"] = self.transaction + if self.fee_microcredits is not None: + out["feeMicrocredits"] = self.fee_microcredits + return out + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "RecordSpend": + """Rebuild an entry serialised with :meth:`to_dict`.""" + return RecordSpend( + id=str(data["id"]), scope=str(data["scope"]), records=tuple(str(r) for r in data["records"]), + program=str(data["program"]), function=str(data["function"]), status=data["status"], + created_at=int(data["createdAt"]), account_type=str(data.get("accountType", "local")), + asset_program=data.get("assetProgram"), fee_record_id=data.get("feeRecordId"), + transaction_id=data.get("transactionId"), transaction=data.get("transaction"), + fee_microcredits=data.get("feeMicrocredits"), + ) + + +class RecordSpendPatch(TypedDict, total=False): + """Fields :meth:`RecordInventoryStore.update` may change. + + A key that is present replaces the stored value; ``transaction: None`` + removes a stored transaction once it is no longer needed for recovery. + """ + + status: RecordSpendStatus + transaction_id: str + transaction: dict[str, Any] | None + fee_microcredits: str + + +class RecordInventoryStore(Protocol): + """Coordinate record spending across participating writers. + + Implement this protocol over any backend: a database shared by several + processes, a key-value service, or a file. The SDK ships an in-memory + implementation for a single process. + + Two guarantees are required. ``acquire`` MUST reserve every nonce in the + entry or none of them, atomically with respect to other callers. Each + mutation MUST be finished before the method returns. Entries whose status + is not ``rejected`` or ``cancelled`` keep their records excluded from + selection; a rejected entry keeps only its ``fee_record_id`` excluded. + """ + + def list(self, scope: str) -> list[RecordSpend]: + """Return every entry recorded under *scope*.""" + ... + + def acquire(self, entry: RecordSpend) -> RecordSpend | None: + """Reserve the entry's records and assign its id, or return ``None`` when any is already held.""" + ... + + def update(self, scope: str, entry_id: str, patch: RecordSpendPatch) -> None: + """Persist a checkpoint for an existing entry.""" + ... + + +@dataclass(frozen=True) +class RecordManagementConfig: + """Configure record reservations and recovery for a client. + + With ``enabled=True`` every write on the client reserves its record + inputs before proving, saves the proved transaction before broadcast, + and journals submission, so participating writers never spend the same + record and an interrupted operation can be reconciled. Disabled, the + configuration still supplies a chain identity and router for reads, + plans, and single joins; nothing is journaled. + + Attributes: + enabled: Turn reservations and recovery on. Defaults to off. + store: Journal shared by participating writers. An enabled + configuration without one uses a process-local in-memory store, + which is lost on exit. + chain_id: Stable chain identity for the reservation scope. Defaults + to the client's network name. A devnode or custom chain MUST set a + distinct value so two local chains do not share a scope. + max_fee_microcredits: Ceiling checked against the proved transaction's + fee before broadcast; a transaction over it is cancelled after + proving. Under delegated proving with the fee master paying, the + fee read is the prover's, so leave this ``None`` unless the + account pays its own fees. ``None`` means no limit. + token_join: ``None`` selects deployed helpers by asset and network; + ``False`` uses native pairwise joins everywhere; a + :class:`TokenJoinRouter` names a dynamic ARC router explicitly. + wait_for_sync: Wait for the record scanner to report itself synced + before each inventory read, so a lagging scanner does not cause + needless joins. Defaults to off. + sync_timeout: Seconds to wait for scanner synchronisation. Defaults + to 300. + """ + + enabled: bool = False + store: RecordInventoryStore | None = None + chain_id: str | None = None + max_fee_microcredits: int | None = None + token_join: TokenJoinRouter | Literal[False] | None = None + wait_for_sync: bool = False + sync_timeout: float = 300.0 + + +@dataclass(frozen=True) +class InventoryResult: + """Outcome of executing a plan or an automatic join. + + Attributes: + status: ``"complete"`` when every step confirmed, otherwise + ``"interrupted"``. No accepted step is rolled back. + transaction_ids: Every prepared or submitted transaction id, + including ones whose acceptance is uncertain. + completed_steps: Steps whose outputs were confirmed and seen by the + scanner. + reason: Why execution stopped; ``None`` on completion. After an + interruption, reconcile, rescan, and plan again; the old plan + describes records that completed steps have already spent. + """ + + status: ResultStatus + transaction_ids: tuple[str, ...] = () + completed_steps: int = 0 + reason: str | None = None + + +@dataclass(frozen=True) +class RecordInventorySnapshot: + """Spendable records of one asset, separated from records held by other operations. + + Attributes: + asset: The resolved record name, amount field, and width. + available: Eligible records no participating operation holds. + reserved: Eligible records the journal excludes. A low available + balance with a large reserved one means funds are committed, not + missing. + balance: Sum of the available amounts in base units. + """ + + asset: ResolvedRecordAsset + available: tuple[InventoryRecord, ...] + reserved: tuple[InventoryRecord, ...] = field(default=()) + balance: int = 0 + + +__all__ = [ + "AmountType", "Distribution", "InputMode", "InventoryError", "InventoryPlan", "InventoryPlanError", + "InventoryRecord", "InventoryResult", "InventoryStep", "InventoryTarget", "MAX_DYNAMIC_JOIN", + "MAX_PLAN_TRANSACTIONS", "MAX_STATIC_JOIN", "MAX_TARGET_RECORDS", "PlannedRecord", "RecordAsset", + "RecordInventorySnapshot", "RecordInventoryStore", "RecordJoinProgram", "RecordManagementConfig", + "RecordReservationConflictError", "RecordSpend", "RecordSpendPatch", "RecordSpendStatus", + "ResolvedRecordAsset", "ResultStatus", "Standard", "StepKind", "TokenJoinRouter", +] diff --git a/sdk/python/tests/fixtures/main_aj_arc20_2_15.aleo b/sdk/python/tests/fixtures/main_aj_arc20_2_15.aleo new file mode 100644 index 00000000..6781399c --- /dev/null +++ b/sdk/python/tests/fixtures/main_aj_arc20_2_15.aleo @@ -0,0 +1,389 @@ +program main_aj_arc20_2_15.aleo; + +function join_2: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + assert.eq r1.owner r2.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r3 (as dynamic.record); + output r3 as dynamic.record; + +function join_3: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r4 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r4 r3 (as dynamic.record dynamic.record) into r5 (as dynamic.record); + output r5 as dynamic.record; + +function join_4: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r5 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r3 (as dynamic.record dynamic.record) into r6 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r6 r4 (as dynamic.record dynamic.record) into r7 (as dynamic.record); + output r7 as dynamic.record; + +function join_5: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r6 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r7 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r6 r7 (as dynamic.record dynamic.record) into r8 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r8 r5 (as dynamic.record dynamic.record) into r9 (as dynamic.record); + output r9 as dynamic.record; + +function join_6: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r7 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r8 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r9 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r7 r8 (as dynamic.record dynamic.record) into r10 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r10 r9 (as dynamic.record dynamic.record) into r11 (as dynamic.record); + output r11 as dynamic.record; + +function join_7: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + input r7 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + assert.eq r1.owner r7.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r8 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r9 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r10 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r8 r9 (as dynamic.record dynamic.record) into r11 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r10 r7 (as dynamic.record dynamic.record) into r12 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r11 r12 (as dynamic.record dynamic.record) into r13 (as dynamic.record); + output r13 as dynamic.record; + +function join_8: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + input r7 as dynamic.record; + input r8 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + assert.eq r1.owner r7.owner; + assert.eq r1.owner r8.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r9 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r10 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r11 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r7 r8 (as dynamic.record dynamic.record) into r12 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r9 r10 (as dynamic.record dynamic.record) into r13 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r11 r12 (as dynamic.record dynamic.record) into r14 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r13 r14 (as dynamic.record dynamic.record) into r15 (as dynamic.record); + output r15 as dynamic.record; + +function join_9: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + input r7 as dynamic.record; + input r8 as dynamic.record; + input r9 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + assert.eq r1.owner r7.owner; + assert.eq r1.owner r8.owner; + assert.eq r1.owner r9.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r10 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r11 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r12 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r7 r8 (as dynamic.record dynamic.record) into r13 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r10 r11 (as dynamic.record dynamic.record) into r14 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r12 r13 (as dynamic.record dynamic.record) into r15 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r14 r15 (as dynamic.record dynamic.record) into r16 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r16 r9 (as dynamic.record dynamic.record) into r17 (as dynamic.record); + output r17 as dynamic.record; + +function join_10: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + input r7 as dynamic.record; + input r8 as dynamic.record; + input r9 as dynamic.record; + input r10 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + assert.eq r1.owner r7.owner; + assert.eq r1.owner r8.owner; + assert.eq r1.owner r9.owner; + assert.eq r1.owner r10.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r11 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r12 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r13 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r7 r8 (as dynamic.record dynamic.record) into r14 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r9 r10 (as dynamic.record dynamic.record) into r15 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r11 r12 (as dynamic.record dynamic.record) into r16 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r13 r14 (as dynamic.record dynamic.record) into r17 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r16 r17 (as dynamic.record dynamic.record) into r18 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r18 r15 (as dynamic.record dynamic.record) into r19 (as dynamic.record); + output r19 as dynamic.record; + +function join_11: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + input r7 as dynamic.record; + input r8 as dynamic.record; + input r9 as dynamic.record; + input r10 as dynamic.record; + input r11 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + assert.eq r1.owner r7.owner; + assert.eq r1.owner r8.owner; + assert.eq r1.owner r9.owner; + assert.eq r1.owner r10.owner; + assert.eq r1.owner r11.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r12 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r13 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r14 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r7 r8 (as dynamic.record dynamic.record) into r15 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r9 r10 (as dynamic.record dynamic.record) into r16 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r12 r13 (as dynamic.record dynamic.record) into r17 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r14 r15 (as dynamic.record dynamic.record) into r18 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r16 r11 (as dynamic.record dynamic.record) into r19 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r17 r18 (as dynamic.record dynamic.record) into r20 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r20 r19 (as dynamic.record dynamic.record) into r21 (as dynamic.record); + output r21 as dynamic.record; + +function join_12: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + input r7 as dynamic.record; + input r8 as dynamic.record; + input r9 as dynamic.record; + input r10 as dynamic.record; + input r11 as dynamic.record; + input r12 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + assert.eq r1.owner r7.owner; + assert.eq r1.owner r8.owner; + assert.eq r1.owner r9.owner; + assert.eq r1.owner r10.owner; + assert.eq r1.owner r11.owner; + assert.eq r1.owner r12.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r13 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r14 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r15 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r7 r8 (as dynamic.record dynamic.record) into r16 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r9 r10 (as dynamic.record dynamic.record) into r17 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r11 r12 (as dynamic.record dynamic.record) into r18 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r13 r14 (as dynamic.record dynamic.record) into r19 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r15 r16 (as dynamic.record dynamic.record) into r20 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r17 r18 (as dynamic.record dynamic.record) into r21 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r19 r20 (as dynamic.record dynamic.record) into r22 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r22 r21 (as dynamic.record dynamic.record) into r23 (as dynamic.record); + output r23 as dynamic.record; + +function join_13: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + input r7 as dynamic.record; + input r8 as dynamic.record; + input r9 as dynamic.record; + input r10 as dynamic.record; + input r11 as dynamic.record; + input r12 as dynamic.record; + input r13 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + assert.eq r1.owner r7.owner; + assert.eq r1.owner r8.owner; + assert.eq r1.owner r9.owner; + assert.eq r1.owner r10.owner; + assert.eq r1.owner r11.owner; + assert.eq r1.owner r12.owner; + assert.eq r1.owner r13.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r14 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r15 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r16 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r7 r8 (as dynamic.record dynamic.record) into r17 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r9 r10 (as dynamic.record dynamic.record) into r18 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r11 r12 (as dynamic.record dynamic.record) into r19 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r14 r15 (as dynamic.record dynamic.record) into r20 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r16 r17 (as dynamic.record dynamic.record) into r21 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r18 r19 (as dynamic.record dynamic.record) into r22 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r20 r21 (as dynamic.record dynamic.record) into r23 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r22 r13 (as dynamic.record dynamic.record) into r24 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r23 r24 (as dynamic.record dynamic.record) into r25 (as dynamic.record); + output r25 as dynamic.record; + +function join_14: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + input r7 as dynamic.record; + input r8 as dynamic.record; + input r9 as dynamic.record; + input r10 as dynamic.record; + input r11 as dynamic.record; + input r12 as dynamic.record; + input r13 as dynamic.record; + input r14 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + assert.eq r1.owner r7.owner; + assert.eq r1.owner r8.owner; + assert.eq r1.owner r9.owner; + assert.eq r1.owner r10.owner; + assert.eq r1.owner r11.owner; + assert.eq r1.owner r12.owner; + assert.eq r1.owner r13.owner; + assert.eq r1.owner r14.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r15 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r16 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r17 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r7 r8 (as dynamic.record dynamic.record) into r18 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r9 r10 (as dynamic.record dynamic.record) into r19 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r11 r12 (as dynamic.record dynamic.record) into r20 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r13 r14 (as dynamic.record dynamic.record) into r21 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r15 r16 (as dynamic.record dynamic.record) into r22 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r17 r18 (as dynamic.record dynamic.record) into r23 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r19 r20 (as dynamic.record dynamic.record) into r24 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r22 r23 (as dynamic.record dynamic.record) into r25 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r24 r21 (as dynamic.record dynamic.record) into r26 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r25 r26 (as dynamic.record dynamic.record) into r27 (as dynamic.record); + output r27 as dynamic.record; + +function join_15: + input r0 as identifier.public; + input r1 as dynamic.record; + input r2 as dynamic.record; + input r3 as dynamic.record; + input r4 as dynamic.record; + input r5 as dynamic.record; + input r6 as dynamic.record; + input r7 as dynamic.record; + input r8 as dynamic.record; + input r9 as dynamic.record; + input r10 as dynamic.record; + input r11 as dynamic.record; + input r12 as dynamic.record; + input r13 as dynamic.record; + input r14 as dynamic.record; + input r15 as dynamic.record; + assert.eq r1.owner r2.owner; + assert.eq r1.owner r3.owner; + assert.eq r1.owner r4.owner; + assert.eq r1.owner r5.owner; + assert.eq r1.owner r6.owner; + assert.eq r1.owner r7.owner; + assert.eq r1.owner r8.owner; + assert.eq r1.owner r9.owner; + assert.eq r1.owner r10.owner; + assert.eq r1.owner r11.owner; + assert.eq r1.owner r12.owner; + assert.eq r1.owner r13.owner; + assert.eq r1.owner r14.owner; + assert.eq r1.owner r15.owner; + call.dynamic r0 'aleo' 'join' with r1 r2 (as dynamic.record dynamic.record) into r16 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r3 r4 (as dynamic.record dynamic.record) into r17 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r5 r6 (as dynamic.record dynamic.record) into r18 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r7 r8 (as dynamic.record dynamic.record) into r19 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r9 r10 (as dynamic.record dynamic.record) into r20 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r11 r12 (as dynamic.record dynamic.record) into r21 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r13 r14 (as dynamic.record dynamic.record) into r22 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r16 r17 (as dynamic.record dynamic.record) into r23 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r18 r19 (as dynamic.record dynamic.record) into r24 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r20 r21 (as dynamic.record dynamic.record) into r25 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r22 r15 (as dynamic.record dynamic.record) into r26 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r23 r24 (as dynamic.record dynamic.record) into r27 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r25 r26 (as dynamic.record dynamic.record) into r28 (as dynamic.record); + call.dynamic r0 'aleo' 'join' with r27 r28 (as dynamic.record dynamic.record) into r29 (as dynamic.record); + output r29 as dynamic.record; + +constructor: + assert.eq edition 0u16; diff --git a/sdk/python/tests/test_facade_inventory.py b/sdk/python/tests/test_facade_inventory.py new file mode 100644 index 00000000..2fb81230 --- /dev/null +++ b/sdk/python/tests/test_facade_inventory.py @@ -0,0 +1,253 @@ +"""The facade's managed write hook and ``aleo.inventory`` wiring. + +_source: ProvableHQ/veil packages/core/test/inventory/configuration.test.ts and +privateFees.test.ts, PR #172 @ 2e324d5, adapted to the facade's verbs. +Proving and broadcast are replaced; reservation, checkpointing, and fee +selection run for real. +""" +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from aleo import Aleo, AsyncAleo, HTTPProvider, RecordReservationConflictError +from aleo.facade.async_client import AsyncBoundCall +from aleo.facade.call import BoundCall +from aleo.facade.programs import Program +from aleo.inventory import InventoryError, MemoryRecordInventoryStore, RecordAsset, RecordManagementConfig +from aleo.mainnet import PrivateKey, Program as RawProgram + +BASE = "https://api.provable.com/v2" +CREDITS_SOURCE = RawProgram.credits().source + + +def record(owner: str, amount: int, nonce: int) -> dict[str, Any]: + return {"program_name": "credits.aleo", "record_name": "credits", "tag": f"{nonce}field", "commitment": f"{nonce}field", + "spent": False, "record_plaintext": f"{{ owner: {owner}.private, microcredits: {amount}u64.private, _nonce: {nonce}group.public }}"} + + +class Provider: + def __init__(self, records: list[dict[str, Any]]) -> None: + self.records = records + + def find(self, account: Any = None, *, program: str | None = None, unspent: bool = True, **_: Any) -> list[dict[str, Any]]: + return [dict(r) for r in self.records if r["program_name"] == program] + + +class AsyncProvider(Provider): + async def find(self, account: Any = None, *, program: str | None = None, unspent: bool = True, **_: Any) -> list[dict[str, Any]]: # type: ignore[override] + return super().find(account, program=program, unspent=unspent) + + +def fake_transaction(tx_id: str, fee_tag: str | None = None, fee: int = 100) -> dict[str, Any]: + inputs: list[dict[str, Any]] = [{"type": "public", "value": f"{fee}u64"}, {"type": "public", "value": "0u64"}] + function = "fee_public" + if fee_tag is not None: + inputs.insert(0, {"type": "record", "tag": fee_tag}) + function = "fee_private" + return {"id": tx_id, "type": "execute", "execution": {"transitions": []}, + "fee": {"transition": {"program": "credits.aleo", "function": function, "inputs": inputs}}} + + +class FakeResult: + def __init__(self, tx: dict[str, Any]) -> None: + self.id = tx["id"] + self.raw = self + + def to_json(self) -> str: + return json.dumps(self._tx) + + @classmethod + def of(cls, tx: dict[str, Any]) -> "FakeResult": + result = cls(tx) + result._tx = tx + return result + + +def setup(monkeypatch: pytest.MonkeyPatch, *, enabled: bool = True, max_fee: int | None = None, fee: int = 100, + records: list[int] | None = None) -> tuple[Aleo, Any, Program, dict[str, Any]]: + """A mainnet client whose proving and broadcast are faked; its inventory is real.""" + config = RecordManagementConfig(enabled=enabled, store=MemoryRecordInventoryStore(), max_fee_microcredits=max_fee) + aleo = Aleo(HTTPProvider(BASE), record_management=config) + account = aleo.account.from_private_key(PrivateKey.random()) + aleo.default_account = account + owner = str(account.address) + aleo.record_provider = Provider([record(owner, amount, index + 1) for index, amount in enumerate(records or [1_000_000, 110, 200])]) + seen: dict[str, Any] = {"builds": [], "submits": [], "nonce_tag": {}} + sequence = [0] + + def build_transaction(self: BoundCall, acct: Any = None, *, priority_fee: int = 0, fee_record: Any = None, + private_fee: bool = False, base_fee: int | None = None) -> FakeResult: + sequence[0] += 1 + tag = None + if private_fee or fee_record is not None: + chosen = fee_record(fee + priority_fee) if callable(fee_record) else str(fee_record) + tag = next(r["tag"] for r in aleo.record_provider.records if r["record_plaintext"] == chosen) + tx = fake_transaction(f"at1tx{sequence[0]}", tag, fee + priority_fee) + seen["builds"].append({"args": list(self.args), "private_fee": private_fee, "fee_record": fee_record, "tx": tx}) + return FakeResult.of(tx) + + monkeypatch.setattr(BoundCall, "build_transaction", build_transaction) + + def submit_transaction(raw: Any) -> str: + # The managed path submits JSON text; the plain path submits the result object. + tx = json.loads(raw if isinstance(raw, str) else raw.to_json()) + seen["submits"].append(tx) + return str(tx["id"]) + + monkeypatch.setattr(aleo.network, "submit_transaction", submit_transaction) + # The synthetic records cannot be authorized by snarkVM, so a prover-side + # delegate (broadcast=False) is answered directly; the broadcasting call + # still runs the real hook. + original_delegate = BoundCall.delegate + + def delegate(self: BoundCall, acct: Any = None, *, broadcast: bool = True, pay_own_fee: bool = False, + fee_record: Any = None, priority_fee: int = 0) -> Any: + if broadcast: + return original_delegate(self, acct, broadcast=True, pay_own_fee=pay_own_fee, fee_record=fee_record, priority_fee=priority_fee) + sequence[0] += 1 + tx = fake_transaction(f"at1dps{sequence[0]}", None, fee) + seen["builds"].append({"args": list(self.args), "delegated": True, "fee_record": fee_record, "tx": tx}) + return {"transaction": tx, "broadcast_result": None} + + monkeypatch.setattr(BoundCall, "delegate", delegate) + credits = Program(aleo, RawProgram.from_source(CREDITS_SOURCE)) + return aleo, account, credits, seen + + +def transfer(credits: Program, aleo: Aleo, index: int = 0) -> BoundCall: + plaintext = aleo.record_provider.records[index]["record_plaintext"] + call = credits.functions.transfer_private(plaintext, str(aleo.default_account.address), 1) + assert isinstance(call, BoundCall) + return call + + +def test_client_exposes_a_lazy_inventory_and_its_configuration() -> None: + aleo = Aleo(HTTPProvider(BASE)) + assert aleo.record_management is None + assert aleo.inventory is aleo.inventory and aleo.inventory.enabled is False + config = RecordManagementConfig(enabled=True) + managed = Aleo(HTTPProvider(BASE, network="testnet"), record_management=config) + assert managed.record_management is config and managed.inventory.enabled is True + assert managed.inventory.store is not None and managed.inventory.network == "testnet" + + +def test_managed_transact_reserves_inputs_saves_the_transaction_and_journals_submission(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, seen = setup(monkeypatch) + tx_id = transfer(credits, aleo).transact() + assert tx_id == "at1tx1" and [s["id"] for s in seen["submits"]] == ["at1tx1"] + entry = aleo.inventory.store.list(aleo.inventory.scope())[0] + assert (entry.records, entry.status, entry.transaction_id, entry.fee_microcredits) == (("1group",), "submitted", "at1tx1", "100") + assert entry.transaction == seen["builds"][0]["tx"] + assert aleo.inventory.reserved_record_ids() == {"1group"} + assert aleo.inventory.get_record_inventory(RecordAsset("credits.aleo")).available[0].id == "2group" + + +def test_managed_transact_refuses_a_reserved_input_before_proving(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, seen = setup(monkeypatch) + transfer(credits, aleo).transact() + with pytest.raises(RecordReservationConflictError) as conflict: + transfer(credits, aleo).transact() + assert conflict.value.record_ids == ("1group",) and len(seen["builds"]) == 1 + + +def test_unmanaged_client_writes_are_unchanged(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, seen = setup(monkeypatch, enabled=False) + assert transfer(credits, aleo).transact() == "at1tx1" + assert transfer(credits, aleo).transact() == "at1tx2" # the same record spends twice: nothing coordinates + assert aleo.inventory.store is None and len(seen["submits"]) == 2 + + +def test_fee_ceiling_is_enforced_before_broadcast(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, seen = setup(monkeypatch, max_fee=50) + with pytest.raises(InventoryError, match="max_fee_microcredits"): + transfer(credits, aleo).transact() + assert seen["submits"] == [] + assert aleo.inventory.store.list(aleo.inventory.scope())[0].status == "cancelled" + assert aleo.inventory.reserved_record_ids() == set() + + +def test_private_fee_record_is_chosen_after_proving_and_reserved_beside_the_inputs(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, seen = setup(monkeypatch) + transfer(credits, aleo).transact(private_fee=True, priority_fee=20) + build = seen["builds"][0] + assert build["private_fee"] is True and callable(build["fee_record"]) + inputs_entry, fee_entry = aleo.inventory.store.list(aleo.inventory.scope()) + assert (inputs_entry.records, inputs_entry.fee_record_id, inputs_entry.fee_microcredits) == (("1group",), None, "120") + assert (fee_entry.records, fee_entry.fee_record_id, fee_entry.status) == (("3group",), "3group", "submitted") + assert build["tx"]["fee"]["transition"]["inputs"][0]["tag"] == "3field" + + +def test_supplied_fee_record_is_preferred_when_eligible(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, _ = setup(monkeypatch, records=[1_000_000, 500, 200]) + transfer(credits, aleo).transact(fee_record=aleo.record_provider.records[1]["record_plaintext"]) + assert aleo.inventory.store.list(aleo.inventory.scope())[1].fee_record_id == "2group" + + +def test_managed_delegate_proves_without_broadcast_and_submits_from_the_client(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, account, credits, seen = setup(monkeypatch) + payload = transfer(credits, aleo).delegate(account) + assert payload["transaction"]["id"] == "at1dps1" and seen["builds"][0]["delegated"] is True + assert [s["id"] for s in seen["submits"]] == ["at1dps1"] + entry = aleo.inventory.store.list(aleo.inventory.scope())[0] + assert (entry.status, entry.transaction_id, entry.records) == ("submitted", "at1dps1", ("1group",)) + with pytest.raises(RecordReservationConflictError): + transfer(credits, aleo).delegate(account) + + +def test_delegate_without_broadcast_stays_unmanaged(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, account, credits, seen = setup(monkeypatch) + transfer(credits, aleo).delegate(account, broadcast=False) + assert aleo.inventory.store.list(aleo.inventory.scope()) == [] and seen["submits"] == [] + + +def test_lost_broadcast_response_keeps_the_prepared_transaction(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, _ = setup(monkeypatch) + + def lost(raw: str) -> str: + raise ConnectionError("response lost") + + monkeypatch.setattr(aleo.network, "submit_transaction", lost) + with pytest.raises(ConnectionError): + transfer(credits, aleo).transact() + entry = aleo.inventory.store.list(aleo.inventory.scope())[0] + assert entry.status == "prepared" and entry.transaction is not None and entry.transaction_id == "at1tx1" + + +def test_records_owned_by_another_account_cannot_be_reserved(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, seen = setup(monkeypatch) + foreign = record("aleo1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3ljyzc", 5, 9)["record_plaintext"] + with pytest.raises(InventoryError, match="another account"): + credits.functions.transfer_private(foreign, str(aleo.default_account.address), 1).transact() # type: ignore[union-attr] + assert seen["builds"] == [] + + +@pytest.mark.asyncio +async def test_async_managed_transact_runs_the_same_hook(monkeypatch: pytest.MonkeyPatch) -> None: + config = RecordManagementConfig(enabled=True, store=MemoryRecordInventoryStore()) + aleo = AsyncAleo(HTTPProvider(BASE), record_management=config) + account = aleo.account.from_private_key(PrivateKey.random()) + aleo.default_account = account + aleo.record_provider = AsyncProvider([record(str(account.address), 1_000_000, 1)]) + submits: list[dict[str, Any]] = [] + + async def build_transaction(self: AsyncBoundCall, acct: Any = None, **kw: Any) -> FakeResult: + return FakeResult.of(fake_transaction("at1async")) + + async def submit_transaction(raw: str) -> str: + submits.append(json.loads(raw)) + return "at1async" + + monkeypatch.setattr(AsyncBoundCall, "build_transaction", build_transaction) + monkeypatch.setattr(aleo.network, "submit_transaction", submit_transaction) + from aleo.facade.async_client import AsyncProgram + credits = AsyncProgram(aleo, RawProgram.from_source(CREDITS_SOURCE)) + call = credits.functions.transfer_private(aleo.record_provider.records[0]["record_plaintext"], str(account.address), 1) + assert await call.transact() == "at1async" + assert [s["id"] for s in submits] == ["at1async"] + entry = aleo.inventory.store.list(aleo.inventory.scope())[0] + assert (entry.status, entry.records) == ("submitted", ("1group",)) + with pytest.raises(RecordReservationConflictError): + await credits.functions.transfer_private(aleo.record_provider.records[0]["record_plaintext"], str(account.address), 1).transact() diff --git a/sdk/python/tests/test_inventory_assets.py b/sdk/python/tests/test_inventory_assets.py new file mode 100644 index 00000000..3896d1f7 --- /dev/null +++ b/sdk/python/tests/test_inventory_assets.py @@ -0,0 +1,247 @@ +"""Program interface parsing, asset resolution, and record decoding. + +_source: ProvableHQ/veil packages/core/test/inventory/assetDefaults.test.ts and +inventory.test.ts ("scanner-backed inventory"), PR #172 @ 2e324d5. Wallet +grant cases are not ported: the Python SDK only sees decrypted plaintext. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from aleo.inventory import ( + InventoryError, RecordAsset, RecordJoinProgram, TokenJoinRouter, decode_record, inventory_record_id, + parse_program_interface, validate_asset, validate_record_join_program, validate_token_join_router, +) +from aleo.inventory.join_programs import ( + join_program_candidates, supported_join_program, validate_record_join_function, validate_token_join_function, +) + +FIXTURES = Path(__file__).parent / "fixtures" +OWNER = "aleo1owner" + + +def fixture(name: str) -> str: + return (FIXTURES / name).read_text() + + +def program_source(program: str, record: str, fields: str, amount_type: str = "u64") -> str: + return ( + f"program {program};\nrecord {record}:\n owner as address.private;\n{fields}\n" + f"function join:\n input r0 as {record}.record;\n input r1 as {record}.record;\n output r2 as {record}.record;\n" + f"function split:\n input r0 as {record}.record;\n input r1 as {amount_type}.private;\n" + f" output r2 as {record}.record;\n output r3 as {record}.record;\n" + ) + + +def owned(program: str, plaintext: str, *, record_name: str | None = None, spent: bool = False, nonce: int = 1) -> dict[str, object]: + return {"program_name": program, "record_name": record_name, "spent": spent, "tag": f"{nonce}field", + "commitment": f"{nonce}field", "record_plaintext": plaintext} + + +def token_plaintext(amount: int, nonce: int, extra: str = "", field: str = "amount", width: str = "u128") -> str: + return f"{{ owner: {OWNER}.private, {field}: {amount}{width}.private, {extra}_nonce: {nonce}group.public }}" + + +# ── interface parser ───────────────────────────────────────────────────────── + +def test_parses_credits_interface_from_the_compiled_program() -> None: + from aleo import mainnet + interface = parse_program_interface(str(mainnet.Program.credits().source)) + assert interface.id == "credits.aleo" + join = interface.functions["join"] + assert [p.kind for p in join.inputs] == ["record", "record"] and join.inputs[0].type == "credits" + split = interface.functions["split"] + assert split.inputs[1].kind == "plaintext" and split.inputs[1].type == "u64" and split.inputs[1].visibility == "private" + assert len(split.outputs) == 2 + assert interface.records["credits"].field("microcredits") is not None + assert interface.records["credits"].field("owner") is not None + # Finalize blocks declare inputs too; they must not leak into functions. + assert all(p.kind != "plaintext" or p.visibility != "public" or f.name != "join" for f in interface.functions.values() for p in f.inputs) + + +def test_parses_the_dynamic_router_and_external_record_types() -> None: + router = parse_program_interface(fixture("main_aj_arc20_2_15.aleo")) + assert router.id == "main_aj_arc20_2_15.aleo" + join_3 = router.functions["join_3"] + assert join_3.inputs[0].kind == "plaintext" and join_3.inputs[0].type == "identifier" and join_3.inputs[0].visibility == "public" + assert [p.kind for p in join_3.inputs[1:]] == ["dynamic_record"] * 3 + assert join_3.outputs[0].kind == "dynamic_record" + static = parse_program_interface( + "import credits.aleo;\nprogram autojoin_credits_2_10.aleo;\nfunction join_2:\n" + " input r0 as credits.aleo/credits.record;\n input r1 as credits.aleo/credits.record;\n" + " output r2 as credits.aleo/credits.record;\n" + ) + assert static.imports == ("credits.aleo",) + assert static.functions["join_2"].inputs[0].kind == "external_record" + assert static.functions["join_2"].inputs[0].type == "credits.aleo/credits" + + +# ── asset resolution ───────────────────────────────────────────────────────── + +@pytest.mark.parametrize("program,record,field,width,standard", [ + ("test_arc20_eth.aleo", "Token", "amount", "u128", "arc20"), + ("test_usdcx_stablecoin.aleo", "Token", "amount", "u128", "arc22"), +]) +def test_reads_deployed_tokens_without_a_standard(program: str, record: str, field: str, width: str, standard: str) -> None: + capabilities = validate_asset(RecordAsset(program), fixture(program)) + assert capabilities.asset.record_name == record + assert capabilities.asset.amount_field == field + assert capabilities.asset.amount_type == width + assert capabilities.asset.standard == standard + + +def test_credits_needs_no_source_and_rejects_overrides() -> None: + capabilities = validate_asset(RecordAsset("credits.aleo"), None) + assert capabilities.asset.record_name == "credits" and capabilities.asset.amount_field == "microcredits" + with pytest.raises(InventoryError, match="credits record"): + validate_asset(RecordAsset("credits.aleo", record_name="Token"), None) + with pytest.raises(InventoryError, match="credits standard"): + validate_asset(RecordAsset("credits.aleo", standard="arc20"), None) + with pytest.raises(InventoryError, match="not deployed"): + validate_asset(RecordAsset("missing.aleo"), None) + + +@pytest.mark.parametrize("fields,field,want", [ + (" amount as u64.private;\n microcredits as u64.private;", "amount", "u64"), + (" microcredits as u64.private;", "microcredits", "u64"), +]) +def test_prefers_amount_and_falls_back_to_microcredits(fields: str, field: str, want: str) -> None: + capabilities = validate_asset(RecordAsset("custom.aleo"), program_source("custom.aleo", "Balance", fields)) + assert capabilities.asset.record_name == "Balance" + assert capabilities.asset.amount_field == field + assert capabilities.asset.amount_type == want + assert capabilities.asset.standard is None + + +def test_explicit_overrides_apply_and_a_missing_override_is_an_error() -> None: + source = program_source("custom.aleo", "Balance", " value as u64.private;\n amount as u64.private;") + asset = RecordAsset("custom.aleo", record_name="Balance", amount_field="value") + assert validate_asset(asset, source).asset.amount_field == "value" + with pytest.raises(InventoryError, match="amount field"): + validate_asset(RecordAsset("custom.aleo", amount_field="missing"), source) + + +def test_selects_the_native_join_record_even_when_an_unrelated_token_record_exists() -> None: + source = program_source("custom.aleo", "Balance", " amount as u64.private;") + \ + "\nrecord Token:\n owner as address.private;\n amount as u128.private;\n" + assert validate_asset(RecordAsset("custom.aleo"), source).asset.record_name == "Balance" + + +@pytest.mark.parametrize("declaration", ["token_id as field", "sender as address", "recipient as address"]) +def test_excludes_records_declaring_compliance_or_registry_fields(declaration: str) -> None: + source = program_source("custom.aleo", "Balance", f" amount as u64.private;\n {declaration}.private;") + capabilities = validate_asset(RecordAsset("custom.aleo"), source) + assert capabilities.excluded is True + record = owned("custom.aleo", token_plaintext(5, 1, field="amount", width="u64")) + assert decode_record(OWNER, RecordAsset("custom.aleo"), record, capabilities) is None + + +def test_rejects_an_incompatible_split_signature() -> None: + source = program_source("custom.aleo", "Balance", " amount as u64.private;", amount_type="u128") + with pytest.raises(InventoryError, match="incompatible"): + validate_asset(RecordAsset("custom.aleo"), source) + + +# ── record decoding ────────────────────────────────────────────────────────── + +ARC20 = RecordAsset("test_arc20_eth.aleo") + + +def arc20_capabilities(): # type: ignore[no-untyped-def] + return validate_asset(ARC20, fixture("test_arc20_eth.aleo")) + + +def test_decodes_owned_token_records_and_skips_others() -> None: + capabilities = arc20_capabilities() + record = decode_record(OWNER, ARC20, owned(ARC20.program, token_plaintext(10, 1)), capabilities) + assert record is not None and record.amount == 10 and record.id == "1group" and record.commitment == "1field" + assert decode_record(OWNER, ARC20, owned("other.aleo", token_plaintext(10, 2)), capabilities) is None + assert decode_record(OWNER, ARC20, owned(ARC20.program, token_plaintext(10, 3), spent=True), capabilities) is None + assert decode_record("aleo1other", ARC20, owned(ARC20.program, token_plaintext(10, 4)), capabilities) is None + assert decode_record(OWNER, ARC20, owned(ARC20.program, token_plaintext(20, 5, "recipient_bound: true.private, ")), capabilities) is None + compliance = owned(ARC20.program, token_plaintext(30, 6, f"sender: aleo1sender.private, recipient: {OWNER}.private, "), record_name="ComplianceRecord") + assert decode_record(OWNER, ARC20, compliance, capabilities) is None + assert decode_record(OWNER, ARC20, owned(ARC20.program, token_plaintext(0, 7)), capabilities) is None + assert decode_record(OWNER, ARC20, owned(ARC20.program, token_plaintext(10, 8, width="u64")), capabilities) is None + + +def test_refuses_records_without_plaintext_instead_of_reporting_empty_inventory() -> None: + capabilities = arc20_capabilities() + with pytest.raises(InventoryError, match="plaintext"): + decode_record(OWNER, ARC20, {"program_name": ARC20.program, "record_name": "Token"}, capabilities) + with pytest.raises(InventoryError, match="plaintext"): + inventory_record_id({"record_ciphertext": "record1..."}) + + +def test_custom_u32_amount_field_decodes_with_its_declared_width() -> None: + source = program_source("custom.aleo", "Balance", " quantity as u32.private;", amount_type="u32") + asset = RecordAsset("custom.aleo", record_name="Balance", amount_field="quantity") + capabilities = validate_asset(asset, source) + record = owned("custom.aleo", f"{{ owner: {OWNER}.private, quantity: 100u32.private, _nonce: 1group.public }}") + decoded = decode_record(OWNER, asset, record, capabilities) + assert decoded is not None and decoded.amount == 100 + + +# ── join helpers ───────────────────────────────────────────────────────────── + +def static_helper(program: str, asset: str, record: str, low: int, high: int) -> str: + out = f"import {asset};\nprogram {program};\n" + for n in range(low, high + 1): + out += f"function join_{n}:\n" + "".join(f" input r{j} as {asset}/{record}.record;\n" for j in range(n)) + out += f" output r{n} as {asset}/{record}.record;\n\n" + return out + + +def test_validates_helper_and_router_metadata() -> None: + validate_record_join_program(RecordJoinProgram("autojoin_credits_2_10.aleo", "records", (3,))) + with pytest.raises(InventoryError, match="helper"): + validate_record_join_program(RecordJoinProgram("x.aleo", "records", (17,))) + with pytest.raises(InventoryError, match="helper"): + validate_record_join_program(RecordJoinProgram("x.aleo", "program_identifier_then_records", (16,))) + assert validate_token_join_router(TokenJoinRouter("main_aj_arc20_2_15.aleo")) == 15 + with pytest.raises(InventoryError, match="2 to 15"): + validate_token_join_router(TokenJoinRouter("main_aj_arc20_2_15.aleo", 16)) + + +def test_candidates_follow_network_program_and_standard() -> None: + credits = validate_asset(RecordAsset("credits.aleo"), None).asset + usdcx = validate_asset(RecordAsset("test_usdcx_stablecoin.aleo"), fixture("test_usdcx_stablecoin.aleo")).asset + eth = arc20_capabilities().asset + assert [h.program for h in join_program_candidates("testnet", credits, None)] == [ + "autojoin_credits_2_10.aleo", "autojoin_credits_11_14.aleo", "autojoin_credits_15_16.aleo"] + assert [h.program for h in join_program_candidates("testnet", usdcx, None)][0] == "aj_test_usdcx_stablecoin_2_10.aleo" + assert [h.program for h in join_program_candidates("mainnet", eth, None)] == ["main_aj_arc20_2_15.aleo"] + assert join_program_candidates("devnet", credits, None) == () + assert join_program_candidates("testnet", credits, False) == () + # An explicit router handles tokens; credits still get their helpers. + assert join_program_candidates("testnet", eth, TokenJoinRouter("test_aj_arc20_2_15.aleo")) == () + assert join_program_candidates("testnet", credits, TokenJoinRouter("test_aj_arc20_2_15.aleo")) != () + + +def test_helper_signature_checks_record_type_and_count() -> None: + credits = validate_asset(RecordAsset("credits.aleo"), None).asset + helper = RecordJoinProgram("autojoin_credits_2_10.aleo", "records", tuple(range(2, 11))) + good = parse_program_interface(static_helper(helper.program, "credits.aleo", "credits", 2, 10)) + assert supported_join_program(helper, good, credits) == helper + validate_record_join_function(helper, good, credits, 3) + wrong = parse_program_interface(static_helper(helper.program, "other.aleo", "credits", 2, 10)) + assert supported_join_program(helper, wrong, credits) is None + with pytest.raises(InventoryError, match="incompatible"): + validate_record_join_function(helper, wrong, credits, 3) + with pytest.raises(InventoryError, match="limits"): + validate_record_join_function(helper, good, credits, 11) + + +def test_router_signature_checks_identifier_visibility_and_arity() -> None: + router = TokenJoinRouter("test_aj_arc20_2_15.aleo") + source = fixture("main_aj_arc20_2_15.aleo").replace("main_aj_arc20_2_15.aleo", router.program) + validate_token_join_function(router, parse_program_interface(source), 4) + with pytest.raises(InventoryError, match="incompatible"): + validate_token_join_function(router, parse_program_interface(source.replace("identifier.public", "identifier.private")), 4) + with pytest.raises(InventoryError, match="2 to 15"): + validate_token_join_function(router, parse_program_interface(source), 16) + eth = arc20_capabilities().asset + dynamic = RecordJoinProgram(router.program, "program_identifier_then_records", tuple(range(2, 16))) + assert supported_join_program(dynamic, parse_program_interface(source), eth) == dynamic diff --git a/sdk/python/tests/test_inventory_engine.py b/sdk/python/tests/test_inventory_engine.py new file mode 100644 index 00000000..5bba58a9 --- /dev/null +++ b/sdk/python/tests/test_inventory_engine.py @@ -0,0 +1,968 @@ +"""Inventory actions against a fake facade: scans, plans, execution, recovery, helpers, private fees. + +_source: ProvableHQ/veil packages/core/test/inventory/{inventory,autojoinHelpers, +configuration,privateFees}.test.ts, PR #172 @ 2e324d5. Selection, planning, +reservation, and lifecycle handling run for real; only chain, scanner, and +prover I/O are replaced. Wallet ("rpc") cases are not ported. +""" +from __future__ import annotations + +import json +import re +import threading +from pathlib import Path +from typing import Any, Callable + +import pytest + +from aleo.facade.errors import ProgramNotFound, TransactionNotFound +from aleo.inventory import ( + AsyncRecordInventory, InventoryError, InventoryPlan, InventoryTarget, MemoryRecordInventoryStore, RecordAsset, + RecordInventory, RecordManagementConfig, RecordReservationConflictError, RecordSpend, TokenJoinRouter, + build_inventory_plan, +) +from aleo.inventory._engine import BuildTransaction, PreparedTransaction + +FIXTURES = Path(__file__).parent / "fixtures" +OWNER = "aleo1owner" +SCOPE = '["testnet","aleo1owner"]' +CREDITS = RecordAsset("credits.aleo", standard="credits") +ARC20 = RecordAsset("test_arc20_eth.aleo", standard="arc20") +ARC22 = RecordAsset("test_usdcx_stablecoin.aleo", standard="arc22") +ROUTER = TokenJoinRouter("test_aj_arc20_2_15.aleo") +NONCE = re.compile(r"_nonce:\s*(\d+group)") + + +def fixture(name: str) -> str: + return (FIXTURES / name).read_text() + + +def field_of(asset: RecordAsset) -> tuple[str, str]: + return ("microcredits", "u64") if asset.program == "credits.aleo" else ("amount", "u128") + + +def make_record(amount: int, nonce: int, asset: RecordAsset = CREDITS, extra: str = "") -> dict[str, Any]: + field, width = field_of(asset) + return { + "program_name": asset.program, "record_name": "credits" if asset.program == "credits.aleo" else "Token", + "tag": f"{nonce}field", "commitment": f"{nonce}field", "spent": False, + "record_plaintext": f"{{ owner: {OWNER}.private, {field}: {amount}{width}.private, {extra}_nonce: {nonce}group.public }}", + } + + +def static_helper(program: str, asset: str, record: str, low: int, high: int) -> str: + out = f"import {asset};\nprogram {program};\n" + for n in range(low, high + 1): + out += f"function join_{n}:\n" + "".join(f" input r{j} as {asset}/{record}.record;\n" for j in range(n)) + out += f" output r{n} as {asset}/{record}.record;\n\n" + return out + + +class Account: + def __init__(self, address: str = OWNER) -> None: + self.address = address + + +class _Bound: + def __init__(self, world: "World", program: str, function: str, args: tuple[str, ...]) -> None: + self.world, self.program, self.function, self.args = world, program, function, args + + def delegate(self, account: Any = None, *, broadcast: bool = True, pay_own_fee: bool = False, + fee_record: Any = None, priority_fee: int = 0) -> dict[str, Any]: + assert broadcast is False + return {"transaction": self.world.build(self.program, self.function, self.args, fee_record, priority_fee)} + + def build_transaction(self, account: Any = None, *, priority_fee: int = 0, fee_record: Any = None, + private_fee: bool = False, base_fee: int | None = None) -> Any: + tx = self.world.build(self.program, self.function, self.args, fee_record if (private_fee or fee_record) else None, priority_fee) + + class _Raw: + def to_json(self) -> str: + return json.dumps(tx) + + class _Result: + id = tx["id"] + raw = _Raw() + return _Result() + + +class _Functions: + def __init__(self, world: "World", program: str) -> None: + self.world, self.program = world, program + + def __getitem__(self, name: str) -> Callable[..., _Bound]: + return lambda *args: _Bound(self.world, self.program, name, tuple(str(a) for a in args)) + + +class _Program: + def __init__(self, world: "World", program: str) -> None: + self.source = world.sources[program] + self.functions = _Functions(world, program) + + +class _Programs: + def __init__(self, world: "World") -> None: + self.world = world + + def get(self, program_id: str) -> _Program: + self.world.lookups.append(program_id) + if program_id not in self.world.sources: + if self.world.lookup_error is not None: + raise self.world.lookup_error + raise ProgramNotFound(program_id) + return _Program(self.world, program_id) + + +class _Provider: + def __init__(self, world: "World") -> None: + self.world = world + + def find(self, account: Any = None, *, program: str | None = None, unspent: bool = True, **_: Any) -> list[dict[str, Any]]: + self.world.scans += 1 + # A real scanner returns fresh dicts on every scan; copying keeps a + # later spend from mutating records a caller already holds. + found = [dict(r) for r in self.world.records if r["program_name"] == program and not (unspent and r["spent"])] + return self.world.scan_filter(found) if self.world.scan_filter else found + + +class _Network: + def __init__(self, world: "World") -> None: + self.world = world + + def submit_transaction(self, raw: str) -> str: + return self.world.submit(json.loads(raw)) + + def get_confirmed_transaction(self, tx_id: str) -> dict[str, Any]: + if self.world.on_confirmed is not None: + return self.world.on_confirmed(tx_id) + if tx_id not in self.world.transactions: + raise TransactionNotFound(tx_id) + return {"status": self.world.confirm_status, "transaction": self.world.transactions[tx_id]} + + +class _Records: + def status(self) -> dict[str, Any]: + return {"ok": True, "data": {"synced": True}} + + +class FakeAleo: + network_name = "testnet" + process = None + + def __init__(self, world: "World") -> None: + self.default_account: Any = Account() + self.programs = _Programs(world) + self.record_provider = _Provider(world) + self.network = _Network(world) + self.records = _Records() + + +class World: + """Scanner, node, and prover state behind one fake facade.""" + + def __init__(self, amounts: list[int], asset: RecordAsset = CREDITS, token_join: TokenJoinRouter | bool | None = None, *, + enabled: bool = True, helpers: bool = False, store: MemoryRecordInventoryStore | None = None, + max_fee: int | None = None, network: str = "testnet", proving: str = "delegated") -> None: + self.asset = asset + self.records = [make_record(amount, index + 1, asset) for index, amount in enumerate(amounts)] + # The node always serves credits.aleo; the engine never reads its source. + self.sources: dict[str, str] = {"credits.aleo": "program credits.aleo;"} + if asset.program != "credits.aleo": + self.sources[asset.program] = fixture(asset.program) + if isinstance(token_join, TokenJoinRouter): + self.sources[token_join.program] = fixture("main_aj_arc20_2_15.aleo").replace("main_aj_arc20_2_15.aleo", token_join.program) + if helpers: + record = "credits" if asset.program == "credits.aleo" else "Token" + prefix = "autojoin_credits" if asset.program == "credits.aleo" else f"aj_{asset.program[:-5]}" + for low, high in ((2, 10), (11, 14), (15, 16)): + helper = f"{prefix}_{low}_{high}.aleo" + self.sources[helper] = static_helper(helper, asset.program, record, low, high) + dynamic = f"{'main' if network == 'mainnet' else 'test'}_aj_arc20_2_15.aleo" + self.sources[dynamic] = fixture("main_aj_arc20_2_15.aleo").replace("main_aj_arc20_2_15.aleo", dynamic) + self.next_nonce = 2000 + self.sequence = 0 + self.transactions: dict[str, dict[str, Any]] = {} + self.pending: dict[str, tuple[list[str], list[dict[str, Any]]]] = {} + self.builds: list[dict[str, Any]] = [] + self.submits: list[dict[str, Any]] = [] + self.lookups: list[str] = [] + self.scans = 0 + self.submit_mode = "ok" + self.confirm_status = "accepted" + self.on_confirmed: Callable[[str], dict[str, Any]] | None = None + self.scan_filter: Callable[[list[dict[str, Any]]], list[dict[str, Any]]] | None = None + self.lookup_error: Exception | None = None + self.fail_proving = False + self.wrong_fee_tag = False + self.aleo = FakeAleo(self) + self.aleo.network_name = network # type: ignore[misc] + config = RecordManagementConfig(enabled=enabled, store=store, token_join=token_join, # type: ignore[arg-type] + max_fee_microcredits=max_fee, chain_id=network if network in ("mainnet", "testnet") else "local:test") + self.inventory = RecordInventory(self.aleo, config, proving=proving) # type: ignore[arg-type] + self.store = self.inventory.store + self.scope = f'["{config.chain_id}","{OWNER}"]' + + # ── prover and node behaviour ──────────────────────────────────────────── + + def _amount(self, plaintext: str) -> int: + field, width = field_of(self.asset) + match = re.search(rf"{field}:\s*(\d+){width}", plaintext) + assert match, plaintext + return int(match.group(1)) + + def _by_nonce(self, nonce: str) -> dict[str, Any]: + return next(r for r in self.records if NONCE.search(r["record_plaintext"]).group(1) == nonce) # type: ignore[union-attr] + + def build(self, program: str, function: str, inputs: tuple[str, ...], fee_record: Any, priority_fee: int) -> dict[str, Any]: + if self.fail_proving: + raise RuntimeError("Proving failed") + plaintexts = [item for item in inputs if item.lstrip().startswith("{")] + nonces = [NONCE.search(item).group(1) for item in plaintexts] # type: ignore[union-attr] + deduction = 10_000 if self.asset.program == "credits.aleo" else 0 + if function.startswith("join"): + amounts = [sum(self._amount(item) for item in plaintexts)] + elif function == "split": + first = int(re.sub(r"u(64|128)$", "", inputs[1])) + amounts = [first, self._amount(plaintexts[0]) - first - deduction] + else: + amounts = [] # a transfer: inputs spent, no inventory outputs + outputs: list[dict[str, Any]] = [] + for amount in amounts: + outputs.append(make_record(amount, self.next_nonce, self.asset)) + self.next_nonce += 1 + self.sequence += 1 + tx_id = f"at1tx{self.sequence}" + if function.startswith("join_"): + dynamic = inputs[0].startswith("'") + if dynamic: + transitions = [ + {"program": self.asset.program, "function": "join", "outputs": [{"type": "record_with_dynamic_id", "id": "intermediatefield", "dynamic_id": "intermediatedynamicfield"}]}, + {"program": self.asset.program, "function": "join", "outputs": [{"type": "record_with_dynamic_id", "id": outputs[0]["commitment"], "dynamic_id": f"dynamic{self.sequence}field"}]}, + {"program": program, "function": function, "outputs": [{"type": "record_dynamic", "id": f"dynamic{self.sequence}field"}]}, + ] + else: + transitions = [ + {"program": self.asset.program, "function": "join", "outputs": [{"type": "record", "id": outputs[0]["commitment"]}]}, + {"program": program, "function": function, "outputs": [{"type": "external_record", "id": "externalfield"}]}, + ] + else: + transitions = [{"program": program, "function": function, + "outputs": [{"type": "record", "id": record["commitment"]} for record in outputs]}] + if callable(fee_record): + chosen = fee_record(100 + priority_fee) + fee_nonce = NONCE.search(chosen).group(1) # type: ignore[union-attr] + nonces.append(fee_nonce) + tag = "999field" if self.wrong_fee_tag else self._by_nonce(fee_nonce)["tag"] + fee = {"transition": {"program": "credits.aleo", "function": "fee_private", "inputs": [ + {"type": "record", "tag": tag}, {"type": "public", "value": "100u64"}, {"type": "public", "value": f"{priority_fee}u64"}]}} + else: + fee = {"transition": {"program": "credits.aleo", "function": "fee_public", "inputs": [ + {"type": "public", "value": "100u64"}, {"type": "public", "value": f"{priority_fee}u64"}]}} + transaction = {"id": tx_id, "type": "execute", "execution": {"transitions": transitions}, "fee": fee} + self.pending[tx_id] = (nonces, outputs) + self.builds.append({"program": program, "function": function, "inputs": list(inputs), "tx": transaction}) + return transaction + + def submit(self, transaction: dict[str, Any]) -> str: + if self.submit_mode == "offline": + raise ConnectionError("offline before acceptance") + if self.submit_mode == "rejected": + from aleo import AleoNetworkError + raise AleoNetworkError("POST /transaction/broadcast returned 400: stale state root", status=400) + tx_id = str(transaction["id"]) + if tx_id in self.pending: + nonces, outputs = self.pending.pop(tx_id) + for record in self.records: + if NONCE.search(record["record_plaintext"]).group(1) in nonces: # type: ignore[union-attr] + record["spent"] = True + self.records.extend(outputs) + self.transactions[tx_id] = transaction + self.submits.append(transaction) + if self.submit_mode == "lost": + raise ConnectionError("response lost") + return tx_id + + # ── helpers for assertions ─────────────────────────────────────────────── + + def journal(self) -> list[RecordSpend]: + assert self.store is not None + return self.store.list(self.scope) + + def hold(self, *nonces: str, fee: str | None = None, status: str = "reserved") -> RecordSpend | None: + assert self.store is not None + return self.store.acquire(RecordSpend(id="", scope=self.scope, records=nonces, program="credits.aleo", function="join", + status=status, created_at=1, fee_record_id=fee)) # type: ignore[arg-type] + + def available_amounts(self, asset: RecordAsset | None = None) -> list[int]: + return sorted(r.amount for r in self.inventory.get_record_inventory(asset or self.asset).available) + + +FAST = {"timeout": 2.0, "poll_interval": 0.001} + + +# ── Scanner-backed inventory ────────────────────────────────────────────────── + +def test_deduplicates_identities_across_scanner_results() -> None: + w = World([100] * 1001) + w.records.append(dict(w.records[0])) # the scanner repeats a record + inventory = w.inventory.get_record_inventory(CREDITS) + assert len(inventory.available) == 1001 and inventory.balance == 100_100 + + +@pytest.mark.parametrize("asset", [ARC20, ARC22]) +def test_filters_compliance_and_recipient_bound_records(asset: RecordAsset) -> None: + w = World([10], asset) + w.records.append(make_record(20, 2, asset, "recipient_bound: true.private, ")) + w.records.append({**make_record(30, 3, asset, f"sender: aleo1sender.private, recipient: {OWNER}.private, "), "record_name": "ComplianceRecord"}) + assert w.inventory.get_record_inventory(asset).balance == 10 + + +def test_refuses_records_without_plaintext_rather_than_claiming_an_empty_inventory() -> None: + w = World([10]) + w.records = [{"program_name": "credits.aleo", "record_name": "credits", "tag": "1field", "spent": False}] + with pytest.raises(InventoryError, match="plaintext"): + w.inventory.get_record_inventory(CREDITS) + + +# ── Execution and recovery ──────────────────────────────────────────────────── + +@pytest.mark.parametrize("asset", [CREDITS, ARC20, ARC22]) +def test_executes_a_dependent_split_plan_and_reaches_an_idempotent_target(asset: RecordAsset) -> None: + w = World([1_030_000], asset) + target = InventoryTarget(records=4, distribution="balanced") + plan = w.inventory.plan_record_inventory(asset, target) + result = w.inventory.rebalance_record_inventory(plan, **FAST) + assert (result.status, result.completed_steps) == ("complete", 3) + assert w.inventory.plan_record_inventory(asset, target).steps == () + assert all(entry.status == "confirmed" for entry in w.journal()) + assert all(entry.transaction is None for entry in w.journal()) + + +def test_autojoins_only_enough_records_and_retains_the_rest() -> None: + w = World([40, 30, 10], ARC20) + result = w.inventory.auto_join(ARC20, min_amount=60, **FAST) + assert result.completed_steps == 1 + assert w.available_amounts() == [10, 70] + assert w.inventory.auto_join(ARC20, min_amount=60, **FAST).completed_steps == 0 + + +def test_atomically_rejects_competing_spending_before_the_second_proof() -> None: + w = World([1_000_000]) + record = w.inventory.get_record_inventory(CREDITS).available[0] + outcomes: list[Any] = [] + barrier = threading.Barrier(2) + + def attempt(amount: int) -> None: + barrier.wait() + try: + outcomes.append(w.inventory.split_record(CREDITS, record, amount)) + except RecordReservationConflictError as exc: + outcomes.append(exc) + + threads = [threading.Thread(target=attempt, args=(amount,)) for amount in (100_000, 200_000)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert sum(isinstance(item, str) for item in outcomes) == 1 + assert sum(isinstance(item, RecordReservationConflictError) for item in outcomes) == 1 + assert len(w.builds) == 1 + + +def test_enforces_the_fee_ceiling_before_broadcast_and_releases_the_build() -> None: + w = World([1_000_000]) + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + result = w.inventory.rebalance_record_inventory(plan, max_fee_microcredits=99, **FAST) + assert result.status == "interrupted" and "max_fee_microcredits" in str(result.reason) + assert w.submits == [] + assert w.journal()[0].status == "cancelled" + + +def test_retains_a_prepared_transaction_when_the_broadcast_response_is_lost_then_reconciles() -> None: + w = World([1_000_000]) + w.submit_mode = "lost" + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + result = w.inventory.rebalance_record_inventory(plan, **FAST) + assert (result.status, result.transaction_ids) == ("interrupted", ("at1tx1",)) + assert (w.journal()[0].status, w.journal()[0].transaction_id) == ("prepared", "at1tx1") + w.inventory.reconcile_record_inventory() + assert w.journal()[0].status == "confirmed" + + +def test_rebroadcasts_the_exact_saved_transaction_without_rebuilding_its_proof() -> None: + w = World([1_000_000]) + w.submit_mode = "offline" + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + w.inventory.rebalance_record_inventory(plan, **FAST) + saved = w.journal()[0] + w.inventory.reconcile_record_inventory() # a missing transaction keeps the reservation + assert w.journal()[0].status == "prepared" + w.submit_mode = "ok" + w.inventory.reconcile_record_inventory(rebroadcast=True) + assert w.submits[-1] == saved.transaction and len(w.builds) == 1 + w.inventory.reconcile_record_inventory() + assert (w.journal()[0].status, w.journal()[0].transaction) == ("confirmed", None) + + +def test_releases_inputs_only_after_a_confirmed_rejection() -> None: + w = World([1_000_000]) + w.confirm_status = "rejected" + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + assert w.inventory.rebalance_record_inventory(plan, **FAST).status == "interrupted" + assert w.journal()[0].status == "rejected" + w.records = [r for r in w.records if r["commitment"] == "1field"] # the chain kept the input unspent + w.records[0]["spent"] = False + assert len(w.inventory.get_record_inventory(CREDITS).available) == 1 + + +def test_does_not_confuse_a_new_deposit_with_the_expected_outputs() -> None: + w = World([1_000_000]) + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + w.scan_filter = lambda found: found if any(r["commitment"] == "1field" and not r["spent"] for r in found) else [make_record(1_000_000, 9999)] + result = w.inventory.rebalance_record_inventory(plan, timeout=0.05, poll_interval=0.001) + assert (result.status, result.completed_steps) == ("interrupted", 0) + assert "Scanner" in str(result.reason) and len(w.builds) == 1 + + +def test_retains_uncertain_inputs_on_a_confirmation_outage() -> None: + w = World([1_000_000]) + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + + def outage(tx_id: str) -> dict[str, Any]: + raise ConnectionError("network offline") + + w.on_confirmed = outage + assert w.inventory.rebalance_record_inventory(plan, **FAST).status == "interrupted" + with pytest.raises(ConnectionError, match="network offline"): + w.inventory.reconcile_record_inventory() + assert w.journal()[0].status == "submitted" + + +def test_rejects_stale_inputs_modified_plans_and_cancellation() -> None: + w = World([1_000_000]) + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + altered = InventoryPlan.from_dict({**plan.to_dict(), "steps": [{**s, "amount": "1"} for s in plan.to_dict()["steps"]]}) + altered = InventoryPlan(**{**altered.__dict__, "asset": plan.asset}) + with pytest.raises(InventoryError, match="modified"): + w.inventory.rebalance_record_inventory(altered, **FAST) + cancel = threading.Event() + cancel.set() + assert w.inventory.rebalance_record_inventory(plan, cancel=cancel, **FAST).status == "interrupted" + w.records = [] + assert "changed" in str(w.inventory.rebalance_record_inventory(plan, **FAST).reason) + assert w.builds == [] + + +def test_plan_bound_to_another_account_or_chain_is_refused() -> None: + w = World([1_000_000]) + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + foreign = InventoryPlan(**{**plan.__dict__, "scope": '["mainnet","aleo1owner"]'}) + with pytest.raises(InventoryError, match="another account or chain"): + w.inventory.rebalance_record_inventory(foreign, **FAST) + + +def test_default_account_change_mid_run_interrupts_and_keeps_reservations() -> None: + w = World([1_000_000]) + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + + def swap_account(tx_id: str) -> dict[str, Any]: + w.aleo.default_account = Account("aleo1other") + return {"status": "accepted", "transaction": w.transactions[tx_id]} + + w.on_confirmed = swap_account + result = w.inventory.rebalance_record_inventory(plan, **FAST) + assert result.status == "interrupted" and "changed" in str(result.reason) + assert w.journal()[0].status == "confirmed" + + +# ── Configured token batch joins ────────────────────────────────────────────── + +def test_forwards_the_public_identifier_and_dynamic_import_for_routed_joins() -> None: + w = World([100, 100, 100], ARC22, ROUTER) + records = w.inventory.get_record_inventory(ARC22).available + w.inventory.join_records(ARC22, records) + build = w.builds[0] + assert (build["program"], build["function"]) == (ROUTER.program, "join_3") + assert build["inputs"] == ["'test_usdcx_stablecoin'", *(r.plaintext for r in records)] + entry = w.journal()[0] + assert (entry.records, entry.asset_program, entry.status, entry.program) == (("1group", "2group", "3group"), ARC22.program, "submitted", ROUTER.program) + + +@pytest.mark.parametrize("count", [2, 15, 16, 30]) +def test_consolidates_records_in_bounded_router_transactions(count: int) -> None: + w = World([100] * count, ARC20, ROUTER) + plan = w.inventory.plan_record_inventory(ARC20, InventoryTarget(records=1), max_transactions=3) + assert plan.token_join == ROUTER + assert len(plan.steps) == -(-(count - 1) // 14) + result = w.inventory.rebalance_record_inventory(plan, **FAST) + assert (result.status, result.completed_steps) == ("complete", len(plan.steps)) + first = min(count, 15) + assert (w.builds[0]["program"], w.builds[0]["function"]) == (ROUTER.program, f"join_{first}") + assert w.inventory.get_record_inventory(ARC20).balance == count * 100 + entry = w.journal()[0] + assert (entry.program, entry.asset_program, entry.function, entry.status) == (ROUTER.program, ARC20.program, f"join_{first}", "confirmed") + assert entry.records == plan.steps[0].inputs + + +@pytest.mark.parametrize("asset", [ARC20, ARC22]) +def test_preserves_target_lots_then_executes_batch_joins_followed_by_native_splits(asset: RecordAsset) -> None: + w = World([100] * 16, asset, ROUTER) + preserve = w.inventory.plan_record_inventory(asset, InventoryTarget(records=4)) + assert len(preserve.steps) == 1 and len(preserve.steps[0].inputs) == 13 + plan = w.inventory.plan_record_inventory(asset, InventoryTarget(records=4, distribution="balanced")) + assert [step.kind for step in plan.steps] == ["join", "join", "split", "split", "split"] + assert w.inventory.rebalance_record_inventory(plan, **FAST).completed_steps == 5 + assert w.available_amounts(asset) == [400, 400, 400, 400] + assert w.inventory.plan_record_inventory(asset, plan.target).steps == () + + +def test_retains_sufficient_record_selection_and_native_credits_joins() -> None: + w = World([40, 30, 20, 10], ARC20, ROUTER) + assert w.inventory.auto_join(ARC20, min_amount=80, **FAST).completed_steps == 1 + assert w.builds[0]["function"] == "join_3" + assert w.available_amounts() == [10, 90] + native = World([100, 100, 100], CREDITS, ROUTER) + plan = native.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=1)) + assert plan.token_join is None and [len(step.inputs) for step in plan.steps] == [2, 2] + assert native.inventory.rebalance_record_inventory(plan, **FAST).status == "complete" + assert all(b["program"] == "credits.aleo" and b["function"] == "join" for b in native.builds) + + +def test_keeps_batch_reservations_atomic_and_enforces_the_proved_fee_limit() -> None: + w = World([100, 100, 100], ARC20, ROUTER) + records = w.inventory.get_record_inventory(ARC20).available + w.inventory.join_records(ARC20, records) + with pytest.raises(RecordReservationConflictError) as conflict: + w.inventory.join_records(ARC20, records[:2]) + assert conflict.value.record_ids == ("1group", "2group") + assert len(w.builds) == 1 + limited = World([100, 100, 100], ARC20, ROUTER) + plan = limited.inventory.plan_record_inventory(ARC20, InventoryTarget(records=1)) + result = limited.inventory.rebalance_record_inventory(plan, max_fee_microcredits=99, **FAST) + assert result.status == "interrupted" and "max_fee_microcredits" in str(result.reason) + assert limited.submits == [] + assert (limited.journal()[0].status, limited.journal()[0].records) == ("cancelled", ("1group", "2group", "3group")) + + +def test_rebroadcasts_an_interrupted_batch_proof_and_replans_dependent_inventory() -> None: + w = World([100] * 16, ARC22, ROUTER) + w.submit_mode = "offline" + plan = w.inventory.plan_record_inventory(ARC22, InventoryTarget(records=1)) + assert w.inventory.rebalance_record_inventory(plan, **FAST).completed_steps == 0 + saved = w.journal()[0] + assert len(saved.records) == 15 + w.submit_mode = "ok" + w.inventory.reconcile_record_inventory(rebroadcast=True) + assert w.submits[-1] == saved.transaction + w.inventory.reconcile_record_inventory() + assert w.inventory.auto_join(ARC22, **FAST).completed_steps == 1 + assert len(w.builds) == 2 + assert w.inventory.get_record_inventory(ARC22).balance == 1600 + + +def test_rejects_incompatible_router_code_duplicates_overflow_and_excessive_sizes_before_proving() -> None: + w = World([100] * 16, ARC20, ROUTER) + records = w.inventory.get_record_inventory(ARC20).available + with pytest.raises(InventoryError, match="2 to 15"): + w.inventory.join_records(ARC20, records) + with pytest.raises(InventoryError, match="distinct"): + w.inventory.join_records(ARC20, [records[0], records[0]]) + huge = World([(1 << 128) - 1, 1], ARC20, ROUTER) + with pytest.raises(InventoryError, match="overflows"): + huge.inventory.join_records(ARC20, huge.inventory.get_record_inventory(ARC20).available) + w.sources[ROUTER.program] = w.sources[ROUTER.program].replace("identifier.public", "identifier.private") + with pytest.raises(InventoryError, match="incompatible"): + w.inventory.join_records(ARC20, records[:2]) + assert w.builds == [] and huge.builds == [] + + +def test_rejects_a_mismatched_dynamic_output_instead_of_selecting_intermediate_records() -> None: + w = World([100, 100, 100], ARC20, ROUTER) + + def tampered(tx_id: str) -> dict[str, Any]: + tx = json.loads(json.dumps(w.transactions[tx_id])) + tx["execution"]["transitions"][-2]["outputs"][0]["type"] = "record" + return {"status": "accepted", "transaction": tx} + + w.on_confirmed = tampered + plan = w.inventory.plan_record_inventory(ARC20, InventoryTarget(records=1)) + result = w.inventory.rebalance_record_inventory(plan, **FAST) + assert result.status == "interrupted" and "underlying token commitment" in str(result.reason) + assert w.journal()[0].status == "confirmed" and len(w.builds) == 1 + + +def test_honors_custom_batch_ceilings_and_executes_plans_built_without_a_router() -> None: + w = World([100] * 4, ARC20, TokenJoinRouter(ROUTER.program, max_records=3)) + plan = w.inventory.plan_record_inventory(ARC20, InventoryTarget(records=1)) + assert [len(step.inputs) for step in plan.steps] == [3, 2] + native = build_inventory_plan(scope=w.scope, asset=plan.asset, inputs=plan.inputs, target=plan.target) + assert w.inventory.rebalance_record_inventory(native, **FAST).completed_steps == 3 + assert all(b["program"] == ARC20.program for b in w.builds) + + +# ── Automatic helper selection ──────────────────────────────────────────────── + +@pytest.mark.parametrize("count,helper", [(3, "autojoin_credits_2_10.aleo"), (11, "autojoin_credits_11_14.aleo"), + (15, "autojoin_credits_15_16.aleo"), (16, "autojoin_credits_15_16.aleo")]) +def test_uses_one_helper_transaction_for_credits_records(count: int, helper: str) -> None: + w = World([100] * count, helpers=True) + plan = w.inventory.plan_record_inventory(RecordAsset("credits.aleo"), InventoryTarget(records=1), max_transactions=1) + assert len(plan.steps) == 1 + assert w.inventory.rebalance_record_inventory(plan, **FAST).completed_steps == 1 + assert (w.builds[0]["program"], w.builds[0]["function"], len(w.builds[0]["inputs"])) == (helper, f"join_{count}", count) + assert w.inventory.get_record_inventory(CREDITS).balance == count * 100 + assert (w.journal()[0].asset_program, w.journal()[0].status) == ("credits.aleo", "confirmed") + + +def test_chains_a_sixteen_record_helper_output_into_a_native_join() -> None: + w = World([100] * 17, helpers=True) + assert w.inventory.auto_join(RecordAsset("credits.aleo"), max_transactions=2, **FAST).completed_steps == 2 + assert [(b["program"], b["function"]) for b in w.builds] == [("autojoin_credits_15_16.aleo", "join_16"), ("credits.aleo", "join")] + assert any("_nonce: 2000group" in item for item in w.builds[1]["inputs"]) + + +def test_prefers_a_dedicated_stablecoin_helper_over_a_generic_arc_helper() -> None: + w = World([10, 20, 30], ARC22, helpers=True) + asset = RecordAsset("test_usdcx_stablecoin.aleo") + w.inventory.join_records(asset, w.inventory.get_record_inventory(asset).available) + assert (w.builds[0]["program"], w.builds[0]["function"]) == ("aj_test_usdcx_stablecoin_2_10.aleo", "join_3") + assert all(item.lstrip().startswith("{") for item in w.builds[0]["inputs"]) + + +@pytest.mark.parametrize("network", ["mainnet", "testnet"]) +def test_uses_the_dynamic_arc_helper_with_a_program_identifier(network: str) -> None: + w = World([10, 20, 30], ARC20, helpers=True, network=network) + asset = RecordAsset("test_arc20_eth.aleo") + assert w.inventory.auto_join(asset, max_transactions=1, **FAST).completed_steps == 1 + build = w.builds[0] + assert (build["program"], build["function"]) == (f"{'main' if network == 'mainnet' else 'test'}_aj_arc20_2_15.aleo", "join_3") + assert build["inputs"][0] == "'test_arc20_eth'" and len(build["inputs"]) == 4 + + +@pytest.mark.parametrize("options", [{"helpers": False}, {"helpers": True, "token_join": False}, {"helpers": True, "network": "local"}]) +def test_falls_back_to_native_joins_for_unavailable_disabled_or_unknown_network_helpers(options: dict[str, Any]) -> None: + w = World([100, 100, 100], **options) + assert w.inventory.auto_join(RecordAsset("credits.aleo"), **FAST).completed_steps == 2 + assert [b["program"] for b in w.builds] == ["credits.aleo", "credits.aleo"] + + +def test_rejects_a_mismatched_external_record_abi_before_choosing_a_helper() -> None: + w = World([100, 100, 100], helpers=True) + w.sources["autojoin_credits_2_10.aleo"] = static_helper("autojoin_credits_2_10.aleo", "other.aleo", "credits", 2, 10) + plan = w.inventory.plan_record_inventory(RecordAsset("credits.aleo"), InventoryTarget(records=1)) + assert [len(step.inputs) for step in plan.steps] == [2, 2] + + +def test_surfaces_helper_lookup_outages_without_submitting_native_transactions() -> None: + from aleo import AleoNetworkError + w = World([100, 100, 100], helpers=True) + w.sources = {} + w.lookup_error = AleoNetworkError("Gateway unavailable", status=503) + with pytest.raises(AleoNetworkError, match="Gateway unavailable"): + w.inventory.auto_join(RecordAsset("credits.aleo"), **FAST) + assert w.builds == [] + + +def test_keeps_the_helper_chosen_during_planning_and_rejects_a_changed_abi_before_submission() -> None: + w = World([100, 100, 100], helpers=True) + plan = w.inventory.plan_record_inventory(RecordAsset("credits.aleo"), InventoryTarget(records=1)) + w.sources["autojoin_credits_2_10.aleo"] = static_helper("autojoin_credits_2_10.aleo", "other.aleo", "credits", 2, 10) + result = w.inventory.rebalance_record_inventory(plan, **FAST) + assert (result.status, result.completed_steps) == ("interrupted", 0) and "incompatible" in str(result.reason) + assert w.builds == [] + + +def test_uses_helpers_for_an_explicit_join_without_enabling_coordination() -> None: + w = World([100, 100, 100], helpers=True, enabled=False) + asset = RecordAsset("credits.aleo") + w.inventory.join_records(asset, w.inventory.get_record_inventory(asset).available) + assert w.builds[0]["function"] == "join_3" + assert w.inventory.store is None and w.inventory.reserved_record_ids() == set() + + +# ── Configuration ───────────────────────────────────────────────────────────── + +def test_disabled_management_allows_reads_plans_and_single_writes_but_not_execution() -> None: + w = World([100_000, 100_000], enabled=False) + inventory = w.inventory.get_record_inventory(CREDITS) + assert len(inventory.available) == 2 and inventory.reserved == () + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=1)) + assert plan.scope == SCOPE and len(plan.steps) == 1 + assert w.inventory.join_records(CREDITS, inventory.available) == "at1tx1" + assert w.inventory.split_record(CREDITS, w.inventory.get_record_inventory(CREDITS).available[0], 10_000) == "at1tx2" + with pytest.raises(InventoryError, match="record management"): + w.inventory.rebalance_record_inventory(plan, **FAST) + with pytest.raises(InventoryError, match="record management"): + w.inventory.reconcile_record_inventory() + + +def test_enabled_configuration_defaults_to_a_private_in_memory_store_and_validates_settings() -> None: + w = World([1]) + assert isinstance(w.inventory.store, MemoryRecordInventoryStore) + with pytest.raises(InventoryError, match="negative"): + RecordInventory(w.aleo, RecordManagementConfig(enabled=True, max_fee_microcredits=-1)) + with pytest.raises(InventoryError, match="2 to 15"): + RecordInventory(w.aleo, RecordManagementConfig(token_join=TokenJoinRouter(ROUTER.program, 16))) + with pytest.raises(ValueError, match="proving"): + RecordInventory(w.aleo, proving="remote") # type: ignore[arg-type] + + +def test_two_clients_sharing_a_store_see_each_others_reservations() -> None: + store = MemoryRecordInventoryStore() + a = World([100, 100], store=store) + b = World([100, 100], store=store) + b.records = a.records + with a.inventory.reserve("credits.aleo", "transfer_private", ["1group", "2group"]): + snapshot = b.inventory.get_record_inventory(CREDITS) + assert snapshot.available == () and [r.id for r in snapshot.reserved] == ["1group", "2group"] + assert b.inventory.reserved_record_ids() == {"1group", "2group"} + with pytest.raises(RecordReservationConflictError): + b.inventory.join_records(CREDITS, snapshot.reserved) + assert b.inventory.reserved_record_ids() == set() # released: nothing was prepared + + +def test_reserve_context_manager_journals_a_self_managed_write() -> None: + w = World([1_000_000]) + with w.inventory.reserve("credits.aleo", "transfer_private", ["1group"]) as checkpoint: + tx = w.build("credits.aleo", "transfer_private", (w.records[0]["record_plaintext"],), None, 0) + checkpoint.prepared(tx["id"], tx) + checkpoint.submitted(w.submit(tx)) + assert (w.journal()[0].status, w.journal()[0].fee_microcredits) == ("submitted", "100") + with pytest.raises(RecordReservationConflictError): + with w.inventory.reserve("credits.aleo", "join", ["1group"]): + pass + with pytest.raises(RuntimeError): + with w.inventory.reserve("credits.aleo", "join", ["2000group"]): + raise RuntimeError("proving failed") + assert w.journal()[-1].status == "cancelled" + + +# ── Managed private fees ────────────────────────────────────────────────────── + +def private_fee_world(amounts: list[int] | None = None, max_fee: int | None = 150) -> World: + return World(amounts if amounts is not None else [500, 110, 200], max_fee=max_fee) + + +def managed(w: World, *, fee_record: str | None = None, priority: int = 0, inputs: list[str] | None = None) -> str: + spend = inputs if inputs is not None else [w.records[0]["record_plaintext"], "aleo1recipient", "1u64"] + + def builder(request: BuildTransaction) -> PreparedTransaction: + tx = w.build(request.program, request.function, request.inputs, request.fee_record, priority) + return PreparedTransaction(tx["id"], tx) + + return w.inventory.managed_write("credits.aleo", "transfer_private", spend, private_fee=True, fee_record=fee_record, builder=builder) + + +def test_selects_and_journals_a_separate_fee_record_including_priority_fees() -> None: + w = private_fee_world() + managed(w, priority=20) + inputs_entry, fee_entry = w.journal() + assert (inputs_entry.records, inputs_entry.fee_record_id, inputs_entry.fee_microcredits, inputs_entry.status) == (("1group",), None, "120", "submitted") + assert (fee_entry.records, fee_entry.fee_record_id, fee_entry.status) == (("3group",), "3group", "submitted") + assert "microcredits:" not in json.dumps([entry.to_dict() for entry in w.journal()]) + + +def test_uses_a_supplied_fee_record_when_eligible_and_replaces_a_reserved_or_input_one() -> None: + w = private_fee_world() + managed(w, fee_record=w.records[2]["record_plaintext"]) + assert w.journal()[1].fee_record_id == "3group" + w = private_fee_world() + w.hold("3group", status="prepared") + managed(w, fee_record=w.records[2]["record_plaintext"]) + assert w.journal()[-1].fee_record_id == "2group" + w = private_fee_world() + managed(w, fee_record=w.records[0]["record_plaintext"]) + assert w.journal()[-1].fee_record_id == "2group" + + +def test_chooses_disjoint_fee_records_when_independent_writes_race() -> None: + w = private_fee_world([100, 100]) + barrier = threading.Barrier(2) + errors: list[BaseException] = [] + + def run() -> None: + barrier.wait() + try: + managed(w, inputs=["aleo1recipient", "1u64"]) + except BaseException as exc: # noqa: BLE001 + errors.append(exc) + + threads = [threading.Thread(target=run) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert errors == [] + assert {entry.fee_record_id for entry in w.journal() if entry.fee_record_id} == {"1group", "2group"} + + +def test_excludes_reserved_records_and_fails_before_proving_when_nothing_covers_the_fee() -> None: + w = private_fee_world([500, 100, 99]) + w.hold("2group", status="prepared") + with pytest.raises(InventoryError, match="private fee") as error: + managed(w) + assert not isinstance(error.value, RecordReservationConflictError) + assert w.submits == [] and len(w.journal()) == 2 and w.journal()[1].status == "cancelled" + + +def test_reports_input_contention_separately_from_private_fee_exhaustion() -> None: + w = private_fee_world() + w.hold("1group", status="prepared") + with pytest.raises(RecordReservationConflictError) as conflict: + managed(w) + assert conflict.value.record_ids == ("1group",) + assert w.builds == [] and len(w.journal()) == 1 + + +def test_releases_both_inputs_when_proving_fails_before_a_prepared_transaction_exists() -> None: + w = private_fee_world() + w.fail_proving = True + with pytest.raises(RuntimeError, match="Proving failed"): + managed(w) + assert [entry.status for entry in w.journal()] == ["cancelled"] + assert w.inventory.reserved_record_ids() == set() + + +def test_rejects_a_prover_that_spends_a_different_fee_record_before_broadcast() -> None: + w = private_fee_world() + w.wrong_fee_tag = True + with pytest.raises(InventoryError, match="reserved private fee record"): + managed(w) + assert w.submits == [] and {entry.status for entry in w.journal()} == {"cancelled"} + + +def test_retains_the_spent_private_fee_after_a_rejected_execution_while_releasing_its_other_inputs() -> None: + w = private_fee_world() + managed(w) + w.confirm_status = "rejected" + w.inventory.reconcile_record_inventory() + assert w.inventory.reserved_record_ids() == {"2group"} + assert w.hold("2group") is None and w.hold("1group") is not None + + +def test_preserves_both_reservations_when_the_broadcast_outcome_is_unknown() -> None: + w = private_fee_world() + w.submit_mode = "offline" + with pytest.raises(ConnectionError, match="offline"): + managed(w) + assert [(entry.status, entry.records) for entry in w.journal()] == [("prepared", ("1group",)), ("prepared", ("2group",))] + + +def test_a_nested_conflict_after_acquisition_is_not_reported_as_retryable() -> None: + w = World([1_000_000, 500]) + record = w.inventory.get_record_inventory(CREDITS).available[0] + + def builder(request: BuildTransaction) -> PreparedTransaction: + # A caller callback runs another managed write on the same input. + w.inventory.split_record(CREDITS, record, 100) + raise AssertionError("unreachable") + + with pytest.raises(InventoryError, match="after acquiring inputs") as error: + w.inventory.managed_write("credits.aleo", "split", [record.plaintext, "100u64"], builder=builder) + assert isinstance(error.value.__cause__, RecordReservationConflictError) + + +# ── Async twin ──────────────────────────────────────────────────────────────── + +class AsyncFakeAleo: + """Awaitable view over the same world state.""" + + def __init__(self, fake: FakeAleo) -> None: + self._fake = fake + self.network_name = fake.network_name + self.process = None + outer = self + + class _AsyncBound: + def __init__(self, bound: _Bound) -> None: + self._bound = bound + + async def delegate(self, account: Any = None, **kw: Any) -> dict[str, Any]: + return self._bound.delegate(account, **kw) + + async def build_transaction(self, account: Any = None, **kw: Any) -> Any: + return self._bound.build_transaction(account, **kw) + + class _AsyncFunctions: + def __init__(self, functions: _Functions) -> None: + self._functions = functions + + def __getitem__(self, name: str) -> Callable[..., _AsyncBound]: + return lambda *args: _AsyncBound(self._functions[name](*args)) + + class _AsyncProgram: + def __init__(self, program: _Program) -> None: + self.source = program.source + self.functions = _AsyncFunctions(program.functions) + + class _P: + async def get(self, pid: str) -> _AsyncProgram: + return _AsyncProgram(outer._fake.programs.get(pid)) + + class _Prov: + async def find(self, account: Any = None, **kw: Any) -> list[dict[str, Any]]: + return outer._fake.record_provider.find(account, **kw) + + class _N: + async def submit_transaction(self, raw: str) -> str: + return outer._fake.network.submit_transaction(raw) + + async def get_confirmed_transaction(self, tx_id: str) -> dict[str, Any]: + return outer._fake.network.get_confirmed_transaction(tx_id) + + class _R: + async def status(self) -> dict[str, Any]: + return outer._fake.records.status() + + self.programs, self.record_provider, self.network, self.records = _P(), _Prov(), _N(), _R() + + @property + def default_account(self) -> Any: + return self._fake.default_account + + +@pytest.mark.asyncio +async def test_async_client_runs_the_same_engine() -> None: + w = World([1_030_000], ARC20, ROUTER) + inventory = AsyncRecordInventory(AsyncFakeAleo(w.aleo), RecordManagementConfig(enabled=True, store=w.store, token_join=ROUTER)) + plan = await inventory.plan_record_inventory(ARC20, InventoryTarget(records=4, distribution="balanced")) + result = await inventory.rebalance_record_inventory(plan, **FAST) + assert (result.status, result.completed_steps) == ("complete", 3) + assert (await inventory.plan_record_inventory(ARC20, plan.target)).steps == () + journal = await inventory.reconcile_record_inventory() + assert all(entry.status == "confirmed" for entry in journal) + w.records = [] + assert (await inventory.get_record_inventory(ARC20)).available == () + + +# ── Review follow-ups ───────────────────────────────────────────────────────── + +def test_a_node_rejected_broadcast_releases_the_inputs_while_a_lost_response_keeps_them() -> None: + w = World([1_000_000]) + w.submit_mode = "rejected" + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=2)) + result = w.inventory.rebalance_record_inventory(plan, **FAST) + assert result.status == "interrupted" and "400" in str(result.reason) + assert w.journal()[0].status == "cancelled" and w.inventory.reserved_record_ids() == set() + w.submit_mode = "lost" + assert w.inventory.rebalance_record_inventory(plan, **FAST).status == "interrupted" + assert w.journal()[-1].status == "prepared" + + +def test_reconcile_releases_reserved_entries_only_on_request() -> None: + w = World([1_000_000]) + w.hold("1group") # a crashed writer that never proved + assert w.inventory.reserved_record_ids() == {"1group"} + w.inventory.reconcile_record_inventory() + assert w.journal()[0].status == "reserved" + w.inventory.reconcile_record_inventory(release_reserved=True) + assert w.journal()[0].status == "cancelled" and w.inventory.reserved_record_ids() == set() + + +def test_rebalance_stops_before_proving_once_the_fee_budget_is_spent() -> None: + w = World([100] * 3) + plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=1)) + assert len(plan.steps) == 2 + result = w.inventory.rebalance_record_inventory(plan, max_fee_microcredits=100, **FAST) + assert (result.status, result.completed_steps) == ("interrupted", 1) + assert "budget exhausted" in str(result.reason) and len(w.builds) == 1 diff --git a/sdk/python/tests/test_inventory_planner.py b/sdk/python/tests/test_inventory_planner.py new file mode 100644 index 00000000..c3e69ed2 --- /dev/null +++ b/sdk/python/tests/test_inventory_planner.py @@ -0,0 +1,152 @@ +"""Planner known answers. + +_source: ProvableHQ/veil packages/core/test/inventory/inventory.test.ts +("inventory planning") and configuration/autojoin suites, PR #172 @ 2e324d5. +Amounts and step shapes are copied as-is; the planner is a line-for-line port. +""" +from __future__ import annotations + +import pytest + +from aleo.inventory import ( + InventoryError, InventoryPlan, InventoryPlanError, InventoryTarget, PlannedRecord, RecordAsset, RecordJoinProgram, + ResolvedRecordAsset, TokenJoinRouter, build_inventory_plan, +) + +SCOPE = '["testnet","aleo1owner"]' +CREDITS = ResolvedRecordAsset("credits.aleo", standard="credits", record_name="credits", amount_field="microcredits", amount_type="u64") +ARC20 = ResolvedRecordAsset("test_arc20_eth.aleo", standard="arc20", record_name="Token", amount_field="amount", amount_type="u128") +ROUTER = TokenJoinRouter("test_aj_arc20_2_15.aleo") + + +def plan(amounts: list[int], count: int, asset: RecordAsset = ARC20, distribution: str = "balanced", **kw: object) -> InventoryPlan: + return build_inventory_plan( + scope=SCOPE, asset=asset, target=InventoryTarget(records=count, distribution=distribution), # type: ignore[arg-type] + inputs=[PlannedRecord(f"{i}group", amount) for i, amount in enumerate(amounts)], **kw, # type: ignore[arg-type] + ) + + +def test_splits_one_arc_token_record_into_exact_balanced_lots() -> None: + result = plan([100], 4) + assert len(result.steps) == 3 + assert [step.outputs[0].amount for step in result.steps] == [25, 25, 25] + assert result.deduction == 0 + + +def test_accounts_for_every_native_credits_split_deduction() -> None: + result = plan([1_030_000], 4, CREDITS) + assert result.deduction == 30_000 + assert [step.amount for step in result.steps] == [250_000, 250_000, 250_000] + + +def test_does_nothing_for_balanced_inventory_and_tolerates_atomic_rounding() -> None: + assert plan([25, 25, 25, 25], 4).steps == () + assert plan([33, 33, 34], 3).steps == () + + +def test_preserves_existing_records_for_count_only_policies() -> None: + result = plan([10, 15, 25, 50], 3, ARC20, "preserve") + assert len(result.steps) == 1 + assert result.steps[0].inputs == ("0group", "1group") + + +def test_preserves_exact_balanced_lots_and_repacks_small_records() -> None: + assert len(plan([25, 25, 50], 4).steps) == 1 + assert len(plan([250_000, 250_000, 510_000], 4, CREDITS).steps) == 1 + result = build_inventory_plan( + scope=SCOPE, asset=ARC20, inputs=[PlannedRecord("1group", 3), PlannedRecord("2group", 3)], + target=InventoryTarget(records=3, min_record_amount=2), + ) + assert [step.kind for step in result.steps] == ["join", "split", "split"] + assert [record.amount for record in result.steps[-1].outputs] == [2, 2] + + +def test_rejects_impossible_counts_deductions_overflow_and_operation_limits() -> None: + with pytest.raises(InventoryPlanError, match="infeasible"): + plan([10_001], 2, CREDITS) + with pytest.raises(InventoryPlanError, match="records"): + plan([10], 0) + with pytest.raises(InventoryPlanError, match="overflow"): + plan([(1 << 128) - 1, 1], 1) + with pytest.raises(InventoryPlanError, match="max_transactions"): + build_inventory_plan( + scope=SCOPE, asset=ARC20, inputs=[PlannedRecord("1group", 100)], target=InventoryTarget(records=4), + max_transactions=2, + ) + + +@pytest.mark.parametrize("count", [2, 15, 16, 30]) +def test_consolidates_records_in_bounded_router_transactions(count: int) -> None: + result = plan([100] * count, 1, ARC20, "preserve", token_join=ROUTER, max_transactions=3) + assert result.token_join == ROUTER + assert len(result.steps) == -(-(count - 1) // 14) + assert all(2 <= len(step.inputs) <= 15 for step in result.steps) + + +def test_router_is_dropped_for_credits_and_native_pairs_remain() -> None: + result = plan([100, 100, 100], 1, CREDITS, "preserve", token_join=ROUTER) + assert result.token_join is None + assert [len(step.inputs) for step in result.steps] == [2, 2] + + +def test_honors_custom_batch_ceilings() -> None: + result = plan([100] * 4, 1, ARC20, "preserve", token_join=TokenJoinRouter(ROUTER.program, max_records=3)) + assert [len(step.inputs) for step in result.steps] == [3, 2] + with pytest.raises(InventoryError, match="2 to 15"): + plan([100] * 4, 1, ARC20, "preserve", token_join=TokenJoinRouter(ROUTER.program, max_records=16)) + + +def test_preserves_target_lots_with_router_then_splits_natively() -> None: + preserve = plan([100] * 16, 4, ARC20, "preserve", token_join=ROUTER) + assert len(preserve.steps) == 1 and len(preserve.steps[0].inputs) == 13 + balanced = plan([100] * 16, 4, ARC20, "balanced", token_join=ROUTER) + assert [step.kind for step in balanced.steps] == ["join", "join", "split", "split", "split"] + + +STATIC = tuple( + RecordJoinProgram(f"autojoin_credits_{lo}_{hi}.aleo", "records", tuple(range(lo, hi + 1))) + for lo, hi in ((2, 10), (11, 14), (15, 16)) +) + + +@pytest.mark.parametrize("count,helper", [(3, "autojoin_credits_2_10.aleo"), (11, "autojoin_credits_11_14.aleo"), + (15, "autojoin_credits_15_16.aleo"), (16, "autojoin_credits_15_16.aleo")]) +def test_uses_one_static_helper_transaction(count: int, helper: str) -> None: + result = plan([100] * count, 1, CREDITS, "preserve", join_programs=STATIC, max_transactions=1) + assert len(result.steps) == 1 + assert result.steps[0].join_program is not None and result.steps[0].join_program.program == helper + assert len(result.steps[0].inputs) == count + + +def test_chains_a_sixteen_record_helper_into_a_native_join() -> None: + result = plan([100] * 17, 1, CREDITS, "preserve", join_programs=STATIC, max_transactions=2) + assert [(step.join_program.program if step.join_program else None, len(step.inputs)) for step in result.steps] == [ + ("autojoin_credits_15_16.aleo", 16), (None, 2), + ] + + +def test_falls_back_to_native_pairs_without_helpers() -> None: + result = plan([100] * 3, 1, CREDITS, "preserve") + assert [len(step.inputs) for step in result.steps] == [2, 2] + assert all(step.join_program is None for step in result.steps) + + +def test_plan_round_trips_through_dict() -> None: + result = plan([100] * 16, 4, ARC20, "balanced", token_join=ROUTER, join_programs=STATIC) + restored = InventoryPlan.from_dict(result.to_dict()) + assert restored.steps == result.steps + assert restored.token_join == ROUTER + assert restored.join_programs == STATIC + assert restored.asset == RecordAsset(ARC20.program, "arc20", "Token", "amount", "u128") + assert restored.target == result.target + + +def test_rebuilding_a_plan_detects_edits() -> None: + original = plan([1_000_000], 2, CREDITS, "preserve") + rebuilt = build_inventory_plan( + scope=original.scope, asset=original.asset, target=original.target, inputs=original.inputs, + max_transactions=original.max_transactions, token_join=original.token_join, join_programs=original.join_programs, + ) + assert rebuilt == original + edited = InventoryPlan.from_dict({**original.to_dict(), "steps": [{**s, "amount": "1"} for s in original.to_dict()["steps"]]}) + assert edited != original diff --git a/sdk/python/tests/test_inventory_store.py b/sdk/python/tests/test_inventory_store.py new file mode 100644 index 00000000..32469e27 --- /dev/null +++ b/sdk/python/tests/test_inventory_store.py @@ -0,0 +1,103 @@ +"""In-memory journal store: atomic acquisition, held records, patches. + +_source: ProvableHQ/veil packages/core/src/inventory/store.ts and +test/inventory/privateFees.test.ts (rejected fee records), PR #172 @ 2e324d5. +""" +from __future__ import annotations + +import threading + +import pytest + +from aleo.inventory import MemoryRecordInventoryStore, RecordSpend, held_record_ids, holds_records +from aleo.inventory.store import reserved_record_ids + +SCOPE = '["testnet","aleo1owner"]' + + +def entry(*records: str, status: str = "reserved", fee: str | None = None, scope: str = SCOPE) -> RecordSpend: + return RecordSpend(id="", scope=scope, records=records, program="credits.aleo", function="join", + status=status, created_at=1, fee_record_id=fee) # type: ignore[arg-type] + + +def test_acquire_is_all_or_nothing_and_assigns_ids() -> None: + store = MemoryRecordInventoryStore() + first = store.acquire(entry("1group", "2group")) + assert first is not None and first.id == "1" + assert store.acquire(entry("2group", "3group")) is None # overlaps a held input + assert store.acquire(entry("3group")) is not None + with pytest.raises(ValueError, match="Duplicate"): + store.acquire(entry("4group", "4group")) + assert [e.records for e in store.list(SCOPE)] == [("1group", "2group"), ("3group",)] + assert store.list('["mainnet","aleo1owner"]') == [] + + +def test_list_and_acquire_return_copies() -> None: + store = MemoryRecordInventoryStore() + saved = store.acquire(entry("1group")) + assert saved is not None + saved.status = "cancelled" # mutating the copy changes nothing + assert store.list(SCOPE)[0].status == "reserved" + + +def test_update_patches_fields_and_clears_transactions() -> None: + store = MemoryRecordInventoryStore() + saved = store.acquire(entry("1group")) + assert saved is not None + store.update(SCOPE, saved.id, {"status": "prepared", "transaction_id": "at1x", "transaction": {"id": "at1x"}, "fee_microcredits": "100"}) + stored = store.list(SCOPE)[0] + assert (stored.status, stored.transaction_id, stored.transaction, stored.fee_microcredits) == ("prepared", "at1x", {"id": "at1x"}, "100") + store.update(SCOPE, saved.id, {"status": "confirmed", "transaction": None}) + assert store.list(SCOPE)[0].transaction is None + with pytest.raises(KeyError): + store.update(SCOPE, "99", {"status": "cancelled"}) + + +def test_held_records_follow_status_and_fee_records() -> None: + assert holds_records(entry("1group", status="confirmed")) is True + assert holds_records(entry("1group", status="cancelled")) is False + assert holds_records(entry("1group", status="rejected")) is False + rejected_with_fee = entry("1group", "2group", status="rejected", fee="2group") + assert holds_records(rejected_with_fee) is True + assert held_record_ids(rejected_with_fee) == ("2group",) + assert held_record_ids(entry("1group", "2group", status="submitted", fee="2group")) == ("1group", "2group") + + +def test_rejected_entries_release_inputs_but_keep_the_charged_fee_record() -> None: + store = MemoryRecordInventoryStore() + saved = store.acquire(entry("1group", "2group", fee="2group")) + assert saved is not None + store.update(SCOPE, saved.id, {"status": "rejected"}) + assert reserved_record_ids(store, SCOPE) == {"2group"} + assert store.acquire(entry("2group")) is None + assert store.acquire(entry("1group")) is not None + assert reserved_record_ids(None, SCOPE) == set() + + +def test_concurrent_acquisitions_never_share_an_input() -> None: + store = MemoryRecordInventoryStore() + barrier = threading.Barrier(8) + results: list[RecordSpend | None] = [] + lock = threading.Lock() + + def worker() -> None: + barrier.wait() + result = store.acquire(entry("1group")) + with lock: + results.append(result) + + threads = [threading.Thread(target=worker) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert sum(result is not None for result in results) == 1 + + +def test_entries_round_trip_with_veil_field_names() -> None: + saved = RecordSpend(id="1", scope=SCOPE, records=("1group",), program="credits.aleo", function="split", + status="prepared", created_at=5, asset_program=None, fee_record_id=None, + transaction_id="at1x", transaction={"id": "at1x"}, fee_microcredits="7") + data = saved.to_dict() + assert data["createdAt"] == 5 and data["transactionId"] == "at1x" and "assetProgram" not in data + assert RecordSpend.from_dict(data) == saved diff --git a/shield-swap-sdk/AGENTS.md b/shield-swap-sdk/AGENTS.md index a9fad179..3d43c60e 100644 --- a/shield-swap-sdk/AGENTS.md +++ b/shield-swap-sdk/AGENTS.md @@ -546,6 +546,48 @@ Raises: with the wrong number of compliance proof bundles. InsufficientRecordsError: If no record covers a funded side. +### `plan_inventory(self, token: 'str', target: 'InventoryTarget', *, max_transactions: 'Optional[int]' = None, account: 'Any' = None) -> 'InventoryPlan'` + +Plan the joins and splits that give *token* the record layout trades need. + +One trade spends one record, so several trades sold at the same time +need several records that each cover a trade. The plan targets the +token's underlying record program (the records a trader holds), not +the AMM wrapper. Reads the token registry, the program, and the +scanner; signs and reserves nothing. Review ``plan.steps`` and +``plan.deduction`` before executing with :meth:`rebalance_inventory`. + +Args: + token: Symbol such as ``"USDCx"`` or the on-chain token id. + target: Desired count, minimum amount in base units, and distribution. + max_transactions: Operation ceiling, 1 through 10000. Defaults to 100. + account: Signer; defaults to ``aleo.default_account``. + +Raises: + ShieldSwapError: The token is unknown or has no underlying program. + InventoryPlanError: The target cannot be reached from the available records. + +Example:: + + plan = dex.plan_inventory("USDCx", InventoryTarget(records=6, min_record_amount=100_000, distribution="balanced")) + +### `rebalance_inventory(self, token: 'str', target: 'InventoryTarget', *, max_transactions: 'Optional[int]' = None, timeout: 'float' = 120.0, poll_interval: 'float' = 1.0, max_fee_microcredits: 'Optional[int]' = None, cancel: 'Any' = None, account: 'Any' = None) -> 'InventoryResult'` + +Reshape *token*'s underlying records to *target*, one confirmed transaction at a time. + +Plans as :meth:`plan_inventory` does, then executes through the +facade's record inventory: each step reserves its inputs, proves, +broadcasts, and waits for the scanner to show its outputs before the +next step. Requires the facade to be built with record management +enabled. No accepted step is rolled back; an interrupted result names +the reason and every submitted transaction. + +Example:: + + result = dex.rebalance_inventory("USDCx", InventoryTarget(records=6), max_fee_microcredits=5_000_000) + if result.status == "interrupted": + dex.inventory.reconcile_record_inventory() + ### `get_pool(self, pool_key: 'str') -> 'g.PoolState'` Static pool configuration (token pair, fee, decimal scales). diff --git a/shield-swap-sdk/README.md b/shield-swap-sdk/README.md index 2bf46806..ea4cf2dc 100644 --- a/shield-swap-sdk/README.md +++ b/shield-swap-sdk/README.md @@ -53,6 +53,10 @@ It uses an in-memory account and swap handle by default; `ENABLE_JOURNAL` option Releases containing the example support `python -m aleo_shield_swap.examples.first_swap.swap`. +The [concurrent-swaps example](./examples/concurrent-swaps) prepares six equal +USDCx records and submits six swaps at once; see +[Concurrent swaps](#concurrent-swaps) for the mechanism. + ## Agents `AGENTS.md` (generated from the SDK's docstrings — always current) is the @@ -176,6 +180,50 @@ Two conveniences trade secret material for service: `aleo.records.register` shares the account's **view key** with the scanning service, which can then see everything the account owns. +## Concurrent swaps + +A private trade spends one record, so trades submitted at the same time need +separate input records and separate blinded identities. + +Record management on the facade covers the records. Build the `Aleo` client +with `record_management=RecordManagementConfig(enabled=True, store=...)` and +every swap reserves its input record before proving. When two automatic +selections pick the same record, the swap that loses the reservation moves to +the next eligible record from the same scan, smallest first, before any proof +is built. An explicit `token_record` is never replaced, and proving or +broadcast errors are never retried. + +A swap journal covers the identities. `ShieldSwap.from_profile()` or +`dex.journal = Journal(path)` issues each swap its own blinded identity and +keeps the handle needed to claim the output. + +Trades need one covering record each. `dex.plan_inventory(token, target)` +plans the joins and splits that reshape the token's underlying records, and +`dex.rebalance_inventory(...)` executes them one confirmed transaction at a +time through `aleo.inventory`: + +```python +from aleo.inventory import InventoryTarget, RecordManagementConfig + +aleo = Aleo(Aleo.HTTPProvider("https://edge.provable.com/api", network="testnet"), + record_management=RecordManagementConfig(enabled=True)) +aleo.default_account = account +dex = ShieldSwap(aleo) + +plan = dex.plan_inventory("USDCx", InventoryTarget(records=4, min_record_amount=100_000, distribution="balanced")) +result = dex.rebalance_inventory("USDCx", InventoryTarget(records=4, min_record_amount=100_000, distribution="balanced")) + +quote = dex.quote(token_in="USDCx", token_out="ETH", amount_in="0.1") +handles = [dex.swap(quote).delegate(wait=False) for _ in range(4)] # or on a thread pool +``` + +Record coordination is off by default. Without it, concurrent swaps must be +given disjoint `token_record` inputs or run in sequence. The +[concurrent-swaps example](./examples/concurrent-swaps) runs the whole flow on +testnet, with a file-backed reservation store and the recovery steps after an +interruption. The SDK's [record management section](../README.md#manage-private-records) +covers the journal, stores, and reconciliation. + ## Async `AsyncShieldSwap` / `AsyncApiClient` mirror the sync **read, swap and claim** diff --git a/shield-swap-sdk/codegen/gen_context.py b/shield-swap-sdk/codegen/gen_context.py index cfe78869..9c684b1b 100644 --- a/shield-swap-sdk/codegen/gen_context.py +++ b/shield-swap-sdk/codegen/gen_context.py @@ -30,6 +30,7 @@ TIER2_CLIENT = ["confirm_airdrop", "quote", "swap", "claim_swap_output", "create_pool", "mint", "increase_liquidity", "decrease_liquidity", "collect", "burn", "plan_rebalance", "rebalance_position", + "plan_inventory", "rebalance_inventory", "get_pool", "get_slot", "get_swap_output", "get_swap_execution", "get_balances", "get_public_balances", "get_private_balances", "derive_pool_key", "derive_tick_key"] diff --git a/shield-swap-sdk/examples/concurrent-swaps/README.md b/shield-swap-sdk/examples/concurrent-swaps/README.md new file mode 100644 index 00000000..d4e4c662 --- /dev/null +++ b/shield-swap-sdk/examples/concurrent-swaps/README.md @@ -0,0 +1,143 @@ +# Concurrent swaps in Python + +Prepare six balanced USDCx records, submit six USDCx → ETH swaps at the same +time, and claim their outputs. The complete runnable example is +[concurrent_swaps.py](./concurrent_swaps.py); the sections below follow it in +order. + +A private trade spends one record. Six trades started together therefore need +six records that each cover a trade, and the six swaps must not pick the same +record. The example reshapes the account's available USDCx records into six +equal lots, then lets each swap select and reserve its own record. + +## Run the example + +From a Python SDK checkout, with `SHIELD_SWAP_PRIVATE_KEY` set to a funded +**testnet** account with DEX access: + +```sh +python -m pip install ./shield-swap-sdk + +# Reads inventory, lists the maintenance operations, and quotes a trade. No submissions. +python shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py + +# Prepares the records, submits six swaps, and claims each successful output. +python shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py --execute +``` + +Preparing the records can join and split **the account's entire available +USDCx inventory**; records held by other operations are left out of the plan. +At least 0.6 USDCx must be available. Execution uses delegated proving with +the prover's fee master paying network fees. `--count` and `--amount` change +the batch size and the amount per trade. + +## 1. Bind one journal to the client + +Record coordination lives on the facade. Building the `Aleo` client with +`record_management` enabled makes every swap reserve its input record before +proving, save the proved transaction before broadcast, and record the +submission. Trades and inventory maintenance then share one journal. + +```python +from aleo import Aleo, HTTPProvider +from aleo.inventory import RecordManagementConfig + +store = JsonFileRecordInventoryStore(directory / "inventory.json") +aleo = Aleo(HTTPProvider("https://edge.provable.com/api", network="testnet"), + record_management=RecordManagementConfig(enabled=True, store=store, chain_id="aleo:testnet")) +``` + +The SDK's default store is in memory and lasts for one process. The example +defines `JsonFileRecordInventoryStore`, a file-backed store under a file lock, +to show what a durable backend provides: reservations and proved transactions +that survive a restart. A database with a uniqueness constraint on held record +nonces gives the same guarantees across machines. + +## 2. Refuse to start over unresolved work + +Before planning, the example reconciles the journal and checks the swap +journal for unclaimed outputs. A transaction the node has not confirmed keeps +its reservation; a missing confirmation is not evidence that it failed. + +```python +journal = aleo.inventory.reconcile_record_inventory() +pending = [entry for entry in journal if entry.status in ("reserved", "prepared", "submitted")] +``` + +## 3. Plan and review the record layout + +`plan_inventory` reads the token's underlying record program, the scanner, and +the deployed join helpers, and returns the joins and splits that reach the +target. Nothing is reserved or signed until the plan is executed. + +```python +from aleo.inventory import InventoryTarget + +plan = dex.plan_inventory("USDCx", InventoryTarget(records=6, min_record_amount=100_000, + distribution="balanced", tolerance_bps=0), + max_transactions=24) +for step in plan.steps: + print(step.kind, len(step.inputs), [output.amount for output in step.outputs]) +``` + +`plan.deduction` reports the credits a native split removes from its outputs. +Network fees are separate. + +## 4. Execute the plan + +Each step reserves its inputs, proves, broadcasts, waits for confirmation, and +waits until the scanner shows the exact outputs before the next step uses +them. An interrupted result names the reason and every submitted transaction; +no accepted step is rolled back. + +```python +result = aleo.inventory.rebalance_record_inventory(plan, timeout=600.0, poll_interval=3.0) +if result.status != "complete": + raise SystemExit(result.reason) +``` + +## 5. Submit the swaps together + +One quote fixes the output floor for every trade; it reserves no pool +liquidity, so a trade can still fail when the price moves. Each `dex.swap` +selects an input record. When another swap reserved that record first, the +swap moves to the next eligible record from the same scan before proving. +An explicit `token_record` is never replaced. + +```python +quote = dex.quote(token_in="USDCx", token_out="ETH", amount_in="0.1", slippage_bps=50) +outcomes = submit_concurrent_swaps(dex, quote, count=6) +``` + +`submit_concurrent_swaps` runs the swaps on a thread pool, waits for each +successful swap's output, and claims it. One failed trade never discards a +sibling's handle, and a failed claim keeps the handle for a later attempt. + +## Keep recovery data + +The default directory is `.shield-swap/concurrent-swaps/testnet/
/`. +Reuse it for the same account. + +| File | Purpose | +| --- | --- | +| `inventory.json` | Record reservations and proved transactions awaiting broadcast, for reconciliation. | +| `swaps.jsonl` | Blinded identities and swap handles needed to claim outputs. Treat it as key material. | +| `batch.pending` | Remains after an interrupted batch so a later run cannot repeat it unnoticed. | + +Keep these files private and out of version control. **Do not run another +trader for the same account while this example is running.** Several clients +in one process must share the same store object. + +After an interruption: + +1. Stop the previous process. +2. Rebuild the client with the same account, network, and files, and call + `aleo.inventory.reconcile_record_inventory()`. A transaction the node has + not seen keeps its reservation; pass `rebroadcast=True` to resubmit a saved + proved transaction with its original id. +3. Call `dex.collect_all()` to claim every finalized swap recorded in + `swaps.jsonl`. +4. Remove `batch.pending` only after every submitted transaction and claim is + resolved, then preview the inventory again before a new batch. + +The example never rebuilds or repeats a swap after an uncertain submission. diff --git a/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py b/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py new file mode 100644 index 00000000..fe8a5a10 --- /dev/null +++ b/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py @@ -0,0 +1,264 @@ +"""Prepare six balanced USDCx records and swap them for ETH at the same time. + +Preview (reads, plans, and quotes; submits nothing): + + python shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py + +Execute (reshapes the account's available USDCx records, submits six swaps, +claims each output): + + python shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py --execute + +Requires ``SHIELD_SWAP_PRIVATE_KEY`` naming a funded testnet account with DEX +access. Execution spends 0.1 USDCx per swap by default and pays network fees +through the delegated prover's fee master. See README.md for the recovery +steps after an interruption. +""" +from __future__ import annotations + +import argparse +import fcntl +import json +import os +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path +from typing import Any + +from aleo import Aleo, HTTPProvider, testnet +from aleo.inventory import ( + InventoryPlan, InventoryTarget, RecordManagementConfig, RecordSpend, RecordSpendPatch, held_record_ids, +) +from aleo_shield_swap import ShieldSwap +from aleo_shield_swap.journal import Journal +from aleo_shield_swap.types import ClaimResult, SwapHandle, SwapQuote + +NETWORK = "testnet" +ENDPOINT = "https://edge.provable.com/api" +UNRESOLVED = ("reserved", "prepared", "submitted") + + +class JsonFileRecordInventoryStore: + """A reservation journal in one JSON file, shared by processes on one machine. + + The SDK ships only an in-memory store. This class shows what a durable + backend needs: every mutation finishes before the method returns, and + ``acquire`` checks and inserts under one file lock so two processes can + never hold the same record. A database with a uniqueness constraint on + held nonces serves the same contract across machines. + + The file holds record nonces, program and function names, and proved + transactions awaiting broadcast; never keys or record plaintext. + """ + + def __init__(self, path: Path) -> None: + self.path = path + self.path.parent.mkdir(parents=True, exist_ok=True) + self._lock_path = path.with_suffix(path.suffix + ".lock") + self._thread_lock = threading.Lock() + + def _read(self) -> list[RecordSpend]: + if not self.path.exists(): + return [] + return [RecordSpend.from_dict(item) for item in json.loads(self.path.read_text() or "[]")] + + def _write(self, entries: list[RecordSpend]) -> None: + tmp = self.path.with_suffix(".tmp") + tmp.write_text(json.dumps([entry.to_dict() for entry in entries], indent=2)) + os.replace(tmp, self.path) + + def _locked(self) -> Any: + store = self + + class _Guard: + def __enter__(self) -> None: + store._thread_lock.acquire() + self.handle = open(store._lock_path, "a+") + fcntl.flock(self.handle, fcntl.LOCK_EX) + + def __exit__(self, *exc: object) -> None: + fcntl.flock(self.handle, fcntl.LOCK_UN) + self.handle.close() + store._thread_lock.release() + return _Guard() + + def list(self, scope: str) -> list[RecordSpend]: + with self._locked(): + return [entry for entry in self._read() if entry.scope == scope] + + def acquire(self, entry: RecordSpend) -> RecordSpend | None: + if len(set(entry.records)) != len(entry.records): + raise ValueError("Duplicate record inputs") + with self._locked(): + entries = self._read() + wanted = set(entry.records) + for old in entries: + if old.scope == entry.scope and wanted.intersection(held_record_ids(old)): + return None + saved = RecordSpend(**{**entry.__dict__, "id": str(len(entries) + 1)}) + entries.append(saved) + self._write(entries) + return saved + + def update(self, scope: str, entry_id: str, patch: RecordSpendPatch) -> None: + with self._locked(): + entries = self._read() + for entry in entries: + if entry.scope == scope and entry.id == entry_id: + if "status" in patch: + entry.status = patch["status"] + if "transaction_id" in patch: + entry.transaction_id = patch["transaction_id"] + if "transaction" in patch: + entry.transaction = patch["transaction"] + if "fee_microcredits" in patch: + entry.fee_microcredits = patch["fee_microcredits"] + self._write(entries) + return + raise KeyError("Unknown record reservation") + + +@dataclass(frozen=True) +class SwapOutcome: + """One trade's result, kept separate so a failed sibling never hides a successful claim. + + Attributes: + index: Position in the batch, from 1. + handle: The swap handle once submission succeeded; it is what a later + claim needs, so keep it when ``claim`` is ``None``. + claim: The confirmed claim with the output and any refund. + error: Why the swap or the claim failed. + """ + + index: int + handle: SwapHandle | None = None + claim: ClaimResult | None = None + error: BaseException | None = None + + +def submit_concurrent_swaps(dex: ShieldSwap, quote: SwapQuote, count: int, *, claim_timeout: float = 600.0) -> list[SwapOutcome]: + """Submit *count* swaps from one quote at the same time, then claim each successful output. + + Each swap selects its own input record; the facade's record management + reserves it before proving, and a swap whose record another swap took + first moves to the next eligible record. A swap and its claim are two + transactions, so each claim waits for the swap output to appear before + submitting. Nothing is retried after an uncertain submission. + """ + def one(index: int) -> SwapOutcome: + try: + handle = dex.swap(quote).delegate(wait=False) + except BaseException as exc: # noqa: BLE001 - reported per trade, never raised mid-batch + return SwapOutcome(index, error=exc) + try: + claim = dex.claim_swap_output(handle, timeout=claim_timeout).delegate(wait=True) + return SwapOutcome(index, handle=handle, claim=claim) + except BaseException as exc: # noqa: BLE001 - the handle is kept for a later claim + return SwapOutcome(index, handle=handle, error=exc) + + with ThreadPoolExecutor(max_workers=count) as pool: + return list(pool.map(one, range(1, count + 1))) + + +def build_client(private_key: str, directory: Path) -> tuple[Aleo, ShieldSwap]: + """Bind an account, a durable reservation journal, and a swap journal to one client.""" + store = JsonFileRecordInventoryStore(directory / "inventory.json") + aleo = Aleo(HTTPProvider(ENDPOINT, network=NETWORK), + record_management=RecordManagementConfig(enabled=True, store=store, chain_id=f"aleo:{NETWORK}")) + key = testnet.PrivateKey.from_string(private_key) + account = aleo.account.from_private_key(key) + aleo.default_account = account + registration = aleo.records.register(account) # shares the view key with the scanner + if not registration.get("ok"): + raise RuntimeError(f"Record scanner registration failed: {registration.get('error')}") + dex = ShieldSwap(aleo) + dex.journal = Journal(directory / "swaps.jsonl") # blinded identities and claim handles + dex.api.authenticate(str(account.address), lambda message: str(key.sign(message.encode()))) + return aleo, dex + + +def refuse_unresolved_work(aleo: Aleo, dex: ShieldSwap, directory: Path) -> None: + """Stop before planning when an earlier run left reservations or claims unresolved.""" + journal = aleo.inventory.reconcile_record_inventory() + pending = [entry for entry in journal if entry.status in UNRESOLVED] + if pending: + raise SystemExit(f"Reconcile {len(pending)} unresolved operations in {directory / 'inventory.json'} before another batch") + if dex.journal is not None and dex.journal.pending_claims(): + raise SystemExit(f"Claim the pending swaps in {directory / 'swaps.jsonl'} (dex.collect_all()) before another batch") + + +def describe_plan(plan: InventoryPlan) -> None: + print(f"{len(plan.steps)} maintenance operations; credits deduction {plan.deduction} microcredits") + for step in plan.steps: + helper = step.join_program.program if step.join_program else "native" + print(f" {step.kind:5} inputs={len(step.inputs):2} outputs={len(step.outputs)} via {helper}") + + +def concurrent_swaps(*, execute: bool, token_in: str = "USDCx", token_out: str = "ETH", + amount_in: str = "0.1", count: int = 6, directory: Path | None = None) -> list[SwapOutcome]: + """Preview or run inventory maintenance followed by a concurrent swap batch.""" + private_key = os.environ.get("SHIELD_SWAP_PRIVATE_KEY") + if not private_key: + raise SystemExit("Set SHIELD_SWAP_PRIVATE_KEY to a funded testnet account with DEX access") + if not 2 <= count <= 16: + raise SystemExit("count must be from 2 through 16") + address = str(testnet.PrivateKey.from_string(private_key).address) + directory = directory or Path(".shield-swap") / "concurrent-swaps" / NETWORK / address + directory.mkdir(parents=True, exist_ok=True) + + aleo, dex = build_client(private_key, directory) + refuse_unresolved_work(aleo, dex, directory) + + token = dex.api.get_token(token_in) + minimum = int(Decimal(amount_in).scaleb(token.decimals)) + # One record per trade lets every swap start at once. Balanced lots with + # zero tolerance give each trade the same spendable amount. + plan = dex.plan_inventory(token_in, InventoryTarget(records=count, min_record_amount=minimum, + distribution="balanced", tolerance_bps=0), + max_transactions=24) + print(f"{count} concurrent {token_in} -> {token_out} swaps of {amount_in} {token_in}; recovery data in {directory}") + describe_plan(plan) + preview = dex.quote(token_in=token_in, token_out=token_out, amount_in=amount_in, slippage_bps=50) + print(f"Per swap: estimated {preview.estimated_amount_out} {token_out}; minimum {preview.minimum_amount_out} {token_out}") + if not execute: + print("Preview only. Pass --execute to prepare the records, submit the swaps, and claim the outputs.") + return [] + + # A marker survives a crash so a later run does not repeat an unfinished batch. + marker = directory / "batch.pending" + if marker.exists(): + raise SystemExit(f"Resolve the previous batch before removing {marker}") + marker.write_text("") + result = aleo.inventory.rebalance_record_inventory(plan, timeout=600.0, poll_interval=3.0) + if result.status != "complete": + raise SystemExit(f"Maintenance interrupted: {result.reason}; keep {directory}") + + # Quote after maintenance so joins and splits do not consume its lifetime. + # A quote fixes the output floor; it reserves no pool liquidity. + quote = dex.quote(token_in=token_in, token_out=token_out, amount_in=amount_in, slippage_bps=50) + outcomes = submit_concurrent_swaps(dex, quote, count) + for outcome in outcomes: + if outcome.claim is not None and outcome.handle is not None: + print(f"{outcome.index}: {outcome.handle.transaction_id} -> {outcome.claim.transaction_id}; " + f"received {outcome.claim.amount_out} base units; refunded {outcome.claim.amount_remaining}") + elif outcome.handle is not None: + print(f"{outcome.index}: swap {outcome.handle.transaction_id} submitted; claim failed: {outcome.error}", file=sys.stderr) + else: + print(f"{outcome.index}: swap failed: {outcome.error}", file=sys.stderr) + failures = [outcome for outcome in outcomes if outcome.claim is None] + if failures: + raise SystemExit(f"{len(failures)} trades need recovery in {directory}; do not repeat the batch") + marker.unlink() + return outcomes + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Prepare records and run concurrent Shield Swap trades on testnet.") + parser.add_argument("--execute", action="store_true", help="prepare records, submit swaps, and claim outputs") + parser.add_argument("--count", type=int, default=6, help="trades to run at once and records to prepare (2-16)") + parser.add_argument("--amount", default="0.1", help="input amount per trade in token units") + args = parser.parse_args() + concurrent_swaps(execute=args.execute, count=args.count, amount_in=args.amount) diff --git a/shield-swap-sdk/python/aleo_shield_swap/AGENTS.md b/shield-swap-sdk/python/aleo_shield_swap/AGENTS.md index a9fad179..3d43c60e 100644 --- a/shield-swap-sdk/python/aleo_shield_swap/AGENTS.md +++ b/shield-swap-sdk/python/aleo_shield_swap/AGENTS.md @@ -546,6 +546,48 @@ Raises: with the wrong number of compliance proof bundles. InsufficientRecordsError: If no record covers a funded side. +### `plan_inventory(self, token: 'str', target: 'InventoryTarget', *, max_transactions: 'Optional[int]' = None, account: 'Any' = None) -> 'InventoryPlan'` + +Plan the joins and splits that give *token* the record layout trades need. + +One trade spends one record, so several trades sold at the same time +need several records that each cover a trade. The plan targets the +token's underlying record program (the records a trader holds), not +the AMM wrapper. Reads the token registry, the program, and the +scanner; signs and reserves nothing. Review ``plan.steps`` and +``plan.deduction`` before executing with :meth:`rebalance_inventory`. + +Args: + token: Symbol such as ``"USDCx"`` or the on-chain token id. + target: Desired count, minimum amount in base units, and distribution. + max_transactions: Operation ceiling, 1 through 10000. Defaults to 100. + account: Signer; defaults to ``aleo.default_account``. + +Raises: + ShieldSwapError: The token is unknown or has no underlying program. + InventoryPlanError: The target cannot be reached from the available records. + +Example:: + + plan = dex.plan_inventory("USDCx", InventoryTarget(records=6, min_record_amount=100_000, distribution="balanced")) + +### `rebalance_inventory(self, token: 'str', target: 'InventoryTarget', *, max_transactions: 'Optional[int]' = None, timeout: 'float' = 120.0, poll_interval: 'float' = 1.0, max_fee_microcredits: 'Optional[int]' = None, cancel: 'Any' = None, account: 'Any' = None) -> 'InventoryResult'` + +Reshape *token*'s underlying records to *target*, one confirmed transaction at a time. + +Plans as :meth:`plan_inventory` does, then executes through the +facade's record inventory: each step reserves its inputs, proves, +broadcasts, and waits for the scanner to show its outputs before the +next step. Requires the facade to be built with record management +enabled. No accepted step is rolled back; an interrupted result names +the reason and every submitted transaction. + +Example:: + + result = dex.rebalance_inventory("USDCx", InventoryTarget(records=6), max_fee_microcredits=5_000_000) + if result.status == "interrupted": + dex.inventory.reconcile_record_inventory() + ### `get_pool(self, pool_key: 'str') -> 'g.PoolState'` Static pool configuration (token pair, fee, decimal scales). diff --git a/shield-swap-sdk/python/aleo_shield_swap/_calls.py b/shield-swap-sdk/python/aleo_shield_swap/_calls.py index f14cb560..65d4c6e1 100644 --- a/shield-swap-sdk/python/aleo_shield_swap/_calls.py +++ b/shield-swap-sdk/python/aleo_shield_swap/_calls.py @@ -12,9 +12,63 @@ """ from __future__ import annotations -from typing import Any, Callable, Generic, TypeVar +from typing import Any, Callable, Generic, Optional, TypeVar + +from aleo.inventory import RecordReservationConflictError, managed_inventory + +from ._core import record_nonce R = TypeVar("R") +T = TypeVar("T") + + +class RecordReselection: + """Move a prepared swap to its next eligible input record after a reservation conflict. + + Built by ``swap()`` when the facade has record management enabled and the + caller let the SDK choose the record. The candidates come from one scan, + smallest first, so the number of attempts is bounded. Only a conflict + that names the current record advances the selection; every other error + propagates, and an explicit ``token_record`` never reselects. + + Args: + candidates: Eligible record plaintexts, smallest first. + assemble: Rebuilds the bound call for a record; the swap's amount, + slippage floor, identity, and nonce are fixed inside it. + guard: Raises when the account, network, or record management + changed since the swap was prepared. + """ + + def __init__(self, candidates: list[str], assemble: Callable[[str], Any], + guard: Callable[[], None]) -> None: + if not candidates: + raise ValueError("RecordReselection needs at least one candidate") + self._candidates = list(candidates) + self._index = 0 + self._assemble = assemble + self._guard = guard + + @property + def current(self) -> str: + """The record plaintext the prepared call currently spends.""" + return self._candidates[self._index] + + @property + def remaining(self) -> int: + """Candidates not yet tried after the current one.""" + return len(self._candidates) - self._index - 1 + + def next_bound(self, error: RecordReservationConflictError) -> Any: + """The bound call for the next candidate, or ``None`` when the error is not about this record or none remain.""" + if record_nonce(self.current) not in error.record_ids: + return None + if self._index + 1 >= len(self._candidates): + return None + self._index += 1 + self._guard() + return self._assemble(self.current) + + def extract_tx_id(payload: Any) -> str: @@ -76,12 +130,14 @@ class DexCall(Generic[R]): def __init__(self, aleo: Any, bound: Any, build_result: Callable[[str, list[str]], R], *, build_before_wait: bool = False, - on_confirmed: Callable[[R], None] | None = None) -> None: + on_confirmed: Callable[[R], None] | None = None, + reselect: Optional[RecordReselection] = None) -> None: self._aleo = aleo self._bound = bound self._build = build_result self._build_before_wait = build_before_wait self._on_confirmed = on_confirmed + self._reselect = reselect def __repr__(self) -> str: return f"DexCall({self._bound!r})" @@ -90,6 +146,17 @@ def simulate(self, account: Any = None) -> Any: """Local authorization — no proof, no send; inspect before spending.""" return self._bound.simulate(account) + def _submit(self, submit: Callable[[Any], T]) -> T: + """Run *submit* on the bound call, moving to the next candidate record on a reservation conflict.""" + while True: + try: + return submit(self._bound) + except RecordReservationConflictError as exc: + replacement = self._reselect.next_bound(exc) if self._reselect is not None else None + if replacement is None: + raise + self._bound = replacement + def transact(self, account: Any = None, *, wait: bool = False, wait_timeout: float = 180.0, **fee_kwargs: Any) -> R: """Prove locally, broadcast, and build the typed result. @@ -97,15 +164,29 @@ def transact(self, account: Any = None, *, wait: bool = False, Root-transition outputs are harvested from the built transaction before broadcast, so the result is complete without waiting for confirmation. Set ``wait=True`` to confirm and update an attached claim journal. + + On a facade with record management enabled the record inputs are + reserved before proving and the proved transaction is saved before + broadcast. When another writer holds the selected record, a swap + that chose its record automatically moves to the next eligible one + before proving; an explicit record is never replaced. """ - tx = self._bound.build_transaction(account, **fee_kwargs) + def submit(bound: Any) -> tuple[str, Any]: + if managed_inventory(self._aleo) is not None: + # The facade verb reserves, journals, and broadcasts; it hands + # back the built transaction for output harvesting. + return bound.transact_with_result(account, **fee_kwargs) + tx = bound.build_transaction(account, **fee_kwargs) + # Same submit path the facade's BoundCall.transact uses. + self._aleo.network.submit_transaction(tx.raw) + return tx.id, tx + + tx_id, tx = self._submit(submit) outputs = root_outputs(tx.decoded(), self._bound.program_id, self._bound.function_name) - # Same submit path the facade's BoundCall.transact uses. - self._aleo.network.submit_transaction(tx.raw) - result = self._build(tx.id, outputs) + result = self._build(tx_id, outputs) if wait: - self._aleo.network.wait_for_transaction(tx.id, timeout=wait_timeout) + self._aleo.network.wait_for_transaction(tx_id, timeout=wait_timeout) if self._on_confirmed is not None: self._on_confirmed(result) return result @@ -119,8 +200,13 @@ def delegate(self, account: Any = None, *, wait: bool = True, the full transaction, so ``wait=False`` returns as soon as the broadcast is accepted — callers that read chain state later (e.g. ``collect_all``) don't need to block on confirmation here. + + On a facade with record management enabled the prover returns the + transaction, the journal saves it, and this client broadcasts it; + a reservation conflict on an automatically selected record moves the + swap to its next eligible record before proving. """ - payload = self._bound.delegate(account, **fee_kwargs) + payload = self._submit(lambda bound: bound.delegate(account, **fee_kwargs)) tx_id = extract_tx_id(payload) decoded = _payload_transitions(payload) confirmed = wait or decoded is None diff --git a/shield-swap-sdk/python/aleo_shield_swap/_core.py b/shield-swap-sdk/python/aleo_shield_swap/_core.py index db0b9824..a6c8ef2d 100644 --- a/shield-swap-sdk/python/aleo_shield_swap/_core.py +++ b/shield-swap-sdk/python/aleo_shield_swap/_core.py @@ -14,11 +14,12 @@ import secrets from dataclasses import dataclass from decimal import Decimal, InvalidOperation -from typing import Any, Optional +from typing import Any, Callable, Optional from aleo.codegen.runtime import parse_plaintext +from aleo.inventory import managed_inventory -from .errors import InsufficientRecordsError +from .errors import InsufficientRecordsError, ShieldSwapError from .tick_math import MAX_SQRT_RATIO_X128, MIN_SQRT_RATIO_X128, u256_to_int @@ -218,20 +219,34 @@ def parse_unsigned_literal(raw: Optional[str], program: str, key: str) -> int: return int(match.group(1)) -def pick_covering_record(records: Any, *, min_amount: int, - token_id: Optional[str], - exclude: Optional[set[str]] = None) -> Optional[str]: - """Smallest unspent token-record plaintext covering *min_amount*, or None. +_NONCE_FIELD = re.compile(r"_nonce:\s*(\d+group)") + + +def record_nonce(plaintext: str) -> Optional[str]: + """The ``_nonce`` of a record plaintext, the identity a reservation journal uses.""" + match = _NONCE_FIELD.search(plaintext) + return match.group(1) if match else None + + +def token_record_candidates(records: Any, *, min_amount: int, + token_id: Optional[str], + exclude: Optional[set[str]] = None, + exclude_nonces: Optional[set[str]] = None) -> list[str]: + """Eligible token-record plaintexts covering *min_amount*, smallest first. *token_id* filters registry-style records; wrapper-program records carry - no ``token_id`` and match any. *exclude* skips records already assigned - (e.g. to earlier swaps of a concurrent batch). + no ``token_id`` and match any. *exclude* skips plaintexts already + assigned (earlier swaps of a batch); *exclude_nonces* skips records held + by a reservation journal even though the scanner still reports them + unspent. One scan bounds a swap's reselection attempts. """ candidates: list[tuple[int, str]] = [] for rec in records: plaintext = record_plaintext(rec) if not plaintext or (exclude and plaintext in exclude): continue + if exclude_nonces and record_nonce(plaintext) in exclude_nonces: + continue info = parse_token_record_info(plaintext) if info is None or info["amount"] < min_amount: continue @@ -240,7 +255,20 @@ def pick_covering_record(records: Any, *, min_amount: int, if token_id is not None and "token_id" in info and info["token_id"] != token_id: continue candidates.append((info["amount"], plaintext)) - return min(candidates)[1] if candidates else None + return [plaintext for _, plaintext in sorted(candidates)] + + +def pick_covering_record(records: Any, *, min_amount: int, + token_id: Optional[str], + exclude: Optional[set[str]] = None, + exclude_nonces: Optional[set[str]] = None) -> Optional[str]: + """Smallest unspent token-record plaintext covering *min_amount*, or None. + + See :func:`token_record_candidates` for the filters. + """ + candidates = token_record_candidates(records, min_amount=min_amount, token_id=token_id, + exclude=exclude, exclude_nonces=exclude_nonces) + return candidates[0] if candidates else None def record_plaintext(rec: Any) -> Optional[str]: @@ -435,20 +463,78 @@ def parse_token_record_info(plaintext: str) -> Optional[dict[str, Any]]: return info -def select_token_record( +def reserved_nonces(aleo: Any, account: Any = None) -> set[str]: + """Record nonces the client's reservation journal currently holds. + + Empty when the facade was built without record management, so selection + behaves as before on an unmanaged client. + """ + inventory = managed_inventory(aleo) + return set() if inventory is None else set(inventory.reserved_record_ids(account)) + + +def underlying_program_for(tokens: Any, token: str) -> str: + """The program whose records fund trades of *token*, from the token registry. + + *token* is a symbol (case-insensitive) or an on-chain token id. Inventory + maintenance works on the underlying program a trader holds records of, + not on the AMM wrapper used while routing a swap. + + Raises: + ShieldSwapError: The token is not listed or has no underlying program. + """ + wanted = token.strip() + for tok in tokens: + symbol = str(getattr(tok, "symbol", "") or "") + if wanted == str(tok.address) or symbol.lower() == wanted.lower(): + program = getattr(tok, "underlying_program", None) + if not program: + raise ShieldSwapError(f"Token {symbol or wanted} has no underlying record program") + return str(program) + raise ShieldSwapError(f"Unknown token {token!r}; pass a symbol or on-chain token id from api.get_tokens()") + + +def swap_context_guard(aleo: Any, acct: Any, resolve_default: Optional[Callable[[], Any]]) -> Callable[[], None]: + """A check that a swap's signer, network, and reservation journal are unchanged. + + Run before a replacement record is bound after a reservation conflict, so + a retry never carries a prepared identity or quote into a different + account, chain, or coordination store. *resolve_default* re-reads the + client's default account when the swap was prepared without an explicit + one. + + Raises: + ShieldSwapError: From the returned check, when anything changed. + """ + address = str(acct.address) + network = str(aleo.network_name) + management = getattr(aleo, "record_management", None) + + def check() -> None: + current = resolve_default() if resolve_default is not None else acct + if (str(current.address) != address or str(aleo.network_name) != network + or getattr(aleo, "record_management", None) is not management): + raise ShieldSwapError("Account, chain, or record management changed during swap preparation") + + return check + + +def select_token_records( aleo: Any, *, program: str, min_amount: int, token_id: Optional[str] = None, account: Any = None, -) -> str: - """One unspent record plaintext from *program* covering *min_amount*. +) -> list[str]: + """Every unspent record plaintext from *program* covering *min_amount*, smallest first. - Scans via ``aleo.record_provider.find`` and picks the smallest covering - record (leaves larger records intact for larger trades). *token_id* - filters registry-style records; wrapper-program records carry no - ``token_id`` and match any. + Scans once via ``aleo.record_provider.find`` and skips records the + reservation journal holds. The list bounds how many times a swap may + reselect after a reservation conflict. + + Raises: + InsufficientRecordsError: No provider is configured or no record covers the amount. """ provider = aleo.record_provider if provider is None: @@ -457,10 +543,30 @@ def select_token_record( "pass token_record= explicitly or configure a scanner." ) records = provider.find(account, program=program, unspent=True) - chosen = pick_covering_record(records, min_amount=min_amount, token_id=token_id) - if chosen is None: + candidates = token_record_candidates(records, min_amount=min_amount, token_id=token_id, + exclude_nonces=reserved_nonces(aleo, account)) + if not candidates: raise InsufficientRecordsError( f"No unspent {program} record covers {min_amount} " f"(token_id={token_id or 'any'}) — privatize funds or pass token_record=." ) - return chosen + return candidates + + +def select_token_record( + aleo: Any, + *, + program: str, + min_amount: int, + token_id: Optional[str] = None, + account: Any = None, +) -> str: + """One unspent record plaintext from *program* covering *min_amount*. + + Scans via ``aleo.record_provider.find`` and picks the smallest covering + record (leaves larger records intact for larger trades), skipping records + the reservation journal holds. *token_id* filters registry-style records; + wrapper-program records carry no ``token_id`` and match any. + """ + return select_token_records(aleo, program=program, min_amount=min_amount, + token_id=token_id, account=account)[0] diff --git a/shield-swap-sdk/python/aleo_shield_swap/async_client.py b/shield-swap-sdk/python/aleo_shield_swap/async_client.py index 3067ecf2..0bf163c2 100644 --- a/shield-swap-sdk/python/aleo_shield_swap/async_client.py +++ b/shield-swap-sdk/python/aleo_shield_swap/async_client.py @@ -13,14 +13,22 @@ import time import logging from decimal import Decimal -from typing import Any, Callable, Generic, Optional, TypeVar +from typing import Any, Awaitable, Callable, Generic, Optional, TypeVar from aleo import AleoNetworkError, ProgramNotFound from aleo.compliance import AsyncFreezeListProofs, ComplianceProofContext, async_freezelist_program +from aleo.inventory import ( + AsyncRecordInventory, InventoryPlan, InventoryResult, InventoryTarget, RecordAsset, RecordReservationConflictError, + managed_inventory, +) from . import _generated as g -from ._calls import extract_tx_id, root_outputs +from ._calls import RecordReselection, extract_tx_id, root_outputs from ._core import ( + reserved_nonces, + swap_context_guard, + token_record_candidates, + underlying_program_for, _amount_to_base_units, decode_position_record, default_merkle_proofs, @@ -62,6 +70,7 @@ InsufficientRecordsError, PoolNotFoundError, PoolNotInitializedError, + ShieldSwapError, SwapOutputNotFinalizedError, ) from ._quotes import quote_request, build_quote, quote_amounts, resolve_quote, multi_hop_inputs @@ -80,14 +89,19 @@ R = TypeVar("R") +T = TypeVar("T") + + class AsyncDexCall(Generic[R]): """Async mirror of :class:`~aleo_shield_swap._calls.DexCall`.""" def __init__(self, aleo: Any, bound: Any, - build_result: Callable[[str, list[Any]], R]) -> None: + build_result: Callable[[str, list[Any]], R], *, + reselect: Optional[RecordReselection] = None) -> None: self._aleo = aleo self._bound = bound self._build = build_result + self._reselect = reselect def simulate(self, account: Any = None) -> Any: """Local authorization — no proof, no send; inspect before spending. @@ -97,6 +111,17 @@ def simulate(self, account: Any = None) -> Any: """ return self._bound.simulate(account) + async def _submit(self, submit: Callable[[Any], Awaitable[T]]) -> T: + """Run *submit* on the bound call, moving to the next candidate record on a reservation conflict.""" + while True: + try: + return await submit(self._bound) + except RecordReservationConflictError as exc: + replacement = self._reselect.next_bound(exc) if self._reselect is not None else None + if replacement is None: + raise + self._bound = replacement + async def transact(self, account: Any = None, *, wait: bool = False, wait_timeout: float = 180.0, **fee_kwargs: Any) -> R: """Prove locally, broadcast, and build the typed result. @@ -104,14 +129,23 @@ async def transact(self, account: Any = None, *, wait: bool = False, Root-transition outputs are harvested from the built transaction before broadcast, so the result is complete without waiting for confirmation. Set ``wait=True`` to wait for acceptance on chain. + With record management enabled on the facade, inputs are reserved + before proving and an automatically selected record is replaced on + a reservation conflict; see :meth:`~aleo_shield_swap._calls.DexCall.transact`. """ - tx = await self._bound.build_transaction(account, **fee_kwargs) + async def submit(bound: Any) -> tuple[str, Any]: + if managed_inventory(self._aleo) is not None: + return await bound.transact_with_result(account, **fee_kwargs) + tx = await bound.build_transaction(account, **fee_kwargs) + await self._aleo.network.submit_transaction(tx.raw) + return tx.id, tx + + tx_id, tx = await self._submit(submit) outputs = root_outputs(tx.decoded(), self._bound.program_id, self._bound.function_name) - await self._aleo.network.submit_transaction(tx.raw) - result = self._build(tx.id, outputs) + result = self._build(tx_id, outputs) if wait: - await self._aleo.network.wait_for_transaction(tx.id, timeout=wait_timeout) + await self._aleo.network.wait_for_transaction(tx_id, timeout=wait_timeout) return result async def delegate(self, account: Any = None, **fee_kwargs: Any) -> R: @@ -121,9 +155,14 @@ async def delegate(self, account: Any = None, **fee_kwargs: Any) -> R: Outputs are harvested from the DPS payload itself when it carries the full transaction, so ``wait=False`` returns as soon as the broadcast is accepted — callers that read chain state later (e.g. - ``collect_all``) don't need to block on confirmation here. + ``collect_all``) don't need to block on confirmation here. With + record management enabled, an automatically selected record is + replaced on a reservation conflict before proving. """ - payload = await self._bound.delegate(account, **fee_kwargs) + async def submit(bound: Any) -> Any: + return await bound.delegate(account, **fee_kwargs) + + payload = await self._submit(submit) tx_id = extract_tx_id(payload) await self._aleo.network.wait_for_transaction(tx_id) tx = await self._aleo.network.get_transaction_object(tx_id) @@ -150,6 +189,10 @@ def __init__(self, aleo: Any, *, program: str = g.PROGRAM_ID, self.amm_freezelist_program: str = SHIELD_SWAP_FREEZELIST #: Freeze-list exclusion proofs, cached per list until its root changes. self.freezelist = AsyncFreezeListProofs(aleo) + #: Private record inventory of the bound client; see :attr:`ShieldSwap.inventory`. + self.inventory: AsyncRecordInventory = ( + aleo.inventory if hasattr(aleo, "inventory") else AsyncRecordInventory(aleo) + ) # allow_token relationships are immutable — cache probes for the # client's lifetime. self._wrapped_cache: dict[str, bool] = {} @@ -344,21 +387,24 @@ async def has_swap_balance(self, token_in_id: str, amount_in: int | str | Decima amount = _amount_to_base_units(amount_in, token_in_id, tokens) program = await self._token_program(token_in_id) records = await provider.find(acct, program=program, unspent=True) - return pick_covering_record(records, min_amount=amount, token_id=token_in_id) is not None + return pick_covering_record(records, min_amount=amount, token_id=token_in_id, + exclude_nonces=reserved_nonces(self._aleo, acct)) is not None - async def _select_token_record(self, *, program: str, min_amount: int, - token_id: Optional[str], account: Any) -> str: + async def _select_token_records(self, *, program: str, min_amount: int, + token_id: Optional[str], account: Any) -> list[str]: + """Eligible, unreserved record plaintexts covering *min_amount*, smallest first, from one scan.""" provider = self._aleo.record_provider if provider is None: raise InsufficientRecordsError( "No record provider configured — pass token_record= explicitly.") records = await provider.find(account, program=program, unspent=True) - chosen = pick_covering_record(records, min_amount=min_amount, token_id=token_id) - if chosen is None: + candidates = token_record_candidates(records, min_amount=min_amount, token_id=token_id, + exclude_nonces=reserved_nonces(self._aleo, account)) + if not candidates: raise InsufficientRecordsError( f"No unspent {program} record covers {min_amount} " f"(token_id={token_id or 'any'}) — privatize funds or pass token_record=.") - return chosen + return candidates async def _select_position_record(self, pool_key: str, account: Any, position_token_id: Optional[str] = None) -> str: @@ -777,11 +823,15 @@ async def swap(self, quote: Optional[SwapQuote] = None, *, pool_key: Optional[st record = token_record program = token_in_program + candidates: list[str] = [] if record is None: program = program or await self._token_program(token_in_id) - record = await self._select_token_record( + # Keep every eligible candidate from one scan so a reservation + # conflict at submission can move to the next record. + candidates = await self._select_token_records( program=program, min_amount=amount_in, token_id=token_in_id, account=acct) + record = candidates[0] token_programs = [program] else: token_programs = [token_in_program] if token_in_program else [] @@ -807,26 +857,34 @@ async def swap(self, quote: Optional[SwapQuote] = None, *, pool_key: Optional[st compliance_proofs, wrapper_proofs, {token_in_id: program}) sender_proof = proofs[0] if proofs else default_merkle_proofs() - inputs = [ - record, identity.blinding_factor, identity.blinded_address, - pool_key, resolved.zero_for_one, - f"{amount_in}u128", f"{resolved.amount_out_min}u128", - int_to_u256_plaintext(resolved.sqrt_price_limit), - f"{swap_nonce}u64", f"{deadline}u32", - pool.token0, pool.token1, - ] - if route.program != self.program: - if resolved.amount_out_min <= 0: - raise ValueError( - "routed swaps require amount_out_min > 0 — pass " - "expected_out or a slippage below 100%") - inputs = [record, sender_proof, *inputs[1:]] - if multi_hop: - inputs = multi_hop_inputs(quote, hop_literals, record, identity, amount_in, - resolved.amount_out_min, swap_nonce, deadline, - sender_proof, route.program != self.program) + if route.program != self.program and resolved.amount_out_min <= 0: + raise ValueError( + "routed swaps require amount_out_min > 0 — pass " + "expected_out or a slippage below 100%") prog = await self._program(route.program) - bound = getattr(prog.functions, route.function)(*inputs) + + def assemble(spend: str) -> Any: + """Bind the call for one input record; every other term is fixed above.""" + inputs = [ + spend, identity.blinding_factor, identity.blinded_address, + pool_key, resolved.zero_for_one, + f"{amount_in}u128", f"{resolved.amount_out_min}u128", + int_to_u256_plaintext(resolved.sqrt_price_limit), + f"{swap_nonce}u64", f"{deadline}u32", + pool.token0, pool.token1, + ] + if route.program != self.program: + inputs = [spend, sender_proof, *inputs[1:]] + if multi_hop: + inputs = multi_hop_inputs(quote, hop_literals, spend, identity, amount_in, + resolved.amount_out_min, swap_nonce, deadline, + sender_proof, route.program != self.program) + return getattr(prog.functions, route.function)(*inputs) + + bound = assemble(record) + reselect = None + if token_record is None and self.inventory.enabled: + reselect = RecordReselection(candidates, assemble, self._swap_context_guard(acct, account is None)) def build_result(tx_id: str, outputs: list[Any]) -> SwapHandle: """Assemble the swap handle, carrying the blinding secret forward. @@ -846,7 +904,33 @@ def build_result(tx_id: str, outputs: list[Any]) -> SwapHandle: transaction_id=tx_id, program=self.program, pool_keys=tuple(h.pool_key for h in quote.hops) if multi_hop else ()) - return AsyncDexCall(self._aleo, bound, build_result) + return AsyncDexCall(self._aleo, bound, build_result, reselect=reselect) + + def _swap_context_guard(self, acct: Any, from_default: bool) -> Callable[[], None]: + """A check that the signer, network, and journal are the ones a swap was prepared for.""" + return swap_context_guard(self._aleo, acct, (lambda: self._account(None)) if from_default else None) + + # ── Record inventory ───────────────────────────────────────────────────── + + async def _underlying_program(self, token: str) -> str: + return underlying_program_for(await self.api.get_tokens(), token) + + async def plan_inventory(self, token: str, target: InventoryTarget, *, + max_transactions: Optional[int] = None, account: Any = None) -> InventoryPlan: + """Plan joins and splits for *token*'s underlying records (awaited). See :meth:`ShieldSwap.plan_inventory`.""" + program = await self._underlying_program(token) + return await self.inventory.plan_record_inventory( + RecordAsset(program), target, max_transactions=max_transactions, account=account) + + async def rebalance_inventory(self, token: str, target: InventoryTarget, *, + max_transactions: Optional[int] = None, timeout: float = 120.0, + poll_interval: float = 1.0, max_fee_microcredits: Optional[int] = None, + cancel: Any = None, account: Any = None) -> InventoryResult: + """Reshape *token*'s underlying records to *target* (awaited). See :meth:`ShieldSwap.rebalance_inventory`.""" + plan = await self.plan_inventory(token, target, max_transactions=max_transactions, account=account) + return await self.inventory.rebalance_record_inventory( + plan, timeout=timeout, poll_interval=poll_interval, + max_fee_microcredits=max_fee_microcredits, cancel=cancel, account=account) async def claim_swap_output(self, handle: SwapHandle, *, timeout: float = 0.0, diff --git a/shield-swap-sdk/python/aleo_shield_swap/client.py b/shield-swap-sdk/python/aleo_shield_swap/client.py index 18ed276b..09acbd15 100644 --- a/shield-swap-sdk/python/aleo_shield_swap/client.py +++ b/shield-swap-sdk/python/aleo_shield_swap/client.py @@ -13,13 +13,15 @@ import math from decimal import Decimal import time -from typing import Any, Optional +from typing import Any, Callable, Optional from aleo import AleoNetworkError, ProgramNotFound from aleo.codegen.runtime import parse_plaintext from aleo.compliance import FreezeListProofs, ComplianceProofContext, freezelist_program +from aleo.inventory import InventoryPlan, InventoryResult, InventoryTarget, RecordAsset, RecordInventory from . import _generated as g +from ._calls import RecordReselection from ._core import ( _amount_to_base_units, decode_position_record, @@ -36,8 +38,12 @@ parse_token_record_info, parse_unsigned_literal, read_mapping_value, + reserved_nonces, resolve_swap_params, select_token_record, + select_token_records, + swap_context_guard, + underlying_program_for, unreadable_airdrop_transactions, ) from .api import ApiClient, ConfirmAirdropResult, api_url_for @@ -144,6 +150,12 @@ def __init__(self, aleo: Any, *, program: str = g.PROGRAM_ID, self.amm_freezelist_program: str = SHIELD_SWAP_FREEZELIST #: Freeze-list exclusion proofs, cached per list until its root changes. self.freezelist = FreezeListProofs(aleo) + #: Private record inventory of the bound client. Build the facade with + #: ``record_management=RecordManagementConfig(enabled=True, ...)`` so + #: trades and inventory maintenance share one reservation journal. + self.inventory: RecordInventory = ( + aleo.inventory if hasattr(aleo, "inventory") else RecordInventory(aleo) + ) def __repr__(self) -> str: return f"ShieldSwap(program={self.program!r}, api={self.api.base_url!r})" @@ -699,7 +711,8 @@ def has_swap_balance(self, token_in_id: str, amount_in: int | str | Decimal, amount = _amount_to_base_units(amount_in, token_in_id, tokens) program = self._token_program(token_in_id) records = provider.find(acct, program=program, unspent=True) - return pick_covering_record(records, min_amount=amount, token_id=token_in_id) is not None + return pick_covering_record(records, min_amount=amount, token_id=token_in_id, + exclude_nonces=reserved_nonces(self._aleo, acct)) is not None def get_private_balances(self, programs: list[str], account: Any = None) -> dict[str, int]: @@ -941,15 +954,19 @@ def swap( # Resolve the record-funding program lazily: an explicit record # needs no registry lookup (its program registration comes from - # token_in_program= or imports=). + # token_in_program= or imports=). Automatic selection keeps every + # eligible candidate from one scan so a reservation conflict at + # submission can move to the next record without rescanning. program = token_in_program record = token_record + candidates: list[str] = [] if record is None: program = program or self._token_program(token_in_id) - record = select_token_record( + candidates = select_token_records( self._aleo, program=program, min_amount=amount_in, token_id=token_in_id, account=acct, ) + record = candidates[0] deadline = get_deadline(self._aleo, deadline_offset_blocks) swap_nonce = nonce if nonce is not None else generate_swap_nonce() @@ -986,32 +1003,41 @@ def swap( compliance_proofs, wrapper_proofs, {token_in_id: program}) sender_proof = proofs[0] if proofs else default_merkle_proofs() - # Core input order per the contract's swap entrypoint: - # record, blinding slots, then pool/direction/amounts/bounds/timing/tokens. - inputs = [ - record, - identity.blinding_factor, - identity.blinded_address, - pool_key, - resolved.zero_for_one, - f"{amount_in}u128", - f"{resolved.amount_out_min}u128", - int_to_u256_plaintext(resolved.sqrt_price_limit), - f"{swap_nonce}u64", - f"{deadline}u32", - pool.token0, - pool.token1, - ] - if route.program != self.program: - # Router deposits the underlying, swaps, and burns the empty - # wrapper change record in one transaction. - inputs = [record, sender_proof, *inputs[1:]] - if multi_hop: - inputs = multi_hop_inputs(quote, hop_literals, record, identity, amount_in, - resolved.amount_out_min, swap_nonce, deadline, - sender_proof, route.program != self.program) - bound = getattr(self._program(route.program).functions, - route.function)(*inputs) + def assemble(spend: str) -> Any: + """Bind the call for one input record; every other term is fixed above.""" + # Core input order per the contract's swap entrypoint: + # record, blinding slots, then pool/direction/amounts/bounds/timing/tokens. + inputs = [ + spend, + identity.blinding_factor, + identity.blinded_address, + pool_key, + resolved.zero_for_one, + f"{amount_in}u128", + f"{resolved.amount_out_min}u128", + int_to_u256_plaintext(resolved.sqrt_price_limit), + f"{swap_nonce}u64", + f"{deadline}u32", + pool.token0, + pool.token1, + ] + if route.program != self.program: + # Router deposits the underlying, swaps, and burns the empty + # wrapper change record in one transaction. + inputs = [spend, sender_proof, *inputs[1:]] + if multi_hop: + inputs = multi_hop_inputs(quote, hop_literals, spend, identity, amount_in, + resolved.amount_out_min, swap_nonce, deadline, + sender_proof, route.program != self.program) + return getattr(self._program(route.program).functions, route.function)(*inputs) + + bound = assemble(record) + # With record management enabled, a conflict on the chosen record at + # submission moves to the next candidate from the same scan. An + # explicit token_record is never replaced. + reselect = None + if token_record is None and self.inventory.enabled: + reselect = RecordReselection(candidates, assemble, self._swap_context_guard(acct, account is None)) def build_result(tx_id: str, outputs: list[Any]) -> SwapHandle: """Assemble the swap handle, carrying the blinding secret forward. @@ -1045,7 +1071,77 @@ def build_result(tx_id: str, outputs: list[Any]) -> SwapHandle: self.journal.record_swap(handle, counter) return handle - return DexCall(self._aleo, bound, build_result, build_before_wait=True) + return DexCall(self._aleo, bound, build_result, build_before_wait=True, reselect=reselect) + + def _swap_context_guard(self, acct: Any, from_default: bool) -> Callable[[], None]: + """A check that the signer, network, and journal are the ones a swap was prepared for. + + Run before a replacement record is bound, so a retry never carries a + prepared identity or quote into a different account, chain, or + coordination store. + """ + return swap_context_guard(self._aleo, acct, (lambda: self._account(None)) if from_default else None) + + # ── Record inventory ───────────────────────────────────────────────────── + + def _underlying_program(self, token: str) -> str: + """The program whose records fund trades of *token* (symbol or on-chain id).""" + return underlying_program_for(self.api.get_tokens(), token) + + def plan_inventory(self, token: str, target: InventoryTarget, *, + max_transactions: Optional[int] = None, account: Any = None) -> InventoryPlan: + """Plan the joins and splits that give *token* the record layout trades need. + + One trade spends one record, so several trades sold at the same time + need several records that each cover a trade. The plan targets the + token's underlying record program (the records a trader holds), not + the AMM wrapper. Reads the token registry, the program, and the + scanner; signs and reserves nothing. Review ``plan.steps`` and + ``plan.deduction`` before executing with :meth:`rebalance_inventory`. + + Args: + token: Symbol such as ``"USDCx"`` or the on-chain token id. + target: Desired count, minimum amount in base units, and distribution. + max_transactions: Operation ceiling, 1 through 10000. Defaults to 100. + account: Signer; defaults to ``aleo.default_account``. + + Raises: + ShieldSwapError: The token is unknown or has no underlying program. + InventoryPlanError: The target cannot be reached from the available records. + + Example:: + + plan = dex.plan_inventory("USDCx", InventoryTarget(records=6, min_record_amount=100_000, distribution="balanced")) + """ + program = self._underlying_program(token) + return self.inventory.plan_record_inventory( + RecordAsset(program), target, max_transactions=max_transactions, account=account, + ) + + def rebalance_inventory(self, token: str, target: InventoryTarget, *, + max_transactions: Optional[int] = None, timeout: float = 120.0, + poll_interval: float = 1.0, max_fee_microcredits: Optional[int] = None, + cancel: Any = None, account: Any = None) -> InventoryResult: + """Reshape *token*'s underlying records to *target*, one confirmed transaction at a time. + + Plans as :meth:`plan_inventory` does, then executes through the + facade's record inventory: each step reserves its inputs, proves, + broadcasts, and waits for the scanner to show its outputs before the + next step. Requires the facade to be built with record management + enabled. No accepted step is rolled back; an interrupted result names + the reason and every submitted transaction. + + Example:: + + result = dex.rebalance_inventory("USDCx", InventoryTarget(records=6), max_fee_microcredits=5_000_000) + if result.status == "interrupted": + dex.inventory.reconcile_record_inventory() + """ + plan = self.plan_inventory(token, target, max_transactions=max_transactions, account=account) + return self.inventory.rebalance_record_inventory( + plan, timeout=timeout, poll_interval=poll_interval, + max_fee_microcredits=max_fee_microcredits, cancel=cancel, account=account, + ) def claim_swap_output( self, @@ -1378,7 +1474,8 @@ def _fresh_record() -> Optional[str]: time.sleep(3.0 * (attempt + 1)) return pick_covering_record(records, min_amount=amount_in, token_id=token_in_id, - exclude=used_records) + exclude=used_records, + exclude_nonces=reserved_nonces(self._aleo, acct)) handles: list[SwapHandle] = [] failures: list[dict] = [] diff --git a/shield-swap-sdk/tests/test_concurrent_swaps_example.py b/shield-swap-sdk/tests/test_concurrent_swaps_example.py new file mode 100644 index 00000000..051e8a46 --- /dev/null +++ b/shield-swap-sdk/tests/test_concurrent_swaps_example.py @@ -0,0 +1,121 @@ +"""Offline checks of the concurrent-swaps example: batch orchestration and the file store. + +_source: ProvableHQ/veil examples/shield-swap/concurrent-swaps.test.ts, PR #172 @ 2e324d5. +Network, proving, and the DEX are replaced; the example's own code runs. +""" +from __future__ import annotations + +import importlib.util +import sys +import threading +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +from aleo.inventory import RecordSpend +from aleo_shield_swap.types import ClaimResult, SwapHandle + +EXAMPLE = Path(__file__).parents[1] / "examples" / "concurrent-swaps" / "concurrent_swaps.py" + + +def load_example() -> ModuleType: + spec = importlib.util.spec_from_file_location("concurrent_swaps_example", EXAMPLE) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module # dataclasses resolve annotations through sys.modules + spec.loader.exec_module(module) + return module + + +class _Call: + def __init__(self, result: Any) -> None: + self._result = result + + def delegate(self, **kw: Any) -> Any: + if isinstance(self._result, BaseException): + raise self._result + return self._result + + +class FakeDex: + """Swaps succeed or fail per index; claims follow a second table.""" + + def __init__(self, swap_errors: dict[int, BaseException] | None = None, claim_errors: dict[int, BaseException] | None = None) -> None: + self.swap_errors = swap_errors or {} + self.claim_errors = claim_errors or {} + self.swaps = 0 + self.claimed: list[str] = [] + self._lock = threading.Lock() + + def swap(self, quote: Any) -> _Call: + with self._lock: + self.swaps += 1 + index = self.swaps + if index in self.swap_errors: + return _Call(self.swap_errors[index]) + return _Call(SwapHandle(swap_id=f"{index}field", blinding_factor="1field", blinded_address="aleo1x", + token_in_id="1field", token_out_id="2field", pool_key="5field", amount_in=10, + transaction_id=f"at1swap{index}", program="shield_swap.aleo")) + + def claim_swap_output(self, handle: SwapHandle, *, timeout: float = 0.0) -> _Call: + index = int(handle.transaction_id.removeprefix("at1swap")) + self.claimed.append(handle.transaction_id) + if index in self.claim_errors: + return _Call(self.claim_errors[index]) + return _Call(ClaimResult(transaction_id=f"at1claim{index}", amount_out=5, amount_remaining=0)) + + +def test_batch_reports_every_trade_and_keeps_handles_of_failed_claims() -> None: + example = load_example() + dex = FakeDex(swap_errors={2: RuntimeError("no record")}, claim_errors={3: TimeoutError("output not finalized")}) + outcomes = example.submit_concurrent_swaps(dex, quote=object(), count=4, claim_timeout=1.0) + # The pool runs trades concurrently, so which batch slot hits which fake + # failure is not fixed; the shape of the report is. + assert [o.index for o in outcomes] == [1, 2, 3, 4] + swap_failed = [o for o in outcomes if o.handle is None] + claim_failed = [o for o in outcomes if o.handle is not None and o.claim is None] + claimed = [o for o in outcomes if o.claim is not None] + assert len(swap_failed) == 1 and isinstance(swap_failed[0].error, RuntimeError) + assert len(claim_failed) == 1 and isinstance(claim_failed[0].error, TimeoutError) + assert claim_failed[0].handle is not None and claim_failed[0].handle.transaction_id == "at1swap3" + assert len(claimed) == 2 and all(o.claim is not None and o.claim.amount_out == 5 for o in claimed) + assert sorted(dex.claimed) == ["at1swap1", "at1swap3", "at1swap4"] # the failed claim was attempted, not skipped + + +def test_json_file_store_reserves_atomically_and_survives_reopen(tmp_path: Path) -> None: + example = load_example() + path = tmp_path / "inventory.json" + store = example.JsonFileRecordInventoryStore(path) + scope = '["aleo:testnet","aleo1owner"]' + + def entry(*records: str) -> RecordSpend: + return RecordSpend(id="", scope=scope, records=records, program="credits.aleo", function="join", + status="reserved", created_at=1) + + first = store.acquire(entry("1group", "2group")) + assert first is not None and first.id == "1" + assert store.acquire(entry("2group", "3group")) is None + store.update(scope, first.id, {"status": "prepared", "transaction_id": "at1x", "transaction": {"id": "at1x"}, "fee_microcredits": "100"}) + reopened = example.JsonFileRecordInventoryStore(path) + saved = reopened.list(scope)[0] + assert (saved.status, saved.transaction_id, saved.transaction, saved.fee_microcredits) == ("prepared", "at1x", {"id": "at1x"}, "100") + reopened.update(scope, first.id, {"status": "confirmed", "transaction": None}) + assert reopened.list(scope)[0].transaction is None + with pytest.raises(KeyError): + reopened.update(scope, "99", {"status": "cancelled"}) + + results: list[RecordSpend | None] = [] + barrier = threading.Barrier(6) + + def race() -> None: + barrier.wait() + results.append(example.JsonFileRecordInventoryStore(path).acquire(entry("9group"))) + + threads = [threading.Thread(target=race) for _ in range(6)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert sum(result is not None for result in results) == 1 diff --git a/shield-swap-sdk/tests/test_gen_context.py b/shield-swap-sdk/tests/test_gen_context.py index 21e7ee37..16caf56e 100644 --- a/shield-swap-sdk/tests/test_gen_context.py +++ b/shield-swap-sdk/tests/test_gen_context.py @@ -64,4 +64,6 @@ def test_page_stays_compact(): # graph, and the indexer's unclaimed/protocol-state views. 32k → 34k # (2026-10) for compliance proofs: six write methods gained a # ``compliance_proofs`` parameter and a one-line note on what it takes. - assert len(_render()) < 34_000 + # 34k → 36k (2026-10-23) for record inventory: ``plan_inventory`` and + # ``rebalance_inventory`` plus the reservation note on ``swap``. + assert len(_render()) < 36_000 diff --git a/shield-swap-sdk/tests/test_inventory.py b/shield-swap-sdk/tests/test_inventory.py new file mode 100644 index 00000000..5edbcf07 --- /dev/null +++ b/shield-swap-sdk/tests/test_inventory.py @@ -0,0 +1,437 @@ +"""Record inventory through the DEX client: reserved exclusion, swap reselection, inventory verbs. + +_source: ProvableHQ/veil packages/shield-swap/test/actions/swap/concurrentSwaps.test.ts +and test/actions/inventory.test.ts, PR #172 @ 2e324d5. Selection, input +assembly, acquisition, and lifecycle handling run for real; the stub replaces +chain, scanner, and prover I/O and performs the facade's managed-write hook. +""" +from __future__ import annotations + +import json +import threading +from typing import Any, Callable + +import pytest + +from aleo.inventory import ( + InventoryTarget, MemoryRecordInventoryStore, PreparedTransaction, RecordInventory, RecordManagementConfig, + RecordReservationConflictError, RecordSpend, +) +from aleo_shield_swap import ShieldSwap +from aleo_shield_swap._api_models import TokenDoc +from aleo_shield_swap._core import record_nonce, select_token_record, select_token_records +from aleo_shield_swap.errors import ShieldSwapError + +from .conftest import POOL_TEXT, SIGNER, SLOT_TEXT, StubAccount, StubAleo + +TOKEN_PROGRAM = "token_a.aleo" +SCOPE = f'["testnet","{SIGNER}"]' +# A token whose declarations satisfy the inventory's join/split interface check. +UNDERLYING_SOURCE = """program test_token_a.aleo; +record Token: + owner as address.private; + amount as u128.private; +function transfer_public: + input r0 as address.public; + input r1 as u128.public; +function transfer_private: + input r0 as Token.record; + input r1 as address.private; + input r2 as u128.private; + output r3 as Token.record; + output r4 as Token.record; +function join: + input r0 as Token.record; + input r1 as Token.record; + output r2 as Token.record; +function split: + input r0 as Token.record; + input r1 as u128.private; + output r2 as Token.record; + output r3 as Token.record; +""" + + +def plaintext(nonce: int, amount: int = 100) -> str: + return f"{{ owner: {SIGNER}.private, amount: {amount}u128.private, _nonce: {nonce}group.public }}" + + +def records(count: int) -> list[dict[str, Any]]: + return [{"program_name": TOKEN_PROGRAM, "record_name": "Token", "spent": False, "tag": f"{n}field", + "commitment": f"{n}field", "record_plaintext": plaintext(n)} for n in range(1, count + 1)] + + +class BarrierStore: + """Delays the first *racers* acquisitions until all have arrived, so every writer picks the same record first.""" + + def __init__(self, racers: int) -> None: + self.backing = MemoryRecordInventoryStore() + self.attempts: list[tuple[str, ...]] = [] + self.on_acquire: Callable[[], Any] | None = None + self._barrier = threading.Barrier(racers) if racers > 1 else None + self._lock = threading.Lock() + + def list(self, scope: str) -> list[RecordSpend]: + return self.backing.list(scope) + + def acquire(self, entry: RecordSpend) -> RecordSpend | None: + with self._lock: + self.attempts.append(tuple(entry.records)) + first = len(self.attempts) <= (self._barrier.parties if self._barrier else 1) + if first and self._barrier is not None: + self._barrier.wait() + if self.on_acquire is not None: + self.on_acquire() + return self.backing.acquire(entry) + + def update(self, scope: str, entry_id: str, patch: Any) -> None: + self.backing.update(scope, entry_id, patch) + + +class _ManagedBound: + """A prepared call whose terminal verbs run the facade's managed-write contract.""" + + def __init__(self, world: Any, program: str, function: str, args: tuple[Any, ...]) -> None: + self._world, self.program_id, self.function_name = world, program, function + self.args = [str(a) for a in args] + world.last_call = (function, list(self.args)) + world.last_program = program + + def simulate(self, account: Any = None) -> str: + return "simulated" + + def _prove(self) -> dict[str, Any]: + self._world.proved.append({"program": self.program_id, "function": self.function_name, "inputs": list(self.args)}) + if self._world.prove_error is not None: + raise self._world.prove_error + tx_id = f"at1tx{len(self._world.proved)}" + return {"id": tx_id, "type": "execute", "execution": {"transitions": [ + {"program": self.program_id, "function": self.function_name, "outputs": [{"value": "77field"}]}]}, + "fee": {"transition": {"program": "credits.aleo", "function": "fee_public", + "inputs": [{"type": "public", "value": "100u64"}, {"type": "public", "value": "0u64"}]}}} + + def delegate(self, account: Any = None, *, broadcast: bool = True, **kw: Any) -> dict[str, Any]: + if not broadcast or not self._world.record_management.enabled: + return {"transaction": self._prove()} + payloads: list[dict[str, Any]] = [] + + def build(request: Any) -> PreparedTransaction: + tx = self._prove() + payloads.append({"transaction": tx}) + return PreparedTransaction(tx["id"], tx) + + self._world.inventory.managed_write(self.program_id, self.function_name, self.args, account=account, builder=build) + return payloads[0] + + def transact_with_result(self, account: Any = None, **kw: Any) -> tuple[str, Any]: + built: list[Any] = [] + + def build(request: Any) -> PreparedTransaction: + result = self.build_transaction(account) + built.append(result) + return PreparedTransaction(result.id, json.loads(result.raw.to_json())) + + tx_id = self._world.inventory.managed_write(self.program_id, self.function_name, self.args, account=account, builder=build) + return tx_id, built[0] + + def build_transaction(self, account: Any = None, **kw: Any) -> Any: + tx = self._prove() + outer = self + + class _Result: + id = tx["id"] + + class raw: + @staticmethod + def to_json() -> str: + return json.dumps(tx) + + def decoded(self) -> list[dict[str, Any]]: + return [{"program": outer.program_id, "function": outer.function_name, "outputs": [{"value": "77field"}]}] + return _Result() + + +class _ManagedFunctions: + def __init__(self, world: Any, program: str) -> None: + self._world, self._program = world, program + + def __getattr__(self, name: str) -> Callable[..., _ManagedBound]: + return lambda *args: _ManagedBound(self._world, self._program, name, args) + + __getitem__ = __getattr__ + + +class _ManagedPrograms: + def __init__(self, world: Any, inner: Any) -> None: + self._world, self._inner = world, inner + + def get(self, pid: str) -> Any: + program = self._inner.get(pid) + if pid in self._world.sources: + program.source = self._world.sources[pid] + program.functions = _ManagedFunctions(self._world, pid) + return program + + +class ManagedStubAleo(StubAleo): + """The conftest stub plus the facade's record management surface.""" + + def __init__(self, record_list: list[dict[str, Any]], *, store: Any = None, enabled: bool = True) -> None: + super().__init__(mappings={"pools": {"5field": POOL_TEXT}, "slots": {"5field": SLOT_TEXT}, + "swap_outputs": {}, "used_blinded_addresses": {}, "from_wrapper_token_id": {}}, + records=record_list) + self.store = store if store is not None else BarrierStore(1) + self.record_management = RecordManagementConfig(enabled=enabled, store=self.store) + self.inventory = RecordInventory(self, self.record_management) + self.programs = _ManagedPrograms(self, self.programs) + self.sources: dict[str, str] = {} + self.proved: list[dict[str, Any]] = [] + self.prove_error: Exception | None = None + self.submitted_json: list[dict[str, Any]] = [] + original = self.network.submit_transaction + self.network.submit_transaction = lambda raw: (self.submitted_json.append(json.loads(raw)) or json.loads(raw)["id"]) if isinstance(raw, str) else original(raw) # type: ignore[method-assign] + + +def dex_for(world: ManagedStubAleo) -> ShieldSwap: + dex = ShieldSwap(world) + dex._token_program = lambda tid: TOKEN_PROGRAM # type: ignore[method-assign] + return dex + + +def swap_call(dex: ShieldSwap, index: int = 1, **kw: Any) -> Any: + return dex.swap(pool_key="5field", token_in_id="1field", amount_in=10, expected_out=10, nonce=index, + token_in_program=TOKEN_PROGRAM, track=False, **kw) + + +def proved_nonces(world: Any) -> list[str]: + return [record_nonce(next(i for i in call["inputs"] if "_nonce" in i)) or "" for call in world.proved] + + +def hold(store: BarrierStore, *nonces: str) -> None: + assert store.backing.acquire(RecordSpend(id="", scope=SCOPE, records=nonces, program=TOKEN_PROGRAM, function="join", + status="reserved", created_at=1)) is not None + + +# ── Selection skips reserved records ────────────────────────────────────────── + +def test_selection_skips_records_the_journal_holds() -> None: + world = ManagedStubAleo(records(2)) + hold(world.store, "1group") + assert record_nonce(select_token_record(world, program=TOKEN_PROGRAM, min_amount=10)) == "2group" + assert [record_nonce(p) for p in select_token_records(world, program=TOKEN_PROGRAM, min_amount=10)] == ["2group"] + dex = dex_for(world) + assert dex.has_swap_balance("1field", 10) is True + hold(world.store, "2group") + assert dex.has_swap_balance("1field", 10) is False + + +# ── Concurrent swaps reselect ───────────────────────────────────────────────── + +def test_reselects_six_distinct_inputs_when_all_first_acquisitions_race() -> None: + world = ManagedStubAleo(records(6), store=BarrierStore(6)) + dex = dex_for(world) + results: list[Any] = [] + lock = threading.Lock() + + def run(index: int) -> None: + try: + handle = swap_call(dex, index).delegate(wait=False) + except BaseException as exc: # noqa: BLE001 + handle = exc + with lock: + results.append(handle) + + threads = [threading.Thread(target=run, args=(i,)) for i in range(1, 7)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert all(not isinstance(r, BaseException) for r in results), results + assert world.store.attempts[:6] == [("1group",)] * 6 + assert sorted(proved_nonces(world)) == [f"{n}group" for n in range(1, 7)] + assert sorted(entry.records[0] for entry in world.store.list(SCOPE)) == [f"{n}group" for n in range(1, 7)] + assert all(entry.status == "submitted" for entry in world.store.list(SCOPE)) + assert all(call["inputs"][0].startswith("{") and "10u128" in call["inputs"] for call in world.proved) + + +def test_stops_after_its_finite_candidates_are_reserved_even_when_the_scanner_stays_stale() -> None: + world = ManagedStubAleo(records(2)) + dex = dex_for(world) + call = swap_call(dex) + world.store.on_acquire = lambda: (hold(world.store, "1group") if not any("1group" in a for a in world.store.attempts[:-1]) else None, + hold(world.store, "2group") if len(world.store.attempts) == 1 else None) + with pytest.raises(RecordReservationConflictError): + call.delegate(wait=False) + assert world.store.attempts == [("1group",), ("2group",)] + assert world.proved == [] + + +def test_keeps_explicitly_supplied_records_fixed_on_contention() -> None: + world = ManagedStubAleo(records(2)) + dex = dex_for(world) + hold(world.store, "1group") + with pytest.raises(RecordReservationConflictError, match="reserved or spent"): + swap_call(dex, token_record=plaintext(1)).delegate(wait=False) + assert world.store.attempts == [("1group",)] + assert world.proved == [] + + +def test_does_not_retry_execution_errors() -> None: + world = ManagedStubAleo(records(2)) + dex = dex_for(world) + world.prove_error = RuntimeError("Execution failed") + with pytest.raises(RuntimeError, match="Execution failed"): + swap_call(dex).delegate(wait=False) + assert proved_nonces(world) == ["1group"] + assert world.store.attempts == [("1group",)] + assert world.store.list(SCOPE)[0].status == "cancelled" + + +def test_refuses_to_retry_after_the_account_changes() -> None: + world = ManagedStubAleo(records(2)) + dex = dex_for(world) + call = swap_call(dex) + + def swap_account() -> None: + hold(world.store, "1group") + other = StubAccount() + other.address = "aleo1other" + world.default_account = other + world.store.on_acquire = None + + world.store.on_acquire = swap_account + with pytest.raises(ShieldSwapError, match="changed"): + call.delegate(wait=False) + assert world.proved == [] + + +def test_preserves_accepted_terms_when_the_first_record_is_taken() -> None: + world = ManagedStubAleo(records(3)) + dex = dex_for(world) + call = swap_call(dex, 7) + world.store.on_acquire = lambda: (hold(world.store, "1group"), setattr(world.store, "on_acquire", None)) + handle = call.delegate(wait=False) + assert handle.amount_in == 10 + assert proved_nonces(world) == ["2group"] + assert world.store.attempts == [("1group",), ("2group",)] + + +def test_retains_legacy_uncoordinated_selection_when_management_is_disabled() -> None: + world = ManagedStubAleo(records(2), enabled=False) + dex = dex_for(world) + swap_call(dex, 1).delegate(wait=False) + swap_call(dex, 2).delegate(wait=False) + assert proved_nonces(world) == ["1group", "1group"] + assert world.store.attempts == [] + + +def test_managed_transact_reserves_and_harvests_outputs() -> None: + world = ManagedStubAleo(records(2)) + dex = dex_for(world) + handle = swap_call(dex).transact() + assert handle.swap_id == "77field" and handle.transaction_id == "at1tx1" + assert [tx["id"] for tx in world.submitted_json] == ["at1tx1"] + assert (world.store.list(SCOPE)[0].status, world.store.list(SCOPE)[0].records) == ("submitted", ("1group",)) + + +# ── Inventory verbs ─────────────────────────────────────────────────────────── + +class _Api: + def __init__(self, tokens: list[TokenDoc]) -> None: + self._tokens = tokens + + def get_tokens(self) -> list[TokenDoc]: + return self._tokens + + +def test_plan_inventory_targets_the_underlying_program() -> None: + underlying = "test_token_a.aleo" + owned = [{"program_name": underlying, "record_name": "Token", "spent": False, "tag": "1field", "commitment": "1field", + "record_plaintext": f"{{ owner: {SIGNER}.private, amount: 1000u128.private, _nonce: 1group.public }}"}] + world = ManagedStubAleo(owned) + world.sources[underlying] = UNDERLYING_SOURCE + dex = dex_for(world) + dex.api = _Api([TokenDoc(address="1field", decimals=6, id="t", name="Asset", symbol="ASSET", # type: ignore[assignment] + amm_token_program="wrapper.aleo", underlying_program=underlying)]) + plan = dex.plan_inventory("asset", InventoryTarget(records=4, distribution="balanced")) + assert plan.asset.program == underlying and len(plan.steps) == 3 + assert dex.plan_inventory("1field", InventoryTarget(records=1)).steps == () + with pytest.raises(ShieldSwapError, match="Unknown token"): + dex.plan_inventory("nope", InventoryTarget(records=1)) + dex.api = _Api([TokenDoc(address="2field", decimals=6, id="t", name="Wrapped", symbol="WRAP")]) # type: ignore[assignment] + with pytest.raises(ShieldSwapError, match="no underlying"): + dex.plan_inventory("WRAP", InventoryTarget(records=1)) + + +# ── Async parity ────────────────────────────────────────────────────────────── + +from aleo.inventory import AsyncRecordInventory # noqa: E402 +from aleo_shield_swap import AsyncShieldSwap # noqa: E402 + +from .test_async_client import AsyncStubAleo # noqa: E402 + + +class _AsyncManagedBound(_ManagedBound): + async def delegate(self, account: Any = None, *, broadcast: bool = True, **kw: Any) -> dict[str, Any]: # type: ignore[override] + if not broadcast or not self._world.record_management.enabled: + return {"transaction": self._prove()} + payloads: list[dict[str, Any]] = [] + + def build(request: Any) -> PreparedTransaction: + tx = self._prove() + payloads.append({"transaction": tx}) + return PreparedTransaction(tx["id"], tx) + + await self._world.inventory.managed_write(self.program_id, self.function_name, self.args, account=account, builder=build) + return payloads[0] + + +class _AsyncManagedFunctions(_ManagedFunctions): + def __getattr__(self, name: str) -> Callable[..., _AsyncManagedBound]: + return lambda *args: _AsyncManagedBound(self._world, self._program, name, args) # type: ignore[arg-type] + + __getitem__ = __getattr__ + + +class _AsyncManagedPrograms: + def __init__(self, world: Any, inner: Any) -> None: + self._world, self._inner = world, inner + + async def get(self, pid: str) -> Any: + program = await self._inner.get(pid) + program.functions = _AsyncManagedFunctions(self._world, pid) + return program + + +class AsyncManagedStubAleo(AsyncStubAleo): + def __init__(self, record_list: list[dict[str, Any]]) -> None: + super().__init__(mappings={"pools": {"5field": POOL_TEXT}, "slots": {"5field": SLOT_TEXT}, + "swap_outputs": {}, "used_blinded_addresses": {}, "from_wrapper_token_id": {}}, + records=record_list) + self.store = BarrierStore(1) + self.record_management = RecordManagementConfig(enabled=True, store=self.store) + self.inventory = AsyncRecordInventory(self, self.record_management) + self.programs = _AsyncManagedPrograms(self, self.programs) + self.proved: list[dict[str, Any]] = [] + self.prove_error: Exception | None = None + + +async def test_async_swap_reselects_after_a_conflict_and_keeps_explicit_records_fixed() -> None: + world = AsyncManagedStubAleo(records(3)) + dex = AsyncShieldSwap(world) + dex._token_program = _async_token_program # type: ignore[method-assign] + call = await dex.swap(pool_key="5field", token_in_id="1field", amount_in=10, expected_out=10, nonce=1, + token_in_program=TOKEN_PROGRAM) + world.store.on_acquire = lambda: (hold(world.store, "1group"), setattr(world.store, "on_acquire", None)) + handle = await call.delegate() + assert handle.amount_in == 10 and proved_nonces(world) == ["2group"] + assert world.store.attempts == [("1group",), ("2group",)] + explicit = await dex.swap(pool_key="5field", token_in_id="1field", amount_in=10, expected_out=10, nonce=2, + token_in_program=TOKEN_PROGRAM, token_record=plaintext(1)) + with pytest.raises(RecordReservationConflictError): + await explicit.delegate() + assert proved_nonces(world) == ["2group"] + + +async def _async_token_program(tid: str) -> str: + return TOKEN_PROGRAM From 7c54200e411b37a33ca4e5e0a1050fc2d8d43eb0 Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 09:14:07 -0400 Subject: [PATCH 02/11] test: live record inventory suite on testnet Port of Veil's recordInventory, recordInventoryStrategy, and concurrentInventory e2e suites for the Shield Swap SDK. Keeps six equal USDCx and ETH records in the e2e account, trades an explicit round trip and repairs the inventory through the stablecoin join helper, submits six swaps at once in each direction with automatic record selection under a forced first-selection collision, claims every output, and reopens the journal to verify confirmed checkpoints and distinct inputs. Passed on testnet in 7m31s (28 transactions). Journals live under .shield-swap/, now ignored. --- shield-swap-sdk/.gitignore | 3 + shield-swap-sdk/README.md | 13 + .../integration/test_record_inventory_live.py | 480 ++++++++++++++++++ 3 files changed, 496 insertions(+) create mode 100644 shield-swap-sdk/tests/integration/test_record_inventory_live.py diff --git a/shield-swap-sdk/.gitignore b/shield-swap-sdk/.gitignore index 7a60b85e..32bc71d0 100644 --- a/shield-swap-sdk/.gitignore +++ b/shield-swap-sdk/.gitignore @@ -1,2 +1,5 @@ __pycache__/ *.pyc + +# Live inventory suite and example journals (reservations, swap handles) +.shield-swap/ diff --git a/shield-swap-sdk/README.md b/shield-swap-sdk/README.md index ea4cf2dc..7dffcf22 100644 --- a/shield-swap-sdk/README.md +++ b/shield-swap-sdk/README.md @@ -283,6 +283,19 @@ executions and is the fast path (~5 minutes). Without it, executions are fully proven locally: expect SNARK parameter downloads and key synthesis on first use. +The record inventory suite (`tests/integration/test_record_inventory_live.py`, +`-m "live and slow"`) keeps six equal USDCx and ETH records in the +`ALEO_E2E_PRIVATE_KEY` account, trades one explicit round trip, then submits +six swaps at once in each direction with automatic record selection and +reopens the journal to check recovery. Each run sells 0.6 USDCx forward and +buys it back, so the account ends the run down the pool fees of twelve trades +plus the network fees the delegated prover pays. A run takes tens of minutes +and keeps its journals under `.shield-swap/inventory-e2e/testnet/
/`. +`SHIELD_SWAP_INVENTORY_RECORDS` (3 to 16) and `SHIELD_SWAP_INVENTORY_SWAP_USDCX` +(default `0.1`) set the batch size and the amount per trade. Run it alone: +another trader on the same account competes for its records and blinded +identities. + ## Compliance diff --git a/shield-swap-sdk/tests/integration/test_record_inventory_live.py b/shield-swap-sdk/tests/integration/test_record_inventory_live.py new file mode 100644 index 00000000..1487e09e --- /dev/null +++ b/shield-swap-sdk/tests/integration/test_record_inventory_live.py @@ -0,0 +1,480 @@ +"""Write tier: record inventory maintenance, recovery, and concurrent swaps on the REAL testnet. + +_source: ProvableHQ/veil packages/shield-swap/test/integration/ +{recordInventoryHarness.ts, recordInventory.e2e.test.ts, +recordInventoryStrategy.e2e.test.ts, concurrentInventory.e2e.test.ts}, +PR #172 @ 2e324d5. + +Keeps ``SHIELD_SWAP_INVENTORY_RECORDS`` (default 6) equal records of USDCx and +ETH in the e2e account, trades USDCx to ETH and back (one explicit round trip, +then a concurrent batch in each direction with automatic record selection), +and verifies record identity, exact token accounting, durable reservation +outcomes, and recovery after reopening the journal. Spends testnet funds and +takes tens of minutes. Opt in with ``-m "live and slow"`` and +``ALEO_E2E_PRIVATE_KEY``; the journal lives under +``.shield-swap/inventory-e2e/testnet/
/`` and survives the run. +""" +from __future__ import annotations + +import importlib.util +import json +import os +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path +from types import ModuleType +from typing import Any, Callable + +import pytest + +from aleo import Aleo, HTTPProvider, TransactionNotFound +from aleo.facade.call import BoundCall +from aleo.inventory import ( + InventoryRecord, InventoryTarget, RecordAsset, RecordInventorySnapshot, RecordManagementConfig, RecordSpend, +) +from aleo_shield_swap import ShieldSwap +from aleo_shield_swap._core import record_nonce +from aleo_shield_swap.journal import Journal +from aleo_shield_swap.types import ClaimResult, SwapHandle + +from .conftest import ENDPOINT, PRIVATE_KEY, write_tier + +pytestmark = [pytest.mark.live, pytest.mark.slow] + +WRITE_TIMEOUT = 600.0 # proving, confirmation, and scanner indexing for one write +REBALANCE_TIMEOUT = 2400.0 # a bounded sequence of dependent joins and splits +EXAMPLE = Path(__file__).parents[2] / "examples" / "concurrent-swaps" / "concurrent_swaps.py" +_failed = False + + +@pytest.fixture(autouse=True) +def _abort_after_first_failure(request: pytest.FixtureRequest) -> Any: + """Later steps depend on earlier ones; after a failure, skip instead of spending more.""" + if _failed: + pytest.skip("an earlier inventory step failed; retain the journal before retrying") + yield + report = getattr(request.node, "rep_call", None) + if report is not None and report.failed: + globals()["_failed"] = True + + +def _example() -> ModuleType: + spec = importlib.util.spec_from_file_location("concurrent_swaps_example_live", EXAMPLE) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def poll(label: str, read: Callable[[], Any], ready: Callable[[Any], bool], timeout: float = WRITE_TIMEOUT) -> Any: + """Wait for a chain or scanner predicate without resubmitting anything.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = read() + if ready(value): + return value + time.sleep(3.0) + raise TimeoutError(f"Timed out waiting for {label}; retain the inventory and swap journals before retrying") + + +def base_units(amount: str, decimals: int) -> int: + return int(Decimal(amount).scaleb(decimals)) + + +@dataclass +class Token: + symbol: str + address: str + decimals: int + program: str + minimum: int + target: InventoryTarget + + @property + def asset(self) -> RecordAsset: + return RecordAsset(self.program) + + +class SwitchableStore: + """Delegates to an inner store that a test may replace for one batch.""" + + def __init__(self, inner: Any) -> None: + self.inner = inner + + def list(self, scope: str) -> list[RecordSpend]: + return self.inner.list(scope) + + def acquire(self, entry: RecordSpend) -> RecordSpend | None: + return self.inner.acquire(entry) + + def update(self, scope: str, entry_id: str, patch: Any) -> None: + self.inner.update(scope, entry_id, patch) + + +class Harness: + """The e2e account with a durable reservation journal, a swap journal, and inventory helpers.""" + + def __init__(self) -> None: + self.count = int(os.environ.get("SHIELD_SWAP_INVENTORY_RECORDS", "6")) + if not 3 <= self.count <= 16: + raise ValueError("SHIELD_SWAP_INVENTORY_RECORDS must be from 3 through 16") + probe = Aleo(HTTPProvider(ENDPOINT, network="testnet")) + self.account = probe.account.from_private_key(PRIVATE_KEY) + self.address = str(self.account.address) + self.directory = Path(os.environ.get("SHIELD_SWAP_INVENTORY_E2E_DIR") + or Path(__file__).parents[2] / ".shield-swap" / "inventory-e2e" / "testnet" / self.address) + self.directory.mkdir(parents=True, exist_ok=True) + self.example = _example() + self.scope = f'["aleo:testnet","{self.address}"]' + self.connect() + unresolved = [e for e in self.aleo.inventory.reconcile_record_inventory() + if e.status not in ("confirmed", "rejected", "cancelled")] + if unresolved: + raise RuntimeError(f"Resolve {len(unresolved)} pending inventory operations in {self.directory} before another run") + registry = {t.symbol: t for t in self.dex.api.get_tokens()} + self.tokens: list[Token] = [] + for symbol, minimum in (("USDCx", os.environ.get("SHIELD_SWAP_INVENTORY_MIN_USDCX", "0.1")), + ("ETH", os.environ.get("SHIELD_SWAP_INVENTORY_MIN_ETH", "0.000001"))): + doc = registry[symbol] + if not doc.underlying_program: + raise RuntimeError(f"{symbol} lacks underlying program metadata") + lot = base_units(minimum, doc.decimals) + self.tokens.append(Token(symbol, doc.address, doc.decimals, doc.underlying_program, lot, + InventoryTarget(records=self.count, min_record_amount=lot, distribution="balanced", tolerance_bps=0))) + for token in self.tokens: + inventory = self.scan(token) + if inventory.balance < token.minimum * (self.count + 1): + raise RuntimeError(f"Insufficient {token.symbol} for {self.count} records plus a trading margin") + print(f"inventory preflight: {token.symbol}, {len(inventory.available)} records, {inventory.balance} base units") + + def connect(self) -> None: + """Bind a fresh client to the same account, journal file, and swap journal.""" + self.file_store = self.example.JsonFileRecordInventoryStore(self.directory / "inventory.json") + # The client holds this wrapper, so a test can interpose on acquisition + # without rebuilding the client's frozen management settings. + self.store = SwitchableStore(self.file_store) + self.aleo = Aleo(HTTPProvider(ENDPOINT, network="testnet"), + record_management=RecordManagementConfig(enabled=True, store=self.store, chain_id="aleo:testnet", + wait_for_sync=True, sync_timeout=WRITE_TIMEOUT)) + self.account = self.aleo.account.from_private_key(PRIVATE_KEY) + self.aleo.default_account = self.account + registration = self.aleo.records.register(self.account) + assert registration.get("ok"), registration + self.dex = ShieldSwap(self.aleo) + self.dex.journal = Journal(self.directory / "swaps.jsonl") + self.dex.api.authenticate(self.address, lambda message: str(self.aleo.account.sign(message.encode(), self.account))) + + def restart(self) -> None: + """Reopen the journal file and rebuild the client, as a new process would.""" + self.connect() + + # ── Inventory helpers ──────────────────────────────────────────────────── + + def scan(self, token: Token) -> RecordInventorySnapshot: + return self.aleo.inventory.get_record_inventory(token.asset) + + def assert_balanced(self, token: Token) -> RecordInventorySnapshot: + inventory = self.scan(token) + assert len(inventory.available) == self.count, f"{token.symbol} record count" + assert inventory.reserved == (), f"{token.symbol} pending inputs" + assert all(record.amount >= token.minimum for record in inventory.available), f"{token.symbol} minimum" + amounts = sorted(record.amount for record in inventory.available) + # Zero tolerance permits only integer rounding: equal lots differ by at most one base unit. + assert amounts[-1] - amounts[0] <= 1, f"{token.symbol} balanced amounts {amounts}" + assert len({record.id for record in inventory.available}) == self.count + return inventory + + def rebalance(self, token: Token) -> Any: + before = self.scan(token) + result = self.dex.rebalance_inventory(token.address, token.target, max_transactions=24, + timeout=WRITE_TIMEOUT, poll_interval=3.0) + assert result.status == "complete", result.reason + after = self.assert_balanced(token) + assert after.balance == before.balance, f"{token.symbol} rebalance conserves value" + print(f"inventory balanced: {self.count} {token.symbol} records, {result.completed_steps} transactions") + return result + + def confirm(self, tx_id: str) -> dict[str, Any]: + def read() -> Any: + try: + return self.aleo.network.get_confirmed_transaction(tx_id) + except TransactionNotFound: + return None + confirmed = poll(f"confirmation of {tx_id}", read, lambda value: value is not None) + assert confirmed["status"] == "accepted", f"Transaction {tx_id} was not accepted" + self.aleo.inventory.reconcile_record_inventory() + return confirmed["transaction"] + + def output_records(self, token: Token, tx_id: str, expected: int) -> list[InventoryRecord]: + transaction = self.confirm(tx_id) + # Only outputs of the accepted transaction may satisfy this step; a + # concurrent deposit or a stale scanner page cannot stand in for one. + commitments = {str(output["id"]) for transition in transaction["execution"]["transitions"] + for output in transition.get("outputs", []) + if output.get("type") in ("record", "record_with_dynamic_id")} + inventory = poll(f"{token.symbol} outputs of {tx_id}", lambda: self.scan(token), + lambda inv: len([r for r in inv.available if r.commitment in commitments]) == expected) + return [record for record in inventory.available if record.commitment in commitments] + + def journal_entry(self, tx_id: str) -> RecordSpend: + entry = next((e for e in self.store.list(self.scope) if e.transaction_id == tx_id), None) + assert entry is not None, f"journal entry for {tx_id}" + return entry + + def pool_key(self, a: Token, b: Token) -> str: + """The deepest pool for the pair, by on-chain liquidity.""" + best, deepest = None, -1 + for pool in self.dex.api.get_pools(): + if {pool.token0, pool.token1} == {a.address, b.address}: + liquidity = self.dex.get_slot(pool.key).liquidity + if liquidity > deepest: + best, deepest = pool.key, liquidity + assert best is not None and deepest > 0, "No USDCx/ETH pool with on-chain liquidity" + return best + + +@pytest.fixture(scope="module") +def h() -> Harness: + if not PRIVATE_KEY: + pytest.skip("ALEO_E2E_PRIVATE_KEY not set") + return Harness() + + +@pytest.fixture(scope="module") +def usdcx(h: Harness) -> Token: + return h.tokens[0] + + +@pytest.fixture(scope="module") +def eth(h: Harness) -> Token: + return h.tokens[1] + + +@pytest.fixture(scope="module") +def pool(h: Harness, usdcx: Token, eth: Token) -> str: + return h.pool_key(usdcx, eth) + + +@pytest.fixture(scope="module") +def trade_amount(usdcx: Token) -> int: + amount = base_units(os.environ.get("SHIELD_SWAP_INVENTORY_SWAP_USDCX", "0.1"), usdcx.decimals) + assert amount > 0 + return amount + + +TRANSACTIONS: list[str] = [] +CONSUMED: set[str] = set() + + +def swap_and_claim(h: Harness, pool: str, source: Token, target: Token, amount: int, record: InventoryRecord) -> tuple[int, InventoryRecord]: + """One explicit-record swap and its claim, checked against independently predicted balances.""" + before_in, before_out = h.scan(source), h.scan(target) + assert record.amount >= amount, f"{source.symbol} covering input" + assert record.id not in CONSUMED, "a previous swap input cannot be reused" + handle = h.dex.swap(pool_key=pool, token_in_id=source.address, amount_in=amount, slippage_bps=100, + token_record=record.plaintext, token_in_program=source.program).delegate(wait=True, wait_timeout=WRITE_TIMEOUT) + CONSUMED.add(record.id) + TRANSACTIONS.append(handle.transaction_id) + assert handle.swap_id, "confirmed swap has a claim id" + claim = h.dex.claim_swap_output(handle, timeout=WRITE_TIMEOUT).delegate(wait=True, wait_timeout=WRITE_TIMEOUT) + TRANSACTIONS.append(claim.transaction_id) + outputs = h.output_records(target, claim.transaction_id, 1) + assert claim.amount_out > 0 and claim.amount_remaining == 0, "small test trades fill completely" + assert outputs[0].amount == claim.amount_out + after_in, after_out = h.scan(source), h.scan(target) + assert after_in.balance == before_in.balance - amount, f"{source.symbol} debit" + assert after_out.balance == before_out.balance + claim.amount_out, f"{target.symbol} credit" + assert not any(r.id == record.id for r in after_in.available), "spent nonce stays excluded" + spend = h.journal_entry(handle.transaction_id) + assert spend.status == "confirmed" and record.id in spend.records + assert handle.swap_id not in {p.swap_id for p in h.dex.journal.pending_claims()} + return claim.amount_out, outputs[0] + + +@write_tier +def test_prepares_balanced_records_above_the_minimum(h: Harness, usdcx: Token, eth: Token, trade_amount: int) -> None: + for token in h.tokens: + h.rebalance(token) + inventory = h.assert_balanced(usdcx) + # Each source record stays above its minimum after a forward swap, so the + # count change from each claim is the only reason maintenance is needed. + assert all(record.amount - trade_amount >= usdcx.minimum for record in inventory.available) + + +@write_tier +def test_round_trip_then_restores_balanced_records(h: Harness, usdcx: Token, eth: Token, pool: str, trade_amount: int) -> None: + before_usdcx, before_eth = h.assert_balanced(usdcx), h.assert_balanced(eth) + amount_out, eth_record = swap_and_claim(h, pool, usdcx, eth, trade_amount, before_usdcx.available[0]) + assert len(h.scan(eth).available) == h.count + 1 + # Spend the exact newly claimed ETH record on the return leg, keeping the + # existing ETH lots and avoiding a second ambiguous selection. + returned, _ = swap_and_claim(h, pool, eth, usdcx, amount_out, eth_record) + assert len(h.scan(usdcx).available) == h.count + 1 + repair = h.dex.plan_inventory(usdcx.address, usdcx.target, max_transactions=24) + assert repair.steps[0].kind == "join" and len(repair.steps[0].inputs) == min(h.count + 1, 16) + assert repair.steps[0].join_program is not None and repair.steps[0].join_program.program.startswith("aj_test_usdcx_stablecoin_") + assert h.rebalance(usdcx).completed_steps == len(repair.steps) + assert h.rebalance(eth).completed_steps == 0 + assert h.assert_balanced(usdcx).balance == before_usdcx.balance - trade_amount + returned + assert h.assert_balanced(eth).balance == before_eth.balance + print(f"round trip: {trade_amount} USDCx -> {amount_out} ETH -> {returned} USDCx base units") + + +class Barrier: + """Holds the first *racers* acquisitions until all have arrived, so every swap selects the same record first.""" + + def __init__(self, store: Any, racers: int) -> None: + self._store, self._racers = store, racers + self.attempts: list[tuple[str, ...]] = [] + self._lock = threading.Lock() + self._gate = threading.Event() + self._deadline = threading.Timer(60.0, self._gate.set) # a failed preparation must not strand the others + self._deadline.start() + + def list(self, scope: str) -> list[RecordSpend]: + return self._store.list(scope) + + def update(self, scope: str, entry_id: str, patch: Any) -> None: + self._store.update(scope, entry_id, patch) + + def acquire(self, entry: RecordSpend) -> RecordSpend | None: + with self._lock: + self.attempts.append(tuple(entry.records)) + first = len(self.attempts) <= self._racers + if len(self.attempts) == self._racers: + self._gate.set() + if first and entry.records: + self._gate.wait() + return self._store.acquire(entry) + + def release(self) -> None: + self._deadline.cancel() + self._gate.set() + + +def batch(h: Harness, pool: str, source: Token, target: Token, amounts: list[int], monkeypatch: pytest.MonkeyPatch) -> tuple[list[tuple[SwapHandle, ClaimResult, InventoryRecord]], list[str]]: + """Submit one swap per amount at the same time with automatic record selection, then claim each.""" + before_in, before_out = h.scan(source), h.scan(target) + barrier = Barrier(h.file_store, len(amounts)) + h.store.inner = barrier + proved: list[str] = [] + active, peak = [0], [0] + counter = threading.Lock() + original_delegate = BoundCall.delegate + + def counting_delegate(self: BoundCall, account: Any = None, *, broadcast: bool = True, **kw: Any) -> Any: + # Count only the prover round trips the hook makes for the swaps, and + # record nonces rather than plaintext. + if broadcast or self.function_name not in ("swap", "swap_from_wrapped"): + return original_delegate(self, account, broadcast=broadcast, **kw) + nonce = next((record_nonce(a) for a in self.args if "_nonce" in a), None) + assert nonce, "swap reached proving without a record input" + with counter: + proved.append(nonce) + active[0] += 1 + peak[0] = max(peak[0], active[0]) + try: + return original_delegate(self, account, broadcast=broadcast, **kw) + finally: + with counter: + active[0] -= 1 + + monkeypatch.setattr(BoundCall, "delegate", counting_delegate) + outcomes: list[Any] = [] + try: + def one(amount: int) -> Any: + try: + handle = h.dex.swap(pool_key=pool, token_in_id=source.address, amount_in=amount, slippage_bps=100, + token_in_program=source.program).delegate(wait=True, wait_timeout=WRITE_TIMEOUT) + TRANSACTIONS.append(handle.transaction_id) + print(f"{source.symbol} -> {target.symbol} swap confirmed: {handle.transaction_id}") + return handle + except BaseException as exc: # noqa: BLE001 - settled below + return exc + with ThreadPoolExecutor(max_workers=len(amounts)) as workers: + outcomes = list(workers.map(one, amounts)) + finally: + barrier.release() + h.store.inner = h.file_store + monkeypatch.undo() + handles = [o for o in outcomes if isinstance(o, SwapHandle)] + + def claim(handle: SwapHandle) -> Any: + try: + result = h.dex.claim_swap_output(handle, timeout=WRITE_TIMEOUT).delegate(wait=True, wait_timeout=WRITE_TIMEOUT) + TRANSACTIONS.append(result.transaction_id) + outputs = h.output_records(target, result.transaction_id, 1) + assert result.amount_out > 0 and result.amount_remaining == 0 + assert outputs[0].amount == result.amount_out + print(f"{source.symbol} -> {target.symbol} claimed {result.amount_out} base units: {result.transaction_id}") + return handle, result, outputs[0] + except BaseException as exc: # noqa: BLE001 - settled below + return exc + + # Settle every claim before asserting, even if a sibling swap or claim failed. + with ThreadPoolExecutor(max_workers=max(1, len(handles))) as workers: + claims = list(workers.map(claim, handles)) + failures = [f"operation {i + 1}: {o}" for i, o in enumerate([*outcomes, *claims]) if isinstance(o, BaseException)] + assert failures == [], "all submitted swaps and successful outputs must settle" + settled = [c for c in claims if not isinstance(c, BaseException)] + assert len(handles) == len(amounts) + assert len({handle.transaction_id for handle in handles}) == len(amounts) + assert len({handle.blinded_address for handle in handles}) == len(amounts) + assert peak[0] > 1, "proving executions overlap" + assert len(proved) == len(amounts) and len(set(proved)) == len(amounts), "every prover consumes a distinct input" + first = [a for a in barrier.attempts if a][: len(amounts)] + assert len(first) == len(amounts) + if source.symbol == "USDCx": + assert len({ids[0] for ids in first}) == 1, "all first selections collide" + assert len([a for a in barrier.attempts if a]) > len(amounts), "the losing selections retry" + for nonce in proved: + assert nonce not in CONSUMED, "inputs from earlier batches are not reused" + CONSUMED.add(nonce) + after_in, after_out = h.scan(source), h.scan(target) + assert after_in.balance == before_in.balance - sum(amounts) + assert after_out.balance == before_out.balance + sum(c.amount_out for _, c, _ in settled) + assert not any(r.id in proved for r in after_in.available), "spent inputs stay excluded" + for handle in handles: + entry = h.journal_entry(handle.transaction_id) + assert entry.status == "confirmed" and len(entry.records) == 1 and entry.records[0] in proved + assert not {p.swap_id for p in h.dex.journal.pending_claims()} & {hd.swap_id for hd in handles} + print(f"{source.symbol} batch verified: {len(handles)} swaps, {len(settled)} claims, {peak[0]} overlapping executions, " + f"{len([a for a in barrier.attempts if a]) - len(handles)} reservation conflicts") + return settled, proved + + +@write_tier +def test_swaps_concurrently_in_both_directions_and_restores_the_inventories( + h: Harness, usdcx: Token, eth: Token, pool: str, trade_amount: int, monkeypatch: pytest.MonkeyPatch, +) -> None: + before_usdcx, before_eth = h.assert_balanced(usdcx), h.assert_balanced(eth) + forward, _ = batch(h, pool, usdcx, eth, [trade_amount] * h.count, monkeypatch) + # Return the amounts received; automatic selection matches each to a + # covering record instead of passing the plaintexts explicitly. + backward, proved = batch(h, pool, eth, usdcx, [c.amount_out for _, c, _ in forward], monkeypatch) + assert set(proved) == {record.id for _, _, record in forward} + for token in h.tokens: + h.rebalance(token) + assert h.assert_balanced(usdcx).balance == before_usdcx.balance - trade_amount * h.count + sum(c.amount_out for _, c, _ in backward) + assert h.assert_balanced(eth).balance == before_eth.balance + + +@write_tier +def test_retains_confirmed_checkpoints_after_reopening_the_journal(h: Harness) -> None: + h.restart() + journal = h.aleo.inventory.reconcile_record_inventory() + assert len(TRANSACTIONS) == 4 + h.count * 4 + for tx_id in TRANSACTIONS: + entry = next((e for e in journal if e.transaction_id == tx_id), None) + assert entry is not None and entry.status == "confirmed", f"confirmed checkpoint for {tx_id}" + assert entry.transaction is None + used = [nonce for e in journal if e.status == "confirmed" for nonce in e.records] + assert len(set(used)) == len(used), "no record was consumed by two confirmed transactions" + for token in h.tokens: + h.assert_balanced(token) + print(json.dumps({"transactions": len(TRANSACTIONS), "journal": len(journal)})) From 78040de987b41f902f4667f7ea8f74ce9f9926b8 Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 09:26:58 -0400 Subject: [PATCH 03/11] feat: ship JSON file and SQLite reservation stores JsonFileRecordInventoryStore keeps the journal in one JSON file under a file lock; SqliteRecordInventoryStore uses the stdlib sqlite3 module with the same table layout and row JSON as Veil's store, so the two SDKs can share a journal. Both record the writer's pid and host and release a reserved entry whose local writer has exited before proving. The store contract tests run against all three stores. The concurrent-swaps example and the live inventory suite use the SQLite store instead of a private class; the README compares the three stores. --- README.md | 27 +- sdk/Readme.md | 2 +- sdk/python/aleo/__init__.py | 2 + sdk/python/aleo/__init__.pyi | 2 + sdk/python/aleo/inventory/__init__.py | 13 +- sdk/python/aleo/inventory/store.py | 333 +++++++++++++++++- sdk/python/tests/test_inventory_store.py | 165 +++++++-- .../examples/concurrent-swaps/README.md | 17 +- .../concurrent-swaps/concurrent_swaps.py | 96 +---- .../integration/test_record_inventory_live.py | 20 +- .../tests/test_concurrent_swaps_example.py | 40 +-- 11 files changed, 520 insertions(+), 197 deletions(-) diff --git a/README.md b/README.md index b7fdca93..35907918 100644 --- a/README.md +++ b/README.md @@ -266,14 +266,25 @@ transaction where one exists (up to 16 credits or dedicated stablecoin records, 15 for other ARC-20 and ARC-22 tokens). `join_records`, `split_record`, and `auto_join` perform single operations without a plan. -The default journal is in memory and lasts for one process. Supply any object -with `list`, `acquire`, and `update` methods as `store` to share reservations -across processes or to resume after a restart; `acquire` must reserve all of an -entry's records or none. After an interruption, `reconcile_record_inventory()` -checks each pending entry against the chain and, with `rebroadcast=True`, -resubmits a saved proved transaction with its original id. A write that finds -its record already reserved raises `RecordReservationConflictError` before any -proof is built, so the caller can choose another record. +The journal is kept by a store chosen with `RecordManagementConfig(store=...)`. +Three ship with the SDK; the choice decides what survives a restart and which +processes coordinate. + +| Store | Survives a restart | Coordinates | Choose it when | +| --- | --- | --- | --- | +| `MemoryRecordInventoryStore` (default) | No | Threads in one process | A script runs to completion and nothing else trades for the account. | +| `JsonFileRecordInventoryStore(path)` | Yes | Processes on one machine | The journal should be readable in an editor; one JSON file under a file lock. | +| `SqliteRecordInventoryStore(path)` | Yes | Processes on one machine | A long-running service, or a journal shared with a Veil client; same layout as Veil's store. | + +Both file-backed stores release a reservation left by a process on the same +machine that exited before it proved anything. Any object with `list`, +`acquire`, and `update` methods works as a store for other backends, such as a +database shared across machines; `acquire` must reserve all of an entry's +records or none. After an interruption, `reconcile_record_inventory()` checks +each pending entry against the chain and, with `rebroadcast=True`, resubmits a +saved proved transaction with its original id. A write that finds its record +already reserved raises `RecordReservationConflictError` before any proof is +built, so the caller can choose another record. The [Shield Swap guide](shield-swap-sdk/README.md#concurrent-swaps) shows the same mechanism preparing records for several trades submitted at once. diff --git a/sdk/Readme.md b/sdk/Readme.md index 33c95418..aace801b 100644 --- a/sdk/Readme.md +++ b/sdk/Readme.md @@ -179,7 +179,7 @@ plan = aleo.inventory.plan_record_inventory( result = aleo.inventory.rebalance_record_inventory(plan, max_fee_microcredits=1_000_000) ``` -`get_record_inventory` reads; `plan_record_inventory` plans without signing; `join_records`, `split_record`, and `auto_join` run single operations; `reconcile_record_inventory` settles pending journal entries against the chain and can rebroadcast a saved proved transaction. The journal is in memory by default; pass any object with `list`, `acquire`, and `update` as `store` for a database or file. The top-level [README](../README.md#manage-private-records) explains the choices and the recovery steps. +`get_record_inventory` reads; `plan_record_inventory` plans without signing; `join_records`, `split_record`, and `auto_join` run single operations; `reconcile_record_inventory` settles pending journal entries against the chain and can rebroadcast a saved proved transaction. The journal is in memory by default; `JsonFileRecordInventoryStore(path)` and `SqliteRecordInventoryStore(path)` keep it across restarts and coordinate processes on one machine, and any object with `list`, `acquire`, and `update` works for another backend. The top-level [README](../README.md#manage-private-records) compares the stores and explains the recovery steps. ## Async (`AsyncAleo`) diff --git a/sdk/python/aleo/__init__.py b/sdk/python/aleo/__init__.py index 8850e4df..92021f42 100644 --- a/sdk/python/aleo/__init__.py +++ b/sdk/python/aleo/__init__.py @@ -55,6 +55,7 @@ InventoryRecord as InventoryRecord, InventoryResult as InventoryResult, InventoryTarget as InventoryTarget, + JsonFileRecordInventoryStore as JsonFileRecordInventoryStore, MemoryRecordInventoryStore as MemoryRecordInventoryStore, RecordAsset as RecordAsset, RecordInventory as RecordInventory, @@ -64,6 +65,7 @@ RecordManagementConfig as RecordManagementConfig, RecordReservationConflictError as RecordReservationConflictError, RecordSpend as RecordSpend, + SqliteRecordInventoryStore as SqliteRecordInventoryStore, TokenJoinRouter as TokenJoinRouter, ) diff --git a/sdk/python/aleo/__init__.pyi b/sdk/python/aleo/__init__.pyi index a624bb9c..f62ee40c 100644 --- a/sdk/python/aleo/__init__.pyi +++ b/sdk/python/aleo/__init__.pyi @@ -36,6 +36,7 @@ from .inventory import ( InventoryRecord as InventoryRecord, InventoryResult as InventoryResult, InventoryTarget as InventoryTarget, + JsonFileRecordInventoryStore as JsonFileRecordInventoryStore, MemoryRecordInventoryStore as MemoryRecordInventoryStore, RecordAsset as RecordAsset, RecordInventory as RecordInventory, @@ -45,6 +46,7 @@ from .inventory import ( RecordManagementConfig as RecordManagementConfig, RecordReservationConflictError as RecordReservationConflictError, RecordSpend as RecordSpend, + SqliteRecordInventoryStore as SqliteRecordInventoryStore, TokenJoinRouter as TokenJoinRouter, ) diff --git a/sdk/python/aleo/inventory/__init__.py b/sdk/python/aleo/inventory/__init__.py index f4a2808d..4b61ef53 100644 --- a/sdk/python/aleo/inventory/__init__.py +++ b/sdk/python/aleo/inventory/__init__.py @@ -49,7 +49,9 @@ ) from .planner import build_inventory_plan as build_inventory_plan from .store import ( + JsonFileRecordInventoryStore as JsonFileRecordInventoryStore, MemoryRecordInventoryStore as MemoryRecordInventoryStore, + SqliteRecordInventoryStore as SqliteRecordInventoryStore, held_record_ids as held_record_ids, holds_records as holds_records, ) @@ -77,11 +79,12 @@ __all__ = [ "AssetCapabilities", "AsyncRecordInventory", "CREDITS_SPLIT_DEDUCTION", "InventoryError", "InventoryPlan", "InventoryPlanError", "InventoryRecord", "InventoryResult", "InventoryStep", "InventoryTarget", - "MemoryRecordInventoryStore", "PlannedRecord", "PreparedTransaction", "ProgramFunction", "ProgramInterface", - "ProgramParam", "ProgramRecord", "RECORD_JOIN_PROGRAMS", "RecordAsset", "RecordField", "RecordInventory", - "RecordInventorySnapshot", "RecordInventoryStore", "RecordJoinProgram", "RecordManagementConfig", - "RecordReservationConflictError", "RecordSpend", "RecordSpendPatch", "ResolvedRecordAsset", "SpendCheckpoint", - "TokenJoinRouter", "build_inventory_plan", "decode_record", "held_record_ids", "holds_records", + "JsonFileRecordInventoryStore", "MemoryRecordInventoryStore", "PlannedRecord", "PreparedTransaction", + "ProgramFunction", "ProgramInterface", "ProgramParam", "ProgramRecord", "RECORD_JOIN_PROGRAMS", "RecordAsset", + "RecordField", "RecordInventory", "RecordInventorySnapshot", "RecordInventoryStore", "RecordJoinProgram", + "RecordManagementConfig", "RecordReservationConflictError", "RecordSpend", "RecordSpendPatch", + "ResolvedRecordAsset", "SpendCheckpoint", "SqliteRecordInventoryStore", "TokenJoinRouter", "build_inventory_plan", + "decode_record", "held_record_ids", "holds_records", "inventory_output_commitments", "inventory_record_id", "managed_inventory", "parse_program_interface", "prepared_from_payload", "prepared_from_result", "record_input_ids", "scope_for", "transaction_fee", "validate_asset", "validate_record_join_program", "validate_token_join_router", diff --git a/sdk/python/aleo/inventory/store.py b/sdk/python/aleo/inventory/store.py index 6f2bca89..30f38a1a 100644 --- a/sdk/python/aleo/inventory/store.py +++ b/sdk/python/aleo/inventory/store.py @@ -1,15 +1,36 @@ """Journal stores for record reservations. -The SDK ships one store, :class:`MemoryRecordInventoryStore`, for a single -process. Anything that satisfies :class:`~aleo.inventory.types.RecordInventoryStore` -can replace it: a database shared by several processes, a key-value service, -or a file. A journal holds record nonces, program and function names, and -proved transactions; it never holds keys or record plaintext. +A journal holds record nonces, program and function names, and proved +transactions awaiting broadcast; it never holds keys or record plaintext. +Three stores ship with the SDK: + +- :class:`MemoryRecordInventoryStore` lasts for one process. Use it for a + script that runs to completion. +- :class:`JsonFileRecordInventoryStore` keeps the journal in one JSON file + under a file lock. Use it when a process may restart and the file can be + inspected by hand; it coordinates processes on one machine. +- :class:`SqliteRecordInventoryStore` keeps the journal in a SQLite database + with the same layout as Veil's store, so the two SDKs can share one + journal. Use it for a long-running service or several processes on one + machine. + +Anything else that satisfies :class:`~aleo.inventory.types.RecordInventoryStore` +can replace them, such as a database shared across machines. The two +file-backed stores also release reservations left by a process on the same +machine that exited before proving, since such a reservation cannot have +reached the network. """ from __future__ import annotations import copy +import json +import os +import socket +import sqlite3 +import sys import threading +from pathlib import Path +from typing import Any from .types import RecordInventoryStore, RecordSpend, RecordSpendPatch @@ -116,4 +137,304 @@ def reserved_record_ids(store: RecordInventoryStore | None, scope: str) -> set[s return out -__all__ = ["MemoryRecordInventoryStore", "RecordInventoryStore", "held_record_ids", "holds_records", "reserved_record_ids"] +# ── Durable stores ──────────────────────────────────────────────────────────── + + +def _apply_patch(entry: RecordSpend, patch: RecordSpendPatch) -> None: + if "status" in patch: + entry.status = patch["status"] + if "transaction_id" in patch: + entry.transaction_id = patch["transaction_id"] + if "transaction" in patch: + entry.transaction = copy.deepcopy(patch["transaction"]) + if "fee_microcredits" in patch: + entry.fee_microcredits = patch["fee_microcredits"] + + +def _process_alive(pid: int) -> bool: + """Whether a process id on this machine is still running.""" + if pid <= 0: + return False + if sys.platform == "win32": # pragma: no cover - exercised on Windows only + import ctypes + handle = ctypes.windll.kernel32.OpenProcess(0x1000, False, pid) # type: ignore[attr-defined] + if not handle: + return False + ctypes.windll.kernel32.CloseHandle(handle) # type: ignore[attr-defined] + return True + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def _private_file(path: Path) -> None: + """Create *path* and its parent readable by the owner only, keeping an existing file's contents.""" + path.parent.mkdir(parents=True, exist_ok=True) + try: + path.parent.chmod(0o700) + except OSError: # pragma: no cover - not every filesystem honours modes + pass + if not path.exists(): + fd = os.open(path, os.O_CREAT | os.O_WRONLY, 0o600) + os.close(fd) + + +class _FileLock: + """An exclusive lock on a sibling file, held for one read-modify-write.""" + + def __init__(self, path: Path) -> None: + self._path = path + self._thread_lock = threading.Lock() + + def __enter__(self) -> None: + self._thread_lock.acquire() + self._handle = open(self._path, "a+") + if sys.platform == "win32": # pragma: no cover - exercised on Windows only + import msvcrt + msvcrt.locking(self._handle.fileno(), msvcrt.LK_LOCK, 1) # type: ignore[attr-defined] + else: + import fcntl + fcntl.flock(self._handle, fcntl.LOCK_EX) + + def __exit__(self, *exc: object) -> None: + try: + if sys.platform == "win32": # pragma: no cover + import msvcrt + msvcrt.locking(self._handle.fileno(), msvcrt.LK_UNLCK, 1) # type: ignore[attr-defined] + else: + import fcntl + fcntl.flock(self._handle, fcntl.LOCK_UN) + self._handle.close() + finally: + self._thread_lock.release() + + +class _DurableStore: + """Shared acquisition, patching, and stale-reservation recovery for the file-backed stores. + + Each entry is stored with the process id and host that wrote it. On every + read under *scope*, a ``reserved`` entry written on this host by a process + that no longer runs is cancelled: the prepared checkpoint is always + written before broadcast, so such an entry never reached the network. + """ + + def __init__(self) -> None: + self._pid = os.getpid() + self._host = socket.gethostname() + + # Subclasses implement these inside one exclusive transaction. + def _rows(self, scope: str) -> list[tuple[RecordSpend, int, str]]: + raise NotImplementedError + + def _insert(self, entry: RecordSpend) -> RecordSpend: + raise NotImplementedError + + def _store(self, entry: RecordSpend) -> None: + raise NotImplementedError + + def _transaction(self) -> Any: + raise NotImplementedError + + def _recover(self, scope: str) -> list[RecordSpend]: + entries: list[RecordSpend] = [] + for entry, pid, host in self._rows(scope): + if entry.status == "reserved" and host == self._host and not _process_alive(pid): + entry.status = "cancelled" + self._store(entry) + entries.append(entry) + return entries + + def list(self, scope: str) -> list[RecordSpend]: + """Return every entry under *scope*, oldest first, after releasing reservations of dead local processes.""" + with self._transaction(): + return self._recover(scope) + + def acquire(self, entry: RecordSpend) -> RecordSpend | None: + """Reserve all of the entry's records or none of them, atomically across processes. + + Raises: + ValueError: The entry names the same record twice. + """ + if len(set(entry.records)) != len(entry.records): + raise ValueError("Duplicate record inputs") + with self._transaction(): + wanted = set(entry.records) + for old in self._recover(entry.scope): + if wanted.intersection(held_record_ids(old)): + return None + return self._insert(copy.deepcopy(entry)) + + def update(self, scope: str, entry_id: str, patch: RecordSpendPatch) -> None: + """Apply a checkpoint to an existing entry. + + Raises: + KeyError: No entry with that id exists under *scope*. + """ + with self._transaction(): + for entry, _, _ in self._rows(scope): + if entry.id == entry_id: + _apply_patch(entry, patch) + self._store(entry) + return + raise KeyError("Unknown record reservation") + + +class JsonFileRecordInventoryStore(_DurableStore): + """A reservation journal in one JSON file, shared by processes on one machine. + + Every call takes an exclusive lock on a sibling ``.lock`` file, reads the + whole file, and writes it back atomically, so two processes can never + hold the same record. The file is created readable by its owner only and + can be inspected with any JSON viewer. It holds nonces, program and + function names, statuses, and proved transactions awaiting broadcast; + never keys or record plaintext. + + Args: + path: The journal file. Its parent directory is created if needed. + """ + + def __init__(self, path: "Path | str") -> None: + super().__init__() + self.path = Path(path) + _private_file(self.path) + self._lock = _FileLock(self.path.with_suffix(self.path.suffix + ".lock")) + self._cache: list[dict[str, Any]] | None = None + + def _transaction(self) -> Any: + store = self + + class _Guard: + def __enter__(self) -> None: + store._lock.__enter__() + text = store.path.read_text() or "[]" + store._cache = [dict(item) for item in json.loads(text)] + + def __exit__(self, kind: object, exc: object, tb: object) -> None: + try: + if exc is None and store._cache is not None: + tmp = store.path.with_suffix(store.path.suffix + ".tmp") + with open(tmp, "w") as handle: + json.dump(store._cache, handle, indent=2) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, store.path) + finally: + store._cache = None + store._lock.__exit__(kind, exc, tb) + return _Guard() + + def _rows(self, scope: str) -> list[tuple[RecordSpend, int, str]]: + assert self._cache is not None + return [(RecordSpend.from_dict(item["data"]), int(item.get("pid", 0)), str(item.get("host", ""))) + for item in self._cache if item["data"].get("scope") == scope] + + def _insert(self, entry: RecordSpend) -> RecordSpend: + assert self._cache is not None + entry.id = str(max((int(item["data"]["id"]) for item in self._cache), default=0) + 1) + self._cache.append({"data": entry.to_dict(), "pid": self._pid, "host": self._host}) + return copy.deepcopy(entry) + + def _store(self, entry: RecordSpend) -> None: + assert self._cache is not None + for item in self._cache: + if item["data"].get("scope") == entry.scope and str(item["data"].get("id")) == entry.id: + item["data"] = entry.to_dict() + return + raise KeyError("Unknown record reservation") + + +class SqliteRecordInventoryStore(_DurableStore): + """A reservation journal in a SQLite database, shared by processes on one machine. + + Uses Python's bundled ``sqlite3``; nothing extra to install. The table + layout and the JSON inside it match Veil's SQLite store, so a journal + written by one SDK is read correctly by the other. Each call runs in a + ``BEGIN IMMEDIATE`` transaction, so contention checks and inserts are + atomic across processes; writes are flushed before the call returns. The + database file is created readable by its owner only. + + Args: + path: The database file. Its parent directory is created if needed. + """ + + def __init__(self, path: "Path | str") -> None: + super().__init__() + self.path = Path(path) + _private_file(self.path) + self._connection = sqlite3.connect(str(self.path), isolation_level=None, check_same_thread=False, timeout=5.0) + self._thread_lock = threading.Lock() + self._connection.executescript( + "PRAGMA busy_timeout = 5000; PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL;\n" + "CREATE TABLE IF NOT EXISTS record_spends (\n" + " id INTEGER PRIMARY KEY AUTOINCREMENT, scope TEXT NOT NULL, data TEXT NOT NULL,\n" + " pid INTEGER NOT NULL, host TEXT NOT NULL\n" + ");\n" + "CREATE INDEX IF NOT EXISTS record_spends_scope ON record_spends(scope);" + ) + + def close(self) -> None: + """Release the database handle; the journal stays on disk.""" + self._connection.close() + + def __enter__(self) -> "SqliteRecordInventoryStore": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def _transaction(self) -> Any: + store = self + + class _Guard: + def __enter__(self) -> None: + store._thread_lock.acquire() + try: + store._connection.execute("BEGIN IMMEDIATE") + except BaseException: + store._thread_lock.release() + raise + + def __exit__(self, kind: object, exc: object, tb: object) -> None: + try: + store._connection.execute("COMMIT" if exc is None else "ROLLBACK") + finally: + store._thread_lock.release() + return _Guard() + + def _rows(self, scope: str) -> list[tuple[RecordSpend, int, str]]: + rows = self._connection.execute( + "SELECT id, data, pid, host FROM record_spends WHERE scope = ? ORDER BY id", (scope,), + ).fetchall() + out: list[tuple[RecordSpend, int, str]] = [] + for row_id, data, pid, host in rows: + entry = RecordSpend.from_dict(json.loads(data)) + if entry.scope != scope or entry.id != str(row_id): + raise ValueError("Invalid inventory journal entry") + out.append((entry, int(pid), str(host))) + return out + + def _insert(self, entry: RecordSpend) -> RecordSpend: + cursor = self._connection.execute( + "INSERT INTO record_spends(scope, data, pid, host) VALUES (?, ?, ?, ?)", + (entry.scope, json.dumps(entry.to_dict()), self._pid, self._host), + ) + entry.id = str(cursor.lastrowid) + self._connection.execute("UPDATE record_spends SET data = ? WHERE id = ?", (json.dumps(entry.to_dict()), int(entry.id))) + return copy.deepcopy(entry) + + def _store(self, entry: RecordSpend) -> None: + changed = self._connection.execute( + "UPDATE record_spends SET data = ? WHERE scope = ? AND id = ?", (json.dumps(entry.to_dict()), entry.scope, int(entry.id)), + ).rowcount + if changed != 1: + raise KeyError("Unknown record reservation") + + +__all__ = [ + "JsonFileRecordInventoryStore", "MemoryRecordInventoryStore", "RecordInventoryStore", "SqliteRecordInventoryStore", + "held_record_ids", "holds_records", "reserved_record_ids", +] diff --git a/sdk/python/tests/test_inventory_store.py b/sdk/python/tests/test_inventory_store.py index 32469e27..83aabd7a 100644 --- a/sdk/python/tests/test_inventory_store.py +++ b/sdk/python/tests/test_inventory_store.py @@ -1,18 +1,31 @@ -"""In-memory journal store: atomic acquisition, held records, patches. +"""Journal stores: one contract, three implementations (memory, JSON file, SQLite). -_source: ProvableHQ/veil packages/core/src/inventory/store.ts and -test/inventory/privateFees.test.ts (rejected fee records), PR #172 @ 2e324d5. +_source: ProvableHQ/veil packages/core/src/inventory/store.ts, +packages/shield-swap-cli/src/storage.ts, and the store cases in +test/inventory/privateFees.test.ts and shield-swap-cli/test/inventory-storage.test.ts, +PR #172 @ 2e324d5. """ from __future__ import annotations +import json +import os +import sqlite3 +import subprocess +import sys import threading +from pathlib import Path +from typing import Any, Callable import pytest -from aleo.inventory import MemoryRecordInventoryStore, RecordSpend, held_record_ids, holds_records +from aleo.inventory import ( + JsonFileRecordInventoryStore, MemoryRecordInventoryStore, RecordSpend, SqliteRecordInventoryStore, held_record_ids, + holds_records, +) from aleo.inventory.store import reserved_record_ids SCOPE = '["testnet","aleo1owner"]' +Factory = Callable[[], Any] def entry(*records: str, status: str = "reserved", fee: str | None = None, scope: str = SCOPE) -> RecordSpend: @@ -20,51 +33,54 @@ def entry(*records: str, status: str = "reserved", fee: str | None = None, scope status=status, created_at=1, fee_record_id=fee) # type: ignore[arg-type] -def test_acquire_is_all_or_nothing_and_assigns_ids() -> None: - store = MemoryRecordInventoryStore() +@pytest.fixture(params=["memory", "json", "sqlite"]) +def make_store(request: pytest.FixtureRequest, tmp_path: Path) -> Factory: + """A factory returning a store over the same backing state each call (a reopen for the durable ones).""" + kind = request.param + memory = MemoryRecordInventoryStore() + if kind == "memory": + return lambda: memory + if kind == "json": + return lambda: JsonFileRecordInventoryStore(tmp_path / "inventory.json") + return lambda: SqliteRecordInventoryStore(tmp_path / "inventory.sqlite") + + +def test_acquire_is_all_or_nothing_and_assigns_ids(make_store: Factory) -> None: + store = make_store() first = store.acquire(entry("1group", "2group")) assert first is not None and first.id == "1" assert store.acquire(entry("2group", "3group")) is None # overlaps a held input - assert store.acquire(entry("3group")) is not None + second = store.acquire(entry("3group")) + assert second is not None and second.id == "2" with pytest.raises(ValueError, match="Duplicate"): store.acquire(entry("4group", "4group")) assert [e.records for e in store.list(SCOPE)] == [("1group", "2group"), ("3group",)] assert store.list('["mainnet","aleo1owner"]') == [] -def test_list_and_acquire_return_copies() -> None: - store = MemoryRecordInventoryStore() +def test_list_and_acquire_return_copies(make_store: Factory) -> None: + store = make_store() saved = store.acquire(entry("1group")) assert saved is not None saved.status = "cancelled" # mutating the copy changes nothing assert store.list(SCOPE)[0].status == "reserved" -def test_update_patches_fields_and_clears_transactions() -> None: - store = MemoryRecordInventoryStore() +def test_update_patches_fields_and_clears_transactions(make_store: Factory) -> None: + store = make_store() saved = store.acquire(entry("1group")) assert saved is not None store.update(SCOPE, saved.id, {"status": "prepared", "transaction_id": "at1x", "transaction": {"id": "at1x"}, "fee_microcredits": "100"}) - stored = store.list(SCOPE)[0] + stored = make_store().list(SCOPE)[0] assert (stored.status, stored.transaction_id, stored.transaction, stored.fee_microcredits) == ("prepared", "at1x", {"id": "at1x"}, "100") store.update(SCOPE, saved.id, {"status": "confirmed", "transaction": None}) - assert store.list(SCOPE)[0].transaction is None + assert make_store().list(SCOPE)[0].transaction is None with pytest.raises(KeyError): store.update(SCOPE, "99", {"status": "cancelled"}) -def test_held_records_follow_status_and_fee_records() -> None: - assert holds_records(entry("1group", status="confirmed")) is True - assert holds_records(entry("1group", status="cancelled")) is False - assert holds_records(entry("1group", status="rejected")) is False - rejected_with_fee = entry("1group", "2group", status="rejected", fee="2group") - assert holds_records(rejected_with_fee) is True - assert held_record_ids(rejected_with_fee) == ("2group",) - assert held_record_ids(entry("1group", "2group", status="submitted", fee="2group")) == ("1group", "2group") - - -def test_rejected_entries_release_inputs_but_keep_the_charged_fee_record() -> None: - store = MemoryRecordInventoryStore() +def test_rejected_entries_release_inputs_but_keep_the_charged_fee_record(make_store: Factory) -> None: + store = make_store() saved = store.acquire(entry("1group", "2group", fee="2group")) assert saved is not None store.update(SCOPE, saved.id, {"status": "rejected"}) @@ -74,13 +90,13 @@ def test_rejected_entries_release_inputs_but_keep_the_charged_fee_record() -> No assert reserved_record_ids(None, SCOPE) == set() -def test_concurrent_acquisitions_never_share_an_input() -> None: - store = MemoryRecordInventoryStore() +def test_concurrent_acquisitions_never_share_an_input(make_store: Factory) -> None: barrier = threading.Barrier(8) results: list[RecordSpend | None] = [] lock = threading.Lock() def worker() -> None: + store = make_store() # separate handles, one backing journal barrier.wait() result = store.acquire(entry("1group")) with lock: @@ -92,6 +108,17 @@ def worker() -> None: for thread in threads: thread.join() assert sum(result is not None for result in results) == 1 + assert len(make_store().list(SCOPE)) == 1 + + +def test_held_records_follow_status_and_fee_records() -> None: + assert holds_records(entry("1group", status="confirmed")) is True + assert holds_records(entry("1group", status="cancelled")) is False + assert holds_records(entry("1group", status="rejected")) is False + rejected_with_fee = entry("1group", "2group", status="rejected", fee="2group") + assert holds_records(rejected_with_fee) is True + assert held_record_ids(rejected_with_fee) == ("2group",) + assert held_record_ids(entry("1group", "2group", status="submitted", fee="2group")) == ("1group", "2group") def test_entries_round_trip_with_veil_field_names() -> None: @@ -101,3 +128,87 @@ def test_entries_round_trip_with_veil_field_names() -> None: data = saved.to_dict() assert data["createdAt"] == 5 and data["transactionId"] == "at1x" and "assetProgram" not in data assert RecordSpend.from_dict(data) == saved + + +# ── Durable stores ──────────────────────────────────────────────────────────── + +@pytest.fixture(params=["json", "sqlite"]) +def durable(request: pytest.FixtureRequest, tmp_path: Path) -> tuple[Factory, Path]: + if request.param == "json": + path = tmp_path / "journal" / "inventory.json" + return (lambda: JsonFileRecordInventoryStore(path)), path + path = tmp_path / "journal" / "inventory.sqlite" + return (lambda: SqliteRecordInventoryStore(path)), path + + +def test_durable_stores_create_private_files_and_survive_reopen(durable: tuple[Factory, Path]) -> None: + make, path = durable + store = make() + assert path.exists() + if sys.platform != "win32": + assert (path.stat().st_mode & 0o077) == 0 and (path.parent.stat().st_mode & 0o077) == 0 + saved = store.acquire(entry("1group")) + assert saved is not None + store.update(SCOPE, saved.id, {"status": "submitted", "transaction_id": "at1x", "transaction": {"id": "at1x"}}) + reopened = make().list(SCOPE) + assert len(reopened) == 1 and reopened[0].status == "submitted" and reopened[0].transaction == {"id": "at1x"} + assert make().acquire(entry("1group")) is None # still held after reopening + + +def test_durable_stores_release_reservations_of_dead_local_processes(durable: tuple[Factory, Path]) -> None: + make, _ = durable + writer = make() + writer._pid = dead_pid() # pyright: ignore[reportPrivateUsage] + held = writer.acquire(entry("1group")) + assert held is not None + live = make() + live._pid = os.getpid() # pyright: ignore[reportPrivateUsage] + prepared = live.acquire(entry("2group")) + assert prepared is not None + live.update(SCOPE, prepared.id, {"status": "prepared", "transaction_id": "at1y", "transaction": {"id": "at1y"}}) + journal = make().list(SCOPE) + # The dead writer's reserved entry never reached the network; a prepared + # entry from a dead writer would be kept, and anything from a live one is. + assert [(e.records, e.status) for e in journal] == [(("1group",), "cancelled"), (("2group",), "prepared")] + assert make().acquire(entry("1group")) is not None + + +def dead_pid() -> int: + """The pid of a process that has already exited.""" + process = subprocess.Popen([sys.executable, "-c", "pass"]) + process.wait() + return process.pid + + +def test_sqlite_layout_matches_veil(tmp_path: Path) -> None: + path = tmp_path / "inventory.sqlite" + store = SqliteRecordInventoryStore(path) + saved = store.acquire(entry("1group", "2group")) + assert saved is not None + store.update(SCOPE, saved.id, {"status": "prepared", "transaction_id": "at1x", "fee_microcredits": "100"}) + store.close() + rows = sqlite3.connect(str(path)).execute("SELECT id, scope, data, pid, host FROM record_spends").fetchall() + assert len(rows) == 1 + row_id, scope, data, pid, host = rows[0] + assert (row_id, scope, pid, bool(host)) == (1, SCOPE, os.getpid(), True) + payload = json.loads(data) + assert payload["id"] == "1" and payload["records"] == ["1group", "2group"] + assert payload["status"] == "prepared" and payload["transactionId"] == "at1x" and payload["feeMicrocredits"] == "100" + assert payload["createdAt"] == 1 and payload["accountType"] == "local" + # A Veil-written row is read back as-is. + with sqlite3.connect(str(path)) as raw: + veil = {"id": "2", "scope": SCOPE, "records": ["9group"], "program": "credits.aleo", "function": "split", + "status": "submitted", "createdAt": 7, "accountType": "local", "transactionId": "at1veil"} + raw.execute("INSERT INTO record_spends(scope, data, pid, host) VALUES (?, ?, ?, ?)", (SCOPE, json.dumps(veil), 1, "elsewhere")) + with SqliteRecordInventoryStore(path) as again: + entries = again.list(SCOPE) + assert entries[1].records == ("9group",) and entries[1].transaction_id == "at1veil" + + +def test_json_file_is_plain_json_with_writer_metadata(tmp_path: Path) -> None: + path = tmp_path / "inventory.json" + store = JsonFileRecordInventoryStore(path) + saved = store.acquire(entry("1group")) + assert saved is not None + items = json.loads(path.read_text()) + assert items[0]["data"]["records"] == ["1group"] and items[0]["pid"] == os.getpid() and items[0]["host"] diff --git a/shield-swap-sdk/examples/concurrent-swaps/README.md b/shield-swap-sdk/examples/concurrent-swaps/README.md index d4e4c662..a6194d16 100644 --- a/shield-swap-sdk/examples/concurrent-swaps/README.md +++ b/shield-swap-sdk/examples/concurrent-swaps/README.md @@ -40,18 +40,21 @@ submission. Trades and inventory maintenance then share one journal. ```python from aleo import Aleo, HTTPProvider -from aleo.inventory import RecordManagementConfig +from aleo.inventory import RecordManagementConfig, SqliteRecordInventoryStore -store = JsonFileRecordInventoryStore(directory / "inventory.json") +store = SqliteRecordInventoryStore(directory / "inventory.sqlite") aleo = Aleo(HTTPProvider("https://edge.provable.com/api", network="testnet"), record_management=RecordManagementConfig(enabled=True, store=store, chain_id="aleo:testnet")) ``` The SDK's default store is in memory and lasts for one process. The example -defines `JsonFileRecordInventoryStore`, a file-backed store under a file lock, -to show what a durable backend provides: reservations and proved transactions -that survive a restart. A database with a uniqueness constraint on held record -nonces gives the same guarantees across machines. +uses the SDK's SQLite store so reservations and proved transactions survive a +restart and several processes on the machine share one journal; its layout +matches Veil's store, so a Veil client can read the same file. +`JsonFileRecordInventoryStore` is the alternative when a journal that can be +read in an editor matters more than throughput. A custom store, such as a +database with a uniqueness constraint on held record nonces, extends the same +guarantees across machines. ## 2. Refuse to start over unresolved work @@ -120,7 +123,7 @@ Reuse it for the same account. | File | Purpose | | --- | --- | -| `inventory.json` | Record reservations and proved transactions awaiting broadcast, for reconciliation. | +| `inventory.sqlite` | Record reservations and proved transactions awaiting broadcast, for reconciliation. | | `swaps.jsonl` | Blinded identities and swap handles needed to claim outputs. Treat it as key material. | | `batch.pending` | Remains after an interrupted batch so a later run cannot repeat it unnoticed. | diff --git a/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py b/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py index fe8a5a10..b50d9ed3 100644 --- a/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py +++ b/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py @@ -17,21 +17,15 @@ from __future__ import annotations import argparse -import fcntl -import json import os import sys -import threading from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from decimal import Decimal from pathlib import Path -from typing import Any from aleo import Aleo, HTTPProvider, testnet -from aleo.inventory import ( - InventoryPlan, InventoryTarget, RecordManagementConfig, RecordSpend, RecordSpendPatch, held_record_ids, -) +from aleo.inventory import InventoryPlan, InventoryTarget, RecordManagementConfig, SqliteRecordInventoryStore from aleo_shield_swap import ShieldSwap from aleo_shield_swap.journal import Journal from aleo_shield_swap.types import ClaimResult, SwapHandle, SwapQuote @@ -41,86 +35,6 @@ UNRESOLVED = ("reserved", "prepared", "submitted") -class JsonFileRecordInventoryStore: - """A reservation journal in one JSON file, shared by processes on one machine. - - The SDK ships only an in-memory store. This class shows what a durable - backend needs: every mutation finishes before the method returns, and - ``acquire`` checks and inserts under one file lock so two processes can - never hold the same record. A database with a uniqueness constraint on - held nonces serves the same contract across machines. - - The file holds record nonces, program and function names, and proved - transactions awaiting broadcast; never keys or record plaintext. - """ - - def __init__(self, path: Path) -> None: - self.path = path - self.path.parent.mkdir(parents=True, exist_ok=True) - self._lock_path = path.with_suffix(path.suffix + ".lock") - self._thread_lock = threading.Lock() - - def _read(self) -> list[RecordSpend]: - if not self.path.exists(): - return [] - return [RecordSpend.from_dict(item) for item in json.loads(self.path.read_text() or "[]")] - - def _write(self, entries: list[RecordSpend]) -> None: - tmp = self.path.with_suffix(".tmp") - tmp.write_text(json.dumps([entry.to_dict() for entry in entries], indent=2)) - os.replace(tmp, self.path) - - def _locked(self) -> Any: - store = self - - class _Guard: - def __enter__(self) -> None: - store._thread_lock.acquire() - self.handle = open(store._lock_path, "a+") - fcntl.flock(self.handle, fcntl.LOCK_EX) - - def __exit__(self, *exc: object) -> None: - fcntl.flock(self.handle, fcntl.LOCK_UN) - self.handle.close() - store._thread_lock.release() - return _Guard() - - def list(self, scope: str) -> list[RecordSpend]: - with self._locked(): - return [entry for entry in self._read() if entry.scope == scope] - - def acquire(self, entry: RecordSpend) -> RecordSpend | None: - if len(set(entry.records)) != len(entry.records): - raise ValueError("Duplicate record inputs") - with self._locked(): - entries = self._read() - wanted = set(entry.records) - for old in entries: - if old.scope == entry.scope and wanted.intersection(held_record_ids(old)): - return None - saved = RecordSpend(**{**entry.__dict__, "id": str(len(entries) + 1)}) - entries.append(saved) - self._write(entries) - return saved - - def update(self, scope: str, entry_id: str, patch: RecordSpendPatch) -> None: - with self._locked(): - entries = self._read() - for entry in entries: - if entry.scope == scope and entry.id == entry_id: - if "status" in patch: - entry.status = patch["status"] - if "transaction_id" in patch: - entry.transaction_id = patch["transaction_id"] - if "transaction" in patch: - entry.transaction = patch["transaction"] - if "fee_microcredits" in patch: - entry.fee_microcredits = patch["fee_microcredits"] - self._write(entries) - return - raise KeyError("Unknown record reservation") - - @dataclass(frozen=True) class SwapOutcome: """One trade's result, kept separate so a failed sibling never hides a successful claim. @@ -165,7 +79,11 @@ def one(index: int) -> SwapOutcome: def build_client(private_key: str, directory: Path) -> tuple[Aleo, ShieldSwap]: """Bind an account, a durable reservation journal, and a swap journal to one client.""" - store = JsonFileRecordInventoryStore(directory / "inventory.json") + # The SDK's default journal is in memory and lasts one process. A SQLite + # journal keeps reservations and proved transactions across restarts and + # coordinates processes on this machine; JsonFileRecordInventoryStore is + # the inspectable alternative. + store = SqliteRecordInventoryStore(directory / "inventory.sqlite") aleo = Aleo(HTTPProvider(ENDPOINT, network=NETWORK), record_management=RecordManagementConfig(enabled=True, store=store, chain_id=f"aleo:{NETWORK}")) key = testnet.PrivateKey.from_string(private_key) @@ -185,7 +103,7 @@ def refuse_unresolved_work(aleo: Aleo, dex: ShieldSwap, directory: Path) -> None journal = aleo.inventory.reconcile_record_inventory() pending = [entry for entry in journal if entry.status in UNRESOLVED] if pending: - raise SystemExit(f"Reconcile {len(pending)} unresolved operations in {directory / 'inventory.json'} before another batch") + raise SystemExit(f"Reconcile {len(pending)} unresolved operations in {directory / 'inventory.sqlite'} before another batch") if dex.journal is not None and dex.journal.pending_claims(): raise SystemExit(f"Claim the pending swaps in {directory / 'swaps.jsonl'} (dex.collect_all()) before another batch") diff --git a/shield-swap-sdk/tests/integration/test_record_inventory_live.py b/shield-swap-sdk/tests/integration/test_record_inventory_live.py index 1487e09e..90b60367 100644 --- a/shield-swap-sdk/tests/integration/test_record_inventory_live.py +++ b/shield-swap-sdk/tests/integration/test_record_inventory_live.py @@ -16,17 +16,14 @@ """ from __future__ import annotations -import importlib.util import json import os -import sys import threading import time from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from decimal import Decimal from pathlib import Path -from types import ModuleType from typing import Any, Callable import pytest @@ -35,6 +32,7 @@ from aleo.facade.call import BoundCall from aleo.inventory import ( InventoryRecord, InventoryTarget, RecordAsset, RecordInventorySnapshot, RecordManagementConfig, RecordSpend, + SqliteRecordInventoryStore, ) from aleo_shield_swap import ShieldSwap from aleo_shield_swap._core import record_nonce @@ -47,7 +45,6 @@ WRITE_TIMEOUT = 600.0 # proving, confirmation, and scanner indexing for one write REBALANCE_TIMEOUT = 2400.0 # a bounded sequence of dependent joins and splits -EXAMPLE = Path(__file__).parents[2] / "examples" / "concurrent-swaps" / "concurrent_swaps.py" _failed = False @@ -62,15 +59,6 @@ def _abort_after_first_failure(request: pytest.FixtureRequest) -> Any: globals()["_failed"] = True -def _example() -> ModuleType: - spec = importlib.util.spec_from_file_location("concurrent_swaps_example_live", EXAMPLE) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - def poll(label: str, read: Callable[[], Any], ready: Callable[[Any], bool], timeout: float = WRITE_TIMEOUT) -> Any: """Wait for a chain or scanner predicate without resubmitting anything.""" deadline = time.monotonic() + timeout @@ -129,7 +117,6 @@ def __init__(self) -> None: self.directory = Path(os.environ.get("SHIELD_SWAP_INVENTORY_E2E_DIR") or Path(__file__).parents[2] / ".shield-swap" / "inventory-e2e" / "testnet" / self.address) self.directory.mkdir(parents=True, exist_ok=True) - self.example = _example() self.scope = f'["aleo:testnet","{self.address}"]' self.connect() unresolved = [e for e in self.aleo.inventory.reconcile_record_inventory() @@ -154,7 +141,10 @@ def __init__(self) -> None: def connect(self) -> None: """Bind a fresh client to the same account, journal file, and swap journal.""" - self.file_store = self.example.JsonFileRecordInventoryStore(self.directory / "inventory.json") + previous = getattr(self, "file_store", None) + if previous is not None: + previous.close() + self.file_store = SqliteRecordInventoryStore(self.directory / "inventory.sqlite") # The client holds this wrapper, so a test can interpose on acquisition # without rebuilding the client's frozen management settings. self.store = SwitchableStore(self.file_store) diff --git a/shield-swap-sdk/tests/test_concurrent_swaps_example.py b/shield-swap-sdk/tests/test_concurrent_swaps_example.py index 051e8a46..141c0611 100644 --- a/shield-swap-sdk/tests/test_concurrent_swaps_example.py +++ b/shield-swap-sdk/tests/test_concurrent_swaps_example.py @@ -1,4 +1,4 @@ -"""Offline checks of the concurrent-swaps example: batch orchestration and the file store. +"""Offline checks of the concurrent-swaps example: batch orchestration. _source: ProvableHQ/veil examples/shield-swap/concurrent-swaps.test.ts, PR #172 @ 2e324d5. Network, proving, and the DEX are replaced; the example's own code runs. @@ -14,7 +14,6 @@ import pytest -from aleo.inventory import RecordSpend from aleo_shield_swap.types import ClaimResult, SwapHandle EXAMPLE = Path(__file__).parents[1] / "examples" / "concurrent-swaps" / "concurrent_swaps.py" @@ -82,40 +81,3 @@ def test_batch_reports_every_trade_and_keeps_handles_of_failed_claims() -> None: assert claim_failed[0].handle is not None and claim_failed[0].handle.transaction_id == "at1swap3" assert len(claimed) == 2 and all(o.claim is not None and o.claim.amount_out == 5 for o in claimed) assert sorted(dex.claimed) == ["at1swap1", "at1swap3", "at1swap4"] # the failed claim was attempted, not skipped - - -def test_json_file_store_reserves_atomically_and_survives_reopen(tmp_path: Path) -> None: - example = load_example() - path = tmp_path / "inventory.json" - store = example.JsonFileRecordInventoryStore(path) - scope = '["aleo:testnet","aleo1owner"]' - - def entry(*records: str) -> RecordSpend: - return RecordSpend(id="", scope=scope, records=records, program="credits.aleo", function="join", - status="reserved", created_at=1) - - first = store.acquire(entry("1group", "2group")) - assert first is not None and first.id == "1" - assert store.acquire(entry("2group", "3group")) is None - store.update(scope, first.id, {"status": "prepared", "transaction_id": "at1x", "transaction": {"id": "at1x"}, "fee_microcredits": "100"}) - reopened = example.JsonFileRecordInventoryStore(path) - saved = reopened.list(scope)[0] - assert (saved.status, saved.transaction_id, saved.transaction, saved.fee_microcredits) == ("prepared", "at1x", {"id": "at1x"}, "100") - reopened.update(scope, first.id, {"status": "confirmed", "transaction": None}) - assert reopened.list(scope)[0].transaction is None - with pytest.raises(KeyError): - reopened.update(scope, "99", {"status": "cancelled"}) - - results: list[RecordSpend | None] = [] - barrier = threading.Barrier(6) - - def race() -> None: - barrier.wait() - results.append(example.JsonFileRecordInventoryStore(path).acquire(entry("9group"))) - - threads = [threading.Thread(target=race) for _ in range(6)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - assert sum(result is not None for result in results) == 1 From e2356dc14dbacb86581b9e510e7917d24d482fec Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 09:30:45 -0400 Subject: [PATCH 04/11] docs: flatten the concurrent-swaps example into one script The example now reads top to bottom like the first-swap example: client setup, the unresolved-work check, planning, execution, and the concurrent batch are inline, with only the per-trade thread body as a nested function. The offline test checks the script's flags and its refusal to run without a key instead of importing helpers. --- .../examples/concurrent-swaps/README.md | 16 +- .../concurrent-swaps/concurrent_swaps.py | 198 +++++++----------- .../tests/test_concurrent_swaps_example.py | 89 ++------ 3 files changed, 110 insertions(+), 193 deletions(-) diff --git a/shield-swap-sdk/examples/concurrent-swaps/README.md b/shield-swap-sdk/examples/concurrent-swaps/README.md index a6194d16..0f2339ca 100644 --- a/shield-swap-sdk/examples/concurrent-swaps/README.md +++ b/shield-swap-sdk/examples/concurrent-swaps/README.md @@ -109,12 +109,20 @@ An explicit `token_record` is never replaced. ```python quote = dex.quote(token_in="USDCx", token_out="ETH", amount_in="0.1", slippage_bps=50) -outcomes = submit_concurrent_swaps(dex, quote, count=6) + +def trade(index): + handle = dex.swap(quote).delegate(wait=False) + claim = dex.claim_swap_output(handle, timeout=600.0).delegate(wait=True) + return index, handle, claim + +with ThreadPoolExecutor(max_workers=6) as pool: + outcomes = list(pool.map(trade, range(1, 7))) ``` -`submit_concurrent_swaps` runs the swaps on a thread pool, waits for each -successful swap's output, and claims it. One failed trade never discards a -sibling's handle, and a failed claim keeps the handle for a later attempt. +The script runs the swaps on a thread pool, waits for each successful swap's +output, and claims it. In the script each trade catches its own errors, so one +failed trade never discards a sibling's handle, and a failed claim keeps the +handle for a later attempt. ## Keep recovery data diff --git a/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py b/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py index b50d9ed3..8fb9068f 100644 --- a/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py +++ b/shield-swap-sdk/examples/concurrent-swaps/concurrent_swaps.py @@ -14,138 +14,86 @@ through the delegated prover's fee master. See README.md for the recovery steps after an interruption. """ -from __future__ import annotations - import argparse import os import sys from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass from decimal import Decimal from pathlib import Path from aleo import Aleo, HTTPProvider, testnet -from aleo.inventory import InventoryPlan, InventoryTarget, RecordManagementConfig, SqliteRecordInventoryStore -from aleo_shield_swap import ShieldSwap -from aleo_shield_swap.journal import Journal -from aleo_shield_swap.types import ClaimResult, SwapHandle, SwapQuote - -NETWORK = "testnet" -ENDPOINT = "https://edge.provable.com/api" -UNRESOLVED = ("reserved", "prepared", "submitted") - - -@dataclass(frozen=True) -class SwapOutcome: - """One trade's result, kept separate so a failed sibling never hides a successful claim. - - Attributes: - index: Position in the batch, from 1. - handle: The swap handle once submission succeeded; it is what a later - claim needs, so keep it when ``claim`` is ``None``. - claim: The confirmed claim with the output and any refund. - error: Why the swap or the claim failed. - """ - - index: int - handle: SwapHandle | None = None - claim: ClaimResult | None = None - error: BaseException | None = None - - -def submit_concurrent_swaps(dex: ShieldSwap, quote: SwapQuote, count: int, *, claim_timeout: float = 600.0) -> list[SwapOutcome]: - """Submit *count* swaps from one quote at the same time, then claim each successful output. - - Each swap selects its own input record; the facade's record management - reserves it before proving, and a swap whose record another swap took - first moves to the next eligible record. A swap and its claim are two - transactions, so each claim waits for the swap output to appear before - submitting. Nothing is retried after an uncertain submission. - """ - def one(index: int) -> SwapOutcome: - try: - handle = dex.swap(quote).delegate(wait=False) - except BaseException as exc: # noqa: BLE001 - reported per trade, never raised mid-batch - return SwapOutcome(index, error=exc) - try: - claim = dex.claim_swap_output(handle, timeout=claim_timeout).delegate(wait=True) - return SwapOutcome(index, handle=handle, claim=claim) - except BaseException as exc: # noqa: BLE001 - the handle is kept for a later claim - return SwapOutcome(index, handle=handle, error=exc) +from aleo.inventory import InventoryTarget, RecordManagementConfig, SqliteRecordInventoryStore +from aleo_shield_swap import Journal, ShieldSwap - with ThreadPoolExecutor(max_workers=count) as pool: - return list(pool.map(one, range(1, count + 1))) +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Prepare records and run concurrent Shield Swap trades on testnet.") + parser.add_argument("--execute", action="store_true", help="prepare records, submit swaps, and claim outputs") + parser.add_argument("--count", type=int, default=6, help="trades to run at once and records to prepare (2-16)") + parser.add_argument("--amount", default="0.1", help="input amount per trade in token units") + args = parser.parse_args() + token_in, token_out, amount_in, count = "USDCx", "ETH", args.amount, args.count + if not 2 <= count <= 16: + raise SystemExit("--count must be from 2 through 16") -def build_client(private_key: str, directory: Path) -> tuple[Aleo, ShieldSwap]: - """Bind an account, a durable reservation journal, and a swap journal to one client.""" - # The SDK's default journal is in memory and lasts one process. A SQLite - # journal keeps reservations and proved transactions across restarts and - # coordinates processes on this machine; JsonFileRecordInventoryStore is - # the inspectable alternative. - store = SqliteRecordInventoryStore(directory / "inventory.sqlite") - aleo = Aleo(HTTPProvider(ENDPOINT, network=NETWORK), - record_management=RecordManagementConfig(enabled=True, store=store, chain_id=f"aleo:{NETWORK}")) + ### STEP 1. Bind the account, a durable reservation journal, and a swap journal to one client. ### + ### Shield Swap trades spend private records. Record management on the Aleo client reserves each + ### trade's record before proving so trades submitted together never pick the same one. + private_key = os.environ.get("SHIELD_SWAP_PRIVATE_KEY") + if not private_key: + raise SystemExit("Set SHIELD_SWAP_PRIVATE_KEY to a funded testnet account with DEX access") key = testnet.PrivateKey.from_string(private_key) + directory = Path(".shield-swap") / "concurrent-swaps" / "testnet" / str(key.address) + directory.mkdir(parents=True, exist_ok=True) + + # The SDK's default journal is in memory and lasts one process. A SQLite journal keeps reservations + # and proved transactions across restarts and coordinates processes on this machine; + # JsonFileRecordInventoryStore is the inspectable alternative. + store = SqliteRecordInventoryStore(directory / "inventory.sqlite") + aleo = Aleo(HTTPProvider("https://edge.provable.com/api", network="testnet"), + record_management=RecordManagementConfig(enabled=True, store=store, chain_id="aleo:testnet")) account = aleo.account.from_private_key(key) aleo.default_account = account registration = aleo.records.register(account) # shares the view key with the scanner if not registration.get("ok"): - raise RuntimeError(f"Record scanner registration failed: {registration.get('error')}") + raise SystemExit(f"Record scanner registration failed: {registration.get('error')}") dex = ShieldSwap(aleo) dex.journal = Journal(directory / "swaps.jsonl") # blinded identities and claim handles dex.api.authenticate(str(account.address), lambda message: str(key.sign(message.encode()))) - return aleo, dex - -def refuse_unresolved_work(aleo: Aleo, dex: ShieldSwap, directory: Path) -> None: - """Stop before planning when an earlier run left reservations or claims unresolved.""" - journal = aleo.inventory.reconcile_record_inventory() - pending = [entry for entry in journal if entry.status in UNRESOLVED] + ### STEP 2. Refuse to start over unresolved work. ### + ### A transaction the node has not confirmed keeps its reservation; a missing confirmation + ### is not evidence that it failed. + pending = [entry for entry in aleo.inventory.reconcile_record_inventory() + if entry.status in ("reserved", "prepared", "submitted")] if pending: raise SystemExit(f"Reconcile {len(pending)} unresolved operations in {directory / 'inventory.sqlite'} before another batch") - if dex.journal is not None and dex.journal.pending_claims(): + if dex.journal.pending_claims(): raise SystemExit(f"Claim the pending swaps in {directory / 'swaps.jsonl'} (dex.collect_all()) before another batch") - -def describe_plan(plan: InventoryPlan) -> None: - print(f"{len(plan.steps)} maintenance operations; credits deduction {plan.deduction} microcredits") - for step in plan.steps: - helper = step.join_program.program if step.join_program else "native" - print(f" {step.kind:5} inputs={len(step.inputs):2} outputs={len(step.outputs)} via {helper}") - - -def concurrent_swaps(*, execute: bool, token_in: str = "USDCx", token_out: str = "ETH", - amount_in: str = "0.1", count: int = 6, directory: Path | None = None) -> list[SwapOutcome]: - """Preview or run inventory maintenance followed by a concurrent swap batch.""" - private_key = os.environ.get("SHIELD_SWAP_PRIVATE_KEY") - if not private_key: - raise SystemExit("Set SHIELD_SWAP_PRIVATE_KEY to a funded testnet account with DEX access") - if not 2 <= count <= 16: - raise SystemExit("count must be from 2 through 16") - address = str(testnet.PrivateKey.from_string(private_key).address) - directory = directory or Path(".shield-swap") / "concurrent-swaps" / NETWORK / address - directory.mkdir(parents=True, exist_ok=True) - - aleo, dex = build_client(private_key, directory) - refuse_unresolved_work(aleo, dex, directory) - + ### STEP 3. Plan the record layout. ### + ### One record per trade lets every swap start at once. Balanced lots with zero tolerance give + ### each trade the same spendable amount. Planning reads; it reserves and signs nothing. token = dex.api.get_token(token_in) minimum = int(Decimal(amount_in).scaleb(token.decimals)) - # One record per trade lets every swap start at once. Balanced lots with - # zero tolerance give each trade the same spendable amount. plan = dex.plan_inventory(token_in, InventoryTarget(records=count, min_record_amount=minimum, distribution="balanced", tolerance_bps=0), max_transactions=24) print(f"{count} concurrent {token_in} -> {token_out} swaps of {amount_in} {token_in}; recovery data in {directory}") - describe_plan(plan) + print(f"{len(plan.steps)} maintenance operations; credits deduction {plan.deduction} microcredits") + for step in plan.steps: + helper = step.join_program.program if step.join_program else "native" + print(f" {step.kind:5} inputs={len(step.inputs):2} outputs={len(step.outputs)} via {helper}") preview = dex.quote(token_in=token_in, token_out=token_out, amount_in=amount_in, slippage_bps=50) print(f"Per swap: estimated {preview.estimated_amount_out} {token_out}; minimum {preview.minimum_amount_out} {token_out}") - if not execute: + if not args.execute: print("Preview only. Pass --execute to prepare the records, submit the swaps, and claim the outputs.") - return [] + raise SystemExit(0) - # A marker survives a crash so a later run does not repeat an unfinished batch. + ### STEP 4. Execute the plan. ### + ### Each step reserves its inputs, proves, broadcasts, waits for confirmation, and waits until the + ### scanner shows the exact outputs before the next step spends them. A marker survives a crash so + ### a later run does not repeat an unfinished batch. marker = directory / "batch.pending" if marker.exists(): raise SystemExit(f"Resolve the previous batch before removing {marker}") @@ -154,29 +102,39 @@ def concurrent_swaps(*, execute: bool, token_in: str = "USDCx", token_out: str = if result.status != "complete": raise SystemExit(f"Maintenance interrupted: {result.reason}; keep {directory}") - # Quote after maintenance so joins and splits do not consume its lifetime. - # A quote fixes the output floor; it reserves no pool liquidity. + ### STEP 5. Submit the swaps together, then claim each output. ### + ### Quote after maintenance so joins and splits do not consume its lifetime. One quote fixes the + ### output floor for every trade; it reserves no pool liquidity, so a trade can still fail when the + ### price moves. Each swap selects its own record; a swap whose record another swap took first moves + ### to the next eligible record before proving. A swap and its claim are two transactions. quote = dex.quote(token_in=token_in, token_out=token_out, amount_in=amount_in, slippage_bps=50) - outcomes = submit_concurrent_swaps(dex, quote, count) - for outcome in outcomes: - if outcome.claim is not None and outcome.handle is not None: - print(f"{outcome.index}: {outcome.handle.transaction_id} -> {outcome.claim.transaction_id}; " - f"received {outcome.claim.amount_out} base units; refunded {outcome.claim.amount_remaining}") - elif outcome.handle is not None: - print(f"{outcome.index}: swap {outcome.handle.transaction_id} submitted; claim failed: {outcome.error}", file=sys.stderr) + + def trade(index): + """Swap, then claim. Each outcome is reported on its own so a failed sibling never hides a successful claim.""" + try: + handle = dex.swap(quote).delegate(wait=False) + except Exception as error: # reported per trade, never raised mid-batch + return index, None, None, error + try: + claim = dex.claim_swap_output(handle, timeout=600.0).delegate(wait=True) + return index, handle, claim, None + except Exception as error: # the handle is kept for a later claim + return index, handle, None, error + + with ThreadPoolExecutor(max_workers=count) as pool: + outcomes = list(pool.map(trade, range(1, count + 1))) + + failures = 0 + for index, handle, claim, error in outcomes: + if claim is not None and handle is not None: + print(f"{index}: {handle.transaction_id} -> {claim.transaction_id}; " + f"received {claim.amount_out} base units; refunded {claim.amount_remaining}") + elif handle is not None: + failures += 1 + print(f"{index}: swap {handle.transaction_id} submitted; claim failed: {error}", file=sys.stderr) else: - print(f"{outcome.index}: swap failed: {outcome.error}", file=sys.stderr) - failures = [outcome for outcome in outcomes if outcome.claim is None] + failures += 1 + print(f"{index}: swap failed: {error}", file=sys.stderr) if failures: - raise SystemExit(f"{len(failures)} trades need recovery in {directory}; do not repeat the batch") + raise SystemExit(f"{failures} trades need recovery in {directory}; do not repeat the batch") marker.unlink() - return outcomes - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Prepare records and run concurrent Shield Swap trades on testnet.") - parser.add_argument("--execute", action="store_true", help="prepare records, submit swaps, and claim outputs") - parser.add_argument("--count", type=int, default=6, help="trades to run at once and records to prepare (2-16)") - parser.add_argument("--amount", default="0.1", help="input amount per trade in token units") - args = parser.parse_args() - concurrent_swaps(execute=args.execute, count=args.count, amount_in=args.amount) diff --git a/shield-swap-sdk/tests/test_concurrent_swaps_example.py b/shield-swap-sdk/tests/test_concurrent_swaps_example.py index 141c0611..1ed47228 100644 --- a/shield-swap-sdk/tests/test_concurrent_swaps_example.py +++ b/shield-swap-sdk/tests/test_concurrent_swaps_example.py @@ -1,83 +1,34 @@ -"""Offline checks of the concurrent-swaps example: batch orchestration. - -_source: ProvableHQ/veil examples/shield-swap/concurrent-swaps.test.ts, PR #172 @ 2e324d5. -Network, proving, and the DEX are replaced; the example's own code runs. -""" +"""Offline checks of the concurrent-swaps example: it parses, documents its flags, and refuses to run without a key.""" from __future__ import annotations -import importlib.util +import os +import subprocess import sys -import threading from pathlib import Path -from types import ModuleType -from typing import Any - -import pytest - -from aleo_shield_swap.types import ClaimResult, SwapHandle EXAMPLE = Path(__file__).parents[1] / "examples" / "concurrent-swaps" / "concurrent_swaps.py" -def load_example() -> ModuleType: - spec = importlib.util.spec_from_file_location("concurrent_swaps_example", EXAMPLE) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module # dataclasses resolve annotations through sys.modules - spec.loader.exec_module(module) - return module - - -class _Call: - def __init__(self, result: Any) -> None: - self._result = result - - def delegate(self, **kw: Any) -> Any: - if isinstance(self._result, BaseException): - raise self._result - return self._result - +def run(*args: str, key: str | None = None) -> subprocess.CompletedProcess[str]: + env = {k: v for k, v in os.environ.items() if k != "SHIELD_SWAP_PRIVATE_KEY"} + if key is not None: + env["SHIELD_SWAP_PRIVATE_KEY"] = key + env["PYTHONPATH"] = str(Path(__file__).parents[1] / "python") + return subprocess.run([sys.executable, str(EXAMPLE), *args], capture_output=True, text=True, env=env, timeout=120) -class FakeDex: - """Swaps succeed or fail per index; claims follow a second table.""" - def __init__(self, swap_errors: dict[int, BaseException] | None = None, claim_errors: dict[int, BaseException] | None = None) -> None: - self.swap_errors = swap_errors or {} - self.claim_errors = claim_errors or {} - self.swaps = 0 - self.claimed: list[str] = [] - self._lock = threading.Lock() +def test_help_documents_the_execute_opt_in() -> None: + result = run("--help") + assert result.returncode == 0 + assert "--execute" in result.stdout and "--count" in result.stdout and "--amount" in result.stdout - def swap(self, quote: Any) -> _Call: - with self._lock: - self.swaps += 1 - index = self.swaps - if index in self.swap_errors: - return _Call(self.swap_errors[index]) - return _Call(SwapHandle(swap_id=f"{index}field", blinding_factor="1field", blinded_address="aleo1x", - token_in_id="1field", token_out_id="2field", pool_key="5field", amount_in=10, - transaction_id=f"at1swap{index}", program="shield_swap.aleo")) - def claim_swap_output(self, handle: SwapHandle, *, timeout: float = 0.0) -> _Call: - index = int(handle.transaction_id.removeprefix("at1swap")) - self.claimed.append(handle.transaction_id) - if index in self.claim_errors: - return _Call(self.claim_errors[index]) - return _Call(ClaimResult(transaction_id=f"at1claim{index}", amount_out=5, amount_remaining=0)) +def test_refuses_to_run_without_a_key_before_touching_the_network() -> None: + result = run() + assert result.returncode != 0 + assert "SHIELD_SWAP_PRIVATE_KEY" in result.stderr -def test_batch_reports_every_trade_and_keeps_handles_of_failed_claims() -> None: - example = load_example() - dex = FakeDex(swap_errors={2: RuntimeError("no record")}, claim_errors={3: TimeoutError("output not finalized")}) - outcomes = example.submit_concurrent_swaps(dex, quote=object(), count=4, claim_timeout=1.0) - # The pool runs trades concurrently, so which batch slot hits which fake - # failure is not fixed; the shape of the report is. - assert [o.index for o in outcomes] == [1, 2, 3, 4] - swap_failed = [o for o in outcomes if o.handle is None] - claim_failed = [o for o in outcomes if o.handle is not None and o.claim is None] - claimed = [o for o in outcomes if o.claim is not None] - assert len(swap_failed) == 1 and isinstance(swap_failed[0].error, RuntimeError) - assert len(claim_failed) == 1 and isinstance(claim_failed[0].error, TimeoutError) - assert claim_failed[0].handle is not None and claim_failed[0].handle.transaction_id == "at1swap3" - assert len(claimed) == 2 and all(o.claim is not None and o.claim.amount_out == 5 for o in claimed) - assert sorted(dex.claimed) == ["at1swap1", "at1swap3", "at1swap4"] # the failed claim was attempted, not skipped +def test_rejects_an_out_of_range_count_before_touching_the_network() -> None: + result = run("--count", "1", key="unused") + assert result.returncode != 0 and "--count must be from 2 through 16" in result.stderr From d8a0c8e32fdf0673c2ed6c5c1220f2b3b4302004 Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 09:35:29 -0400 Subject: [PATCH 05/11] fix: keep the JSON journal owner-only across rewrites The atomic rewrite created its temporary file with the default umask and then replaced the 0600 journal with it. Create the temporary file with mode 0600 and assert the mode after a write in the durable store test. --- sdk/python/aleo/inventory/store.py | 5 ++++- sdk/python/tests/test_inventory_store.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sdk/python/aleo/inventory/store.py b/sdk/python/aleo/inventory/store.py index 30f38a1a..d4ffc248 100644 --- a/sdk/python/aleo/inventory/store.py +++ b/sdk/python/aleo/inventory/store.py @@ -316,8 +316,11 @@ def __enter__(self) -> None: def __exit__(self, kind: object, exc: object, tb: object) -> None: try: if exc is None and store._cache is not None: + # Write a private temporary file, then replace, so the + # journal is never partially written and keeps mode 0600. tmp = store.path.with_suffix(store.path.suffix + ".tmp") - with open(tmp, "w") as handle: + descriptor = os.open(tmp, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "w") as handle: json.dump(store._cache, handle, indent=2) handle.flush() os.fsync(handle.fileno()) diff --git a/sdk/python/tests/test_inventory_store.py b/sdk/python/tests/test_inventory_store.py index 83aabd7a..6bdb2d25 100644 --- a/sdk/python/tests/test_inventory_store.py +++ b/sdk/python/tests/test_inventory_store.py @@ -150,6 +150,8 @@ def test_durable_stores_create_private_files_and_survive_reopen(durable: tuple[F saved = store.acquire(entry("1group")) assert saved is not None store.update(SCOPE, saved.id, {"status": "submitted", "transaction_id": "at1x", "transaction": {"id": "at1x"}}) + if sys.platform != "win32": + assert (path.stat().st_mode & 0o077) == 0, "writes keep the journal owner-only" reopened = make().list(SCOPE) assert len(reopened) == 1 and reopened[0].status == "submitted" and reopened[0].transaction == {"id": "at1x"} assert make().acquire(entry("1group")) is None # still held after reopening From 0fe8ddb36074924794e4cac3ebbc330ba8621b59 Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 09:56:21 -0400 Subject: [PATCH 06/11] refactor: manage the build and submit halves instead of a combined verb On a client with record management enabled, build_transaction now reserves the call's record inputs before proving and saves the proved transaction to the journal, and submit_transaction records the broadcast (a 4xx refusal releases the reservation, an unknown outcome keeps it). transact is the two in sequence again, matching how snarkVM separates building a transaction from broadcasting it, and transact_with_result is removed. A built transaction that will not be sent is released with result.discard(). Shield Swap's DexCall returns to the plain build-then-submit split and is protected without changes. The checkpoint rules move into one SpendCheckpoint class shared by the engine and the facade. --- README.md | 12 +- sdk/Readme.md | 2 +- sdk/python/aleo/facade/async_client.py | 113 +++---- sdk/python/aleo/facade/call.py | 135 +++++---- sdk/python/aleo/facade/client.py | 13 +- sdk/python/aleo/facade/network.py | 26 +- sdk/python/aleo/inventory/_engine.py | 280 ++++++++++++------ sdk/python/aleo/inventory/client.py | 129 ++++---- sdk/python/tests/test_facade_inventory.py | 95 ++++-- .../python/aleo_shield_swap/_calls.py | 21 +- .../python/aleo_shield_swap/async_client.py | 5 +- shield-swap-sdk/tests/test_inventory.py | 42 ++- 12 files changed, 543 insertions(+), 330 deletions(-) diff --git a/README.md b/README.md index 35907918..b6a9894e 100644 --- a/README.md +++ b/README.md @@ -229,10 +229,14 @@ reads an asset's spendable records, plans the joins and splits that reach a chosen layout, executes that plan one confirmed transaction at a time, and keeps writers that share a journal from spending the same record. -Build the client with record management enabled so every `transact` and -`delegate` reserves its record inputs before proving and saves the proved -transaction before broadcast. The example below reshapes the account's private -credits into four equal records of at least one credit each: +Build the client with record management enabled and the two halves of a +write each do their part of the journal work, the way snarkVM separates +building a transaction from broadcasting it: `build_transaction` reserves the +call's record inputs before proving and saves the proved transaction, and +`submit_transaction` records the broadcast. `transact` is the two in sequence; +`delegate` does the same through the prover. A built transaction that will not +be sent is released with `result.discard()`. The example below reshapes the +account's private credits into four equal records of at least one credit each: ```python from aleo import Aleo, HTTPProvider diff --git a/sdk/Readme.md b/sdk/Readme.md index aace801b..be8d2fa4 100644 --- a/sdk/Readme.md +++ b/sdk/Readme.md @@ -166,7 +166,7 @@ credits.functions.transfer_private(record, str(recipient.address), 1) \ ### Record inventory — join, split, plan, reserve, recover -One transaction spends whole records, so a large payment needs one record that covers it and several payments submitted together need one record each. `aleo.inventory` reads an asset's spendable records, plans the joins and splits that reach a target layout, and executes that plan one confirmed transaction at a time. Build the client with `record_management` enabled to make every `transact` and `delegate` reserve its record inputs before proving and save the proved transaction before broadcast; writers sharing the same store never spend the same record. +One transaction spends whole records, so a large payment needs one record that covers it and several payments submitted together need one record each. `aleo.inventory` reads an asset's spendable records, plans the joins and splits that reach a target layout, and executes that plan one confirmed transaction at a time. Build the client with `record_management` enabled and the journal follows the two halves of a write: `build_transaction` reserves the record inputs before proving and saves the proved transaction, `submit_transaction` records the broadcast (`transact` is both in sequence, `delegate` does the same through the prover), and `result.discard()` releases a built transaction that will not be sent. Writers sharing the same store never spend the same record. ```python from aleo.inventory import InventoryTarget, RecordAsset, RecordManagementConfig diff --git a/sdk/python/aleo/facade/async_client.py b/sdk/python/aleo/facade/async_client.py index ec63bc0a..f0ffe37f 100644 --- a/sdk/python/aleo/facade/async_client.py +++ b/sdk/python/aleo/facade/async_client.py @@ -682,20 +682,27 @@ async def get_transactions(self, block_height: int) -> list[Any]: async def submit_transaction(self, transaction: Any) -> str: """Broadcast *transaction* to the network and return the transaction ID. - Accepts either a :class:`Transaction` object (anything with a - ``__str__`` serialisation the node accepts) or a raw JSON string. - - Parameters - ---------- - transaction: - A :class:`Transaction` object or a serialised transaction string. + Accepts a :class:`~aleo.facade.call.TransactionResult`, a + :class:`Transaction` object, or a raw JSON string. A result built on + a client with record management enabled carries its journal + checkpoint; see :meth:`~aleo.facade.network.NetworkModule.submit_transaction`. Returns ------- str The transaction ID returned by the node. """ - return str(await self._nc().submit_transaction(transaction)) + checkpoint = getattr(transaction, "checkpoint", None) + raw = getattr(transaction, "raw", transaction) + try: + tx_id = str(await self._nc().submit_transaction(raw)) + except BaseException as exc: + if checkpoint is not None: + checkpoint.broadcast_failed(exc) + raise + if checkpoint is not None: + checkpoint.submitted(tx_id) + return tx_id send_raw_transaction = submit_transaction @@ -840,7 +847,41 @@ async def build_transaction( """Run the full prove ladder and return an assembled transaction (async). Process ops run inside ``asyncio.to_thread``; network calls are awaited. + On a client built with record management enabled, the record inputs + are reserved before proving and the proved transaction is saved to the + journal; see :meth:`~aleo.facade.call.BoundCall.build_transaction`. """ + from .call import _managed_inventory, _fee_record_text, _transaction_json, _after_acquisition # pyright: ignore[reportPrivateUsage] + managed = _managed_inventory(self._client) + if managed is None: + return TransactionResult(await self._prove_transaction( + account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, + )) + checkpoint = await managed.begin_spend( + self.program_id, self.function_name, self.args, account=account, + private_fee=private_fee or fee_record is not None, fee_record=_fee_record_text(fee_record), + ) + try: + tx = await self._prove_transaction( + account, priority_fee=priority_fee, fee_record=checkpoint.fee_record_selector or fee_record, + private_fee=private_fee, base_fee=base_fee, + ) + checkpoint.prepared(str(tx.id), _transaction_json(tx)) + except BaseException as exc: + checkpoint.discard() + raise _after_acquisition(exc) from exc + return TransactionResult(tx, checkpoint) + + async def _prove_transaction( + self, + account: Any = None, + *, + priority_fee: int = 0, + fee_record: Any = None, + private_fee: bool = False, + base_fee: int | None = None, + ) -> Any: + """Authorize, execute, prove, and assemble the ``Transaction``; no journal, no broadcast.""" net = self._net() locator = self._locator query_url = self._query_url @@ -873,13 +914,12 @@ def _prove_execution() -> Any: base_fee=base_fee, ) - def _prove_fee_and_assemble() -> TransactionResult: + def _prove_fee_and_assemble() -> Any: try: _, fee_trace = process.execute(fee_auth) fee_trace.prepare(net.Query.rest(query_url)) fee = fee_trace.prove_fee(block_height=height) - tx = net.Transaction.from_execution(execution, fee) - return TransactionResult(tx) + return net.Transaction.from_execution(execution, fee) except Exception as exc: raise ExecutionError( f"Failed to prove fee / assemble transaction for {locator}: {exc}", @@ -901,49 +941,18 @@ async def transact( ) -> str: """Build and broadcast; return the transaction id (async). - On a client built with record management enabled, record inputs are - reserved before proving and the proved transaction is saved before - broadcast; see :meth:`~aleo.facade.call.BoundCall.transact`. - """ - tx_id, _ = await self.transact_with_result( - account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, - ) - return tx_id - - async def transact_with_result( - self, - account: Any = None, - *, - priority_fee: int = 0, - fee_record: Any = None, - private_fee: bool = False, - base_fee: int | None = None, - ) -> tuple[str, TransactionResult]: - """Build, broadcast, and also return the built transaction (async). - - See :meth:`~aleo.facade.call.BoundCall.transact_with_result`. + On a client built with record management enabled this is the two + managed halves in sequence: the build reserves the record inputs and + saves the proved transaction, the broadcast records the submission. """ - from .call import _managed_inventory, _fee_record_text, _prepared_from_result # pyright: ignore[reportPrivateUsage] - managed = _managed_inventory(self._client) - if managed is None: - result = await self.build_transaction( - account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, - ) - return await self._client.network.submit_transaction(result.raw), result - built: list[TransactionResult] = [] - - async def build(request: Any) -> Any: - result = await self.build_transaction( - account, priority_fee=priority_fee, fee_record=request.fee_record, - private_fee=request.private_fee, base_fee=base_fee, - ) - built.append(result) - return _prepared_from_result(result) - tx_id = await managed.managed_write( - self.program_id, self.function_name, self.args, account=account, - private_fee=private_fee or fee_record is not None, fee_record=_fee_record_text(fee_record), builder=build, + result = await self.build_transaction( + account, + priority_fee=priority_fee, + fee_record=fee_record, + private_fee=private_fee, + base_fee=base_fee, ) - return tx_id, built[0] + return await self._client.network.submit_transaction(result) async def delegate( self, diff --git a/sdk/python/aleo/facade/call.py b/sdk/python/aleo/facade/call.py index 61b97f00..05ed31f8 100644 --- a/sdk/python/aleo/facade/call.py +++ b/sdk/python/aleo/facade/call.py @@ -110,18 +110,33 @@ class TransactionResult: ---------- raw: The underlying network :class:`Transaction` (escape hatch). + checkpoint: + On a client with record management enabled, the journal entry that + reserved this transaction's record inputs and saved the proved + transaction. ``aleo.network.submit_transaction(result)`` records the + broadcast against it; ``None`` on an unmanaged client. """ - __slots__ = ("_raw",) + __slots__ = ("_raw", "checkpoint") - def __init__(self, raw: Any) -> None: + def __init__(self, raw: Any, checkpoint: Any = None) -> None: self._raw = raw + self.checkpoint = checkpoint @property def raw(self) -> Any: """The underlying network ``Transaction`` object (escape hatch).""" return self._raw + def discard(self) -> None: + """Release the record reservation of a built transaction that will not be submitted. + + Only meaningful on a managed client; after ``submit_transaction`` it + does nothing, because the network may have spent the records. + """ + if self.checkpoint is not None: + self.checkpoint.discard() + @property def id(self) -> str: """The transaction id string.""" @@ -400,11 +415,51 @@ def build_transaction( private_fee: Force a private fee sourced from ``aleo.record_provider``. + On a client built with record management enabled, the call's record + inputs are reserved before proving and the proved transaction is + saved to the journal before this returns; a private fee record is + chosen and reserved once the exact fee is known. Submit the result + with ``aleo.network.submit_transaction(result)`` to record the + broadcast, or call ``result.discard()`` to release the reservation of + a transaction that will not be sent. A record another writer holds + raises :class:`~aleo.inventory.RecordReservationConflictError` before + any proof is built. + Returns ------- TransactionResult Inspectable wrapper over the assembled ``Transaction``. """ + managed = _managed_inventory(self._client) + if managed is None: + return TransactionResult(self._prove_transaction( + account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, + )) + checkpoint = managed.begin_spend( + self.program_id, self.function_name, self.args, account=account, + private_fee=private_fee or fee_record is not None, fee_record=_fee_record_text(fee_record), + ) + try: + tx = self._prove_transaction( + account, priority_fee=priority_fee, fee_record=checkpoint.fee_record_selector or fee_record, + private_fee=private_fee, base_fee=base_fee, + ) + checkpoint.prepared(str(tx.id), _transaction_json(tx)) + except BaseException as exc: + checkpoint.discard() + raise _after_acquisition(exc) from exc + return TransactionResult(tx, checkpoint) + + def _prove_transaction( + self, + account: Any = None, + *, + priority_fee: int = 0, + fee_record: Any = None, + private_fee: bool = False, + base_fee: int | None = None, + ) -> Any: + """Authorize, execute, prove, and assemble the ``Transaction`` locally; no journal, no broadcast.""" net = self._net() process = self._client.process auth = self._build_authorization(account) @@ -443,7 +498,7 @@ def build_transaction( detail=str(exc), ) from exc - return TransactionResult(tx) + return tx # ``.prove`` is the documented alias of build_transaction. prove = build_transaction @@ -461,63 +516,23 @@ def transact( ) -> str: """Build the transaction (:meth:`build_transaction`) and broadcast it. - On a client built with record management enabled, the call's record - inputs are reserved before proving, the proved transaction is saved - before broadcast, and a private fee record is chosen and reserved - once the exact fee is known. A record another writer holds raises - :class:`~aleo.inventory.RecordReservationConflictError` before any - proof is built. + On a client built with record management enabled this is the two + managed halves in sequence: the build reserves the record inputs and + saves the proved transaction, the broadcast records the submission. Returns ------- str The transaction id returned by the node. """ - return self.transact_with_result( - account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, - )[0] - - def transact_with_result( - self, - account: Any = None, - *, - priority_fee: int = 0, - fee_record: Any = None, - private_fee: bool = False, - base_fee: int | None = None, - ) -> tuple[str, TransactionResult]: - """Build, broadcast, and also return the built transaction. - - The same as :meth:`transact`, for callers that read the transaction's - outputs without waiting for confirmation. The returned result is the - transaction as proved locally; on a managed client it is also the - transaction saved to the journal before broadcast. - - Returns - ------- - tuple[str, TransactionResult] - The transaction id returned by the node and the built transaction. - """ - managed = _managed_inventory(self._client) - if managed is None: - result = self.build_transaction( - account, priority_fee=priority_fee, fee_record=fee_record, private_fee=private_fee, base_fee=base_fee, - ) - return self._client.network.submit_transaction(result.raw), result - built: list[TransactionResult] = [] - - def build(request: Any) -> Any: - result = self.build_transaction( - account, priority_fee=priority_fee, fee_record=request.fee_record, - private_fee=request.private_fee, base_fee=base_fee, - ) - built.append(result) - return _prepared_from_result(result) - tx_id = managed.managed_write( - self.program_id, self.function_name, self.args, account=account, - private_fee=private_fee or fee_record is not None, fee_record=_fee_record_text(fee_record), builder=build, + result = self.build_transaction( + account, + priority_fee=priority_fee, + fee_record=fee_record, + private_fee=private_fee, + base_fee=base_fee, ) - return tx_id, built[0] + return self._client.network.submit_transaction(result) # ── Verb: delegate (the flagship DPS path) ────────────────────────────── @@ -629,9 +644,17 @@ def _fee_record_text(fee_record: Any) -> str | None: return None if fee_record is None else str(fee_record) -def _prepared_from_result(result: Any) -> Any: - from ..inventory.client import prepared_from_result - return prepared_from_result(result) +def _transaction_json(transaction: Any) -> dict[str, Any]: + import json + return json.loads(str(transaction.to_json())) + + +def _after_acquisition(error: BaseException) -> BaseException: + """A conflict raised after this call reserved its inputs must not look retryable.""" + from ..inventory import InventoryError, RecordReservationConflictError + if isinstance(error, RecordReservationConflictError): + return InventoryError("Record spending failed after acquiring inputs") + return error def _prepared_from_payload(payload: Any) -> Any: diff --git a/sdk/python/aleo/facade/client.py b/sdk/python/aleo/facade/client.py index 5db7bc03..fc0bf5d6 100644 --- a/sdk/python/aleo/facade/client.py +++ b/sdk/python/aleo/facade/client.py @@ -37,12 +37,13 @@ class Aleo: configures the underlying network connection. record_management: Optional :class:`~aleo.inventory.RecordManagementConfig`. With - ``enabled=True`` every ``transact`` and ``delegate`` on this client - reserves its record inputs before proving, saves the proved - transaction before broadcast, and journals the submission, so - writers sharing the store never spend the same record. ``None`` - (the default) leaves writes unchanged; ``aleo.inventory`` still - reads, plans, and performs single joins or splits. + ``enabled=True`` the journal follows each write's two halves: + ``build_transaction`` reserves the record inputs before proving and + saves the proved transaction, ``submit_transaction`` records the + broadcast, and ``delegate`` does both through the prover, so writers + sharing the store never spend the same record. ``None`` (the + default) leaves writes unchanged; ``aleo.inventory`` still reads, + plans, and performs single joins or splits. """ # Expose HTTPProvider as a nested class attribute so callers can write diff --git a/sdk/python/aleo/facade/network.py b/sdk/python/aleo/facade/network.py index 55ac9167..d13959bb 100644 --- a/sdk/python/aleo/facade/network.py +++ b/sdk/python/aleo/facade/network.py @@ -442,20 +442,38 @@ def get_deployment_transaction_for_program(self, program_id: str) -> Any: def submit_transaction(self, transaction: Any) -> str: """Broadcast *transaction* to the network and return the transaction ID. - Accepts either a :class:`Transaction` object (anything with a - ``__str__`` serialisation the node accepts) or a raw JSON string. + Accepts a :class:`~aleo.facade.call.TransactionResult` from + ``build_transaction``, a :class:`Transaction` object (anything with a + ``__str__`` serialisation the node accepts), or a raw JSON string. + + A result built on a client with record management enabled carries + its journal checkpoint: an accepted broadcast is recorded as + submitted, a transaction the node refuses with a 4xx releases its + reservation, and any other failure keeps the reservation for + reconciliation because the outcome is unknown. Parameters ---------- transaction: - A :class:`Transaction` object or a serialised transaction string. + A built result, a :class:`Transaction` object, or a serialised + transaction string. Returns ------- str The transaction ID returned by the node. """ - return str(self._nc().submit_transaction(transaction)) + checkpoint = getattr(transaction, "checkpoint", None) + raw = getattr(transaction, "raw", transaction) + try: + tx_id = str(self._nc().submit_transaction(raw)) + except BaseException as exc: + if checkpoint is not None: + checkpoint.broadcast_failed(exc) + raise + if checkpoint is not None: + checkpoint.submitted(tx_id) + return tx_id # ``send_raw_transaction`` is the web3.py-parity alias — same callable. send_raw_transaction = submit_transaction diff --git a/sdk/python/aleo/inventory/_engine.py b/sdk/python/aleo/inventory/_engine.py index 86be18e3..bceee7a6 100644 --- a/sdk/python/aleo/inventory/_engine.py +++ b/sdk/python/aleo/inventory/_engine.py @@ -291,6 +291,178 @@ def _now_ms() -> int: return int(time.time() * 1000) +class SpendCheckpoint: + """The journal state of one spend, from reservation to submission. + + A spend is the two halves snarkVM separates: proving a transaction and + broadcasting it. :meth:`open_spend` reserves the record inputs before + proving; :meth:`prepared` saves the proved transaction before broadcast + and enforces the fee ceiling; :meth:`submitted` records that the network + accepted it. A spend that never reaches the network is released with + :meth:`discard`, or by :meth:`broadcast_failed` when the node refuses + the transaction outright. Under a private fee, :attr:`fee_record_selector` + chooses and reserves the credits record once the exact fee is known. + + Attributes: + entry: The journal entry holding the input reservation. + entries: Every entry this spend holds; a pinned fee record adds one. + transaction_id: The id recorded at preparation or submission. + """ + + def __init__( + self, store: RecordInventoryStore, scope: str, scope_now: Callable[[], str], entry: RecordSpend, *, + fee_limit: int | None, + ) -> None: + self._store = store + self._scope = scope + self._scope_now = scope_now + self._fee_limit = fee_limit + self._base = entry + self.entry = entry + self.entries: list[RecordSpend] = [entry] + self.transaction_id: str | None = None + self.is_prepared = False + self.is_submitted = False + self.is_closed = False + self._fee_candidates: list[InventoryRecord] = [] + self._fee_choice: InventoryRecord | None = None + self.fee_record_selector: FeeRecordSelector | None = None + + @property + def input_ids(self) -> tuple[str, ...]: + """The reserved execution input nonces.""" + return self._base.records + + def use_fee_candidates(self, available: Sequence[InventoryRecord], preferred: str | None = None) -> FeeRecordSelector: + """Prepare private-fee selection from the credits records available now. + + The returned callable is handed to the prover as ``fee_record``. It + runs once the execution is proved and the exact fee is known, picks + the smallest sufficient record (the preferred nonce first when it is + eligible), and reserves it as its own journal entry so a rejected + execution can keep it held while releasing the inputs. + """ + inputs = set(self._base.records) + self._fee_candidates = sorted( + (item for item in available if item.id not in inputs and item.plaintext and item.tag), + key=lambda item: (item.id != preferred, item.amount, item.id), + ) + + def select(required: int) -> str: + if self._fee_limit is not None and required > self._fee_limit: + raise InventoryError("Estimated transaction exceeds max_fee_microcredits") + if self._scope_now() != self._scope: + raise InventoryError("Account or chain changed during private fee selection") + for candidate in self._fee_candidates: + if candidate.amount < required: + continue + fee_entry = self._store.acquire(replace(self._base, records=(candidate.id,), fee_record_id=candidate.id)) + if fee_entry is not None: + self.entries.append(fee_entry) + self._fee_choice = candidate + return candidate.plaintext + raise InventoryError( + "No available credits record covers the private fee separately from the transaction inputs; " + "refresh records or fund private credits" + ) + + self.fee_record_selector = select + return select + + def prepared(self, tx_id: str, transaction: Mapping[str, Any], fee_microcredits: int | None = None) -> int: + """Save the proved transaction before broadcast; returns the fee in microcredits. + + Raises: + InventoryError: The account or chain changed, the prover did not + spend the reserved fee record, the fee exceeds the pinned + record or ``max_fee_microcredits``, or the fee cannot be read. + Call :meth:`discard` (or let the caller's cleanup do it) + because nothing was broadcast. + """ + if self._scope_now() != self._scope: + raise InventoryError("Account or chain changed during record spending") + chosen = self._fee_choice + if chosen is not None: + # Check the prover honoured the pinned record before saving a + # transaction that recovery could broadcast. + fee_transition = _dict(_dict(transaction.get("fee")).get("transition")) + fee_inputs = _list(fee_transition.get("inputs")) + first = _dict(fee_inputs[0]) if fee_inputs else {} + if ( + fee_transition.get("program") != CREDITS_PROGRAM or fee_transition.get("function") != "fee_private" + or first.get("type") != "record" or first.get("tag") != chosen.tag + ): + raise InventoryError("Prover did not spend the reserved private fee record") + fee = transaction_fee(transaction) if fee_microcredits is None else fee_microcredits + if chosen is not None and fee > chosen.amount: + raise InventoryError("Private fee exceeds the reserved credits record balance") + if self._fee_limit is not None and fee > self._fee_limit: + raise InventoryError("Proved transaction exceeds max_fee_microcredits") + for item in self.entries: + self._store.update(self._scope, item.id, { + "status": "prepared", "transaction_id": tx_id, "transaction": dict(transaction), "fee_microcredits": str(fee), + }) + self.transaction_id = tx_id + self.is_prepared = True + return fee + + def submitted(self, tx_id: str) -> None: + """Record that the network accepted the broadcast; the reservation now outlives any error.""" + # Mark before touching the store: a storage failure must not release + # inputs the network may already have accepted. + self.is_submitted = True + self.transaction_id = tx_id + for item in self.entries: + self._store.update(self._scope, item.id, {"status": "submitted", "transaction_id": tx_id}) + + def confirmed(self, tx_id: str) -> None: + """Record acceptance and drop the saved transaction.""" + for item in self.entries: + self._store.update(self._scope, item.id, {"status": "confirmed", "transaction_id": tx_id, "transaction": None}) + + def broadcast_failed(self, error: BaseException) -> None: + """Decide what a failed broadcast means for the reservation. + + A 4xx from the broadcast endpoint means the node refused the + transaction before admitting it, so nothing can reach the chain and + the inputs are released. Any other failure (timeout, lost response, + 5xx) is uncertain and keeps them for reconciliation. + """ + status = getattr(error, "status", None) + if isinstance(error, AleoNetworkError) and isinstance(status, int) and 400 <= status < 500: + self.discard() + + def discard(self) -> None: + """Release the reservation of a spend that was never submitted. + + Safe to call more than once. Does nothing after :meth:`submitted`, + because the network may have spent the records. + """ + if self.is_submitted or self.is_closed: + return + self.is_closed = True + for item in self.entries: + self._store.update(self._scope, item.id, {"status": "cancelled"}) + + +def open_spend( + store: RecordInventoryStore, scope: str, scope_now: Callable[[], str], ids: Sequence[str], program: str, function: str, + *, asset_program: str | None = None, fee_limit: int | None = None, +) -> SpendCheckpoint: + """Reserve the input nonces of one spend before any proving starts. + + Raises: + RecordReservationConflictError: Another operation holds one of them. + """ + entry = store.acquire(RecordSpend( + id="", scope=scope, records=tuple(ids), program=program, function=function, status="reserved", + created_at=_now_ms(), asset_program=asset_program, + )) + if entry is None: + raise RecordReservationConflictError(tuple(ids)) + return SpendCheckpoint(store, scope, scope_now, entry, fee_limit=fee_limit) + + # ── The engine ─────────────────────────────────────────────────────────────── @@ -477,103 +649,34 @@ def _spend( if observed is not None: observed.append(prepared.id) return (yield Broadcast(prepared.transaction)) - store = self.management.store - scope = self.scope - base = RecordSpend( - id="", scope=scope, records=tuple(ids), program=program, function=function, status="reserved", - created_at=_now_ms(), asset_program=asset_program, - ) # Acquire every input nonce together before any proving starts. - entry = store.acquire(base) - if entry is None: - raise RecordReservationConflictError(ids) - entries = [entry] - fee_choice: list[InventoryRecord] = [] + checkpoint = open_spend( + self.management.store, self.scope, lambda: self.scope, ids, program, function, + asset_program=asset_program, fee_limit=limit, + ) selector: FeeRecordSelector | None = None - if private_fee: - credits = yield from self.get_record_inventory(RecordAsset(CREDITS_PROGRAM)) - preferred = inventory_record_id({"record_plaintext": preferred_fee_record}) if preferred_fee_record else None - candidates = sorted( - (item for item in credits.available if item.id not in ids and item.plaintext and item.tag), - key=lambda item: (item.id != preferred, item.amount, item.id), - ) - - def select(required: int) -> str: - # Called once the execution is proved and the exact fee is - # known. The chosen record joins the journal as its own entry, - # so a rejected execution can keep it held while releasing the - # inputs. - if limit is not None and required > limit: - raise InventoryError("Estimated transaction exceeds max_fee_microcredits") - if self.scope != scope: - raise InventoryError("Account or chain changed during private fee selection") - for candidate in candidates: - if candidate.amount < required: - continue - fee_entry = store.acquire(replace(base, records=(candidate.id,), fee_record_id=candidate.id)) - if fee_entry is not None: - entries.append(fee_entry) - fee_choice.append(candidate) - return candidate.plaintext - raise InventoryError( - "No available credits record covers the private fee separately from the transaction inputs; " - "refresh records or fund private credits" - ) - - selector = select - prepared_flag = False - submitted = False try: + if private_fee: + credits = yield from self.get_record_inventory(RecordAsset(CREDITS_PROGRAM)) + preferred = inventory_record_id({"record_plaintext": preferred_fee_record}) if preferred_fee_record else None + selector = checkpoint.use_fee_candidates(credits.available, preferred) prepared = yield BuildTransaction(program, function, tuple(inputs), tuple(imports), private_fee, selector) - if self.scope != scope: - raise InventoryError("Account or chain changed during record spending") - transaction = prepared.transaction - if fee_choice: - # Check the prover honoured the pinned record before saving a - # transaction that recovery could broadcast. - chosen = fee_choice[0] - fee_transition = _dict(_dict(transaction.get("fee")).get("transition")) - fee_inputs = _list(fee_transition.get("inputs")) - first = _dict(fee_inputs[0]) if fee_inputs else {} - if ( - fee_transition.get("program") != CREDITS_PROGRAM or fee_transition.get("function") != "fee_private" - or first.get("type") != "record" or first.get("tag") != chosen.tag - ): - raise InventoryError("Prover did not spend the reserved private fee record") - fee = transaction_fee(transaction) - if fee_choice and fee > fee_choice[0].amount: - raise InventoryError("Private fee exceeds the reserved credits record balance") - if limit is not None and fee > limit: - raise InventoryError("Proved transaction exceeds max_fee_microcredits") - for item in entries: - store.update(scope, item.id, { - "status": "prepared", "transaction_id": prepared.id, "transaction": transaction, "fee_microcredits": str(fee), - }) - prepared_flag = True + checkpoint.prepared(prepared.id, prepared.transaction) if observed is not None: observed.append(prepared.id) try: - tx_id = yield Broadcast(transaction) - except AleoNetworkError as exc: - # A 4xx from the broadcast endpoint means the node refused the - # transaction before admitting it, so nothing can reach the - # chain and the inputs can be released. Any other failure - # (timeout, lost response, 5xx) is uncertain and keeps them. - if exc.status is not None and 400 <= exc.status < 500: - prepared_flag = False + tx_id = yield Broadcast(prepared.transaction) + except BaseException as exc: + checkpoint.broadcast_failed(exc) raise - # Mark submission before touching the store: a storage failure must - # not release inputs the network may already have accepted. - submitted = True - for item in entries: - store.update(scope, item.id, {"status": "submitted", "transaction_id": tx_id}) + checkpoint.submitted(tx_id) return tx_id except BaseException as exc: # Nothing was broadcast before the prepared checkpoint; afterwards - # the outcome is uncertain and the reservations stay. - if not prepared_flag and not submitted: - for item in entries: - store.update(scope, item.id, {"status": "cancelled"}) + # the outcome is uncertain and the reservations stay unless the + # node refused the transaction outright. + if not checkpoint.is_prepared: + checkpoint.discard() if isinstance(exc, RecordReservationConflictError): # A nested write or a callback raised its own conflict after this # spend acquired its inputs; it must not authorise a retry. @@ -788,6 +891,7 @@ def auto_join( __all__ = [ "DEFAULT", "Broadcast", "BuildTransaction", "ConfirmedTransaction", "FeeRecordSelector", "FindRecords", - "InventoryEngine", "Management", "PreparedTransaction", "ProgramSource", "Request", "Sleep", "Step", - "WaitForScannerSync", "inventory_output_commitments", "positive_integer", "record_input_ids", "transaction_fee", + "InventoryEngine", "Management", "PreparedTransaction", "ProgramSource", "Request", "Sleep", "SpendCheckpoint", "Step", + "WaitForScannerSync", "inventory_output_commitments", "open_spend", "positive_integer", "record_input_ids", + "transaction_fee", ] diff --git a/sdk/python/aleo/inventory/client.py b/sdk/python/aleo/inventory/client.py index bcd0b5b5..b967f90e 100644 --- a/sdk/python/aleo/inventory/client.py +++ b/sdk/python/aleo/inventory/client.py @@ -18,15 +18,14 @@ from ..facade.errors import ProgramNotFound, TransactionNotFound from ._engine import ( Broadcast, BuildTransaction, ConfirmedTransaction, FindRecords, InventoryEngine, Management, PreparedTransaction, - ProgramSource, Request, Sleep, Step, transaction_fee, + ProgramSource, Request, Sleep, SpendCheckpoint, Step, open_spend, record_input_ids, ) from .assets import CREDITS_PROGRAM from .interface import parse_program_interface from .store import MemoryRecordInventoryStore from .types import ( InventoryError, InventoryPlan, InventoryRecord, InventoryResult, InventoryTarget, RecordAsset, - RecordInventorySnapshot, RecordInventoryStore, RecordJoinProgram, RecordManagementConfig, - RecordReservationConflictError, RecordSpend, + RecordInventorySnapshot, RecordInventoryStore, RecordJoinProgram, RecordManagementConfig, RecordSpend, ) Proving = Literal["delegated", "local"] @@ -34,52 +33,6 @@ AsyncBuilder = Callable[[BuildTransaction], "PreparedTransaction | Awaitable[PreparedTransaction]"] -class SpendCheckpoint: - """Report a self-managed write's progress to the journal. - - Yielded by :meth:`RecordInventory.reserve`. Call :meth:`prepared` as soon - as the proved transaction exists and before it is broadcast, then - :meth:`submitted`; recovery can then resubmit the saved transaction - without building a second spend. - - Attributes: - entry: The journal entry holding the reservation. - """ - - def __init__(self, store: RecordInventoryStore, scope: str, entry: RecordSpend, fee_limit: int | None) -> None: - self._store = store - self._scope = scope - self._fee_limit = fee_limit - self.entry = entry - self.is_prepared = False - self.is_submitted = False - - def prepared(self, tx_id: str, transaction: Mapping[str, Any], fee_microcredits: int | None = None) -> None: - """Save the proved transaction before broadcast and enforce the fee ceiling. - - Raises: - InventoryError: The fee exceeds ``max_fee_microcredits`` or cannot - be read from the transaction. The reservation is released on - exit because nothing was broadcast. - """ - fee = transaction_fee(transaction) if fee_microcredits is None else fee_microcredits - if self._fee_limit is not None and fee > self._fee_limit: - raise InventoryError("Proved transaction exceeds max_fee_microcredits") - self._store.update(self._scope, self.entry.id, { - "status": "prepared", "transaction_id": tx_id, "transaction": dict(transaction), "fee_microcredits": str(fee), - }) - self.is_prepared = True - - def submitted(self, tx_id: str) -> None: - """Record that the network accepted the broadcast; the reservation now outlives any error.""" - self.is_submitted = True - self._store.update(self._scope, self.entry.id, {"status": "submitted", "transaction_id": tx_id}) - - def confirmed(self, tx_id: str) -> None: - """Record acceptance and drop the saved transaction.""" - self._store.update(self._scope, self.entry.id, {"status": "confirmed", "transaction_id": tx_id, "transaction": None}) - - class _InventoryBase: """Shared construction, scope, and journal access for both clients.""" @@ -152,30 +105,31 @@ def reserve( Use it around a write that proves and broadcasts through another path. Inside the block, call ``checkpoint.prepared`` once the proved transaction exists and ``checkpoint.submitted`` after broadcast. - Leaving the block before ``prepared``, by return or by exception, + Leaving the block before ``submitted``, by return or by exception, releases the reservation because nothing can have reached the - network; afterwards it is kept for reconciliation. + network through this checkpoint. Raises: InventoryError: Management is disabled. RecordReservationConflictError: A record is already held. """ - store = self._management.store - if not self._management.enabled or store is None: - raise InventoryError("reserve requires record management to be enabled with a shared store") - scope = self.scope(account) - entry = store.acquire(RecordSpend( - id="", scope=scope, records=tuple(record_ids), program=program, function=function, status="reserved", - created_at=int(time.time() * 1000), asset_program=asset_program, - )) - if entry is None: - raise RecordReservationConflictError(tuple(record_ids)) - checkpoint = SpendCheckpoint(store, scope, entry, self._management.max_fee_microcredits) + checkpoint = self._open_spend(record_ids, program, function, asset_program=asset_program, account=account) try: yield checkpoint finally: - if not checkpoint.is_prepared and not checkpoint.is_submitted: - store.update(scope, entry.id, {"status": "cancelled"}) + checkpoint.discard() + + def _open_spend( + self, record_ids: Sequence[str], program: str, function: str, *, asset_program: str | None, account: Any, + ) -> SpendCheckpoint: + store = self._management.store + if not self._management.enabled or store is None: + raise InventoryError("Record reservation requires record management to be enabled with a shared store") + engine = self._engine(account) + return open_spend( + store, engine.scope, lambda: engine.scope, record_ids, program, function, + asset_program=asset_program, fee_limit=self._management.max_fee_microcredits, + ) # ── Program registration for local authorization ───────────────────────── @@ -333,6 +287,36 @@ def _register_programs(self, program_ids: Sequence[str]) -> None: stack.extend(parse_program_interface(sources[program_id]).imports) self._register_sources(sources) + def begin_spend( + self, program: str, function: str, inputs: Sequence[str], *, account: Any = None, + private_fee: bool = False, fee_record: str | None = None, + ) -> SpendCheckpoint: + """Reserve a call's record inputs before it is proved; the first half of a managed spend. + + The facade's ``build_transaction`` calls this on a managed client. + Record plaintexts among *inputs* owned by the signer are reserved + together. With a private fee, the credits inventory is scanned now and + ``checkpoint.fee_record_selector`` chooses and reserves the fee record + once the exact fee is known (*fee_record* is preferred when eligible). + Call ``checkpoint.prepared`` after proving, pass the result to + ``aleo.network.submit_transaction`` to record the broadcast, or + ``checkpoint.discard()`` to release a spend that is never sent. + + Raises: + RecordReservationConflictError: An input is already held. + """ + engine = self._engine(account) + checkpoint = self._open_spend(record_input_ids(inputs, engine.address), program, function, asset_program=None, account=account) + if private_fee or fee_record is not None: + try: + credits = self.get_record_inventory(RecordAsset(CREDITS_PROGRAM), account=account) + preferred = record_input_ids([fee_record], engine.address)[0] if fee_record else None + checkpoint.use_fee_candidates(credits.available, preferred) + except BaseException: + checkpoint.discard() + raise + return checkpoint + # ── Actions ────────────────────────────────────────────────────────────── def get_record_inventory(self, asset: RecordAsset, *, account: Any = None) -> RecordInventorySnapshot: @@ -635,6 +619,23 @@ async def _register_programs(self, program_ids: Sequence[str]) -> None: stack.extend(parse_program_interface(sources[program_id]).imports) self._register_sources(sources) + async def begin_spend( + self, program: str, function: str, inputs: Sequence[str], *, account: Any = None, + private_fee: bool = False, fee_record: str | None = None, + ) -> SpendCheckpoint: + """Reserve a call's record inputs before it is proved (awaited). See :meth:`RecordInventory.begin_spend`.""" + engine = self._engine(account) + checkpoint = self._open_spend(record_input_ids(inputs, engine.address), program, function, asset_program=None, account=account) + if private_fee or fee_record is not None: + try: + credits = await self.get_record_inventory(RecordAsset(CREDITS_PROGRAM), account=account) + preferred = record_input_ids([fee_record], engine.address)[0] if fee_record else None + checkpoint.use_fee_candidates(credits.available, preferred) + except BaseException: + checkpoint.discard() + raise + return checkpoint + async def get_record_inventory(self, asset: RecordAsset, *, account: Any = None) -> RecordInventorySnapshot: """Read the account's spendable records of one asset (awaited). See :meth:`RecordInventory.get_record_inventory`.""" return await self._run(self._engine(account).get_record_inventory(asset), account) diff --git a/sdk/python/tests/test_facade_inventory.py b/sdk/python/tests/test_facade_inventory.py index 2fb81230..d86330f2 100644 --- a/sdk/python/tests/test_facade_inventory.py +++ b/sdk/python/tests/test_facade_inventory.py @@ -51,20 +51,16 @@ def fake_transaction(tx_id: str, fee_tag: str | None = None, fee: int = 100) -> "fee": {"transition": {"program": "credits.aleo", "function": function, "inputs": inputs}}} -class FakeResult: +class FakeTransaction: + """Stands in for the proved network ``Transaction``: an id and a JSON form.""" + def __init__(self, tx: dict[str, Any]) -> None: self.id = tx["id"] - self.raw = self + self._tx = tx def to_json(self) -> str: return json.dumps(self._tx) - @classmethod - def of(cls, tx: dict[str, Any]) -> "FakeResult": - result = cls(tx) - result._tx = tx - return result - def setup(monkeypatch: pytest.MonkeyPatch, *, enabled: bool = True, max_fee: int | None = None, fee: int = 100, records: list[int] | None = None) -> tuple[Aleo, Any, Program, dict[str, Any]]: @@ -78,8 +74,8 @@ def setup(monkeypatch: pytest.MonkeyPatch, *, enabled: bool = True, max_fee: int seen: dict[str, Any] = {"builds": [], "submits": [], "nonce_tag": {}} sequence = [0] - def build_transaction(self: BoundCall, acct: Any = None, *, priority_fee: int = 0, fee_record: Any = None, - private_fee: bool = False, base_fee: int | None = None) -> FakeResult: + def prove(self: BoundCall, acct: Any = None, *, priority_fee: int = 0, fee_record: Any = None, + private_fee: bool = False, base_fee: int | None = None) -> FakeTransaction: sequence[0] += 1 tag = None if private_fee or fee_record is not None: @@ -87,17 +83,17 @@ def build_transaction(self: BoundCall, acct: Any = None, *, priority_fee: int = tag = next(r["tag"] for r in aleo.record_provider.records if r["record_plaintext"] == chosen) tx = fake_transaction(f"at1tx{sequence[0]}", tag, fee + priority_fee) seen["builds"].append({"args": list(self.args), "private_fee": private_fee, "fee_record": fee_record, "tx": tx}) - return FakeResult.of(tx) + return FakeTransaction(tx) - monkeypatch.setattr(BoundCall, "build_transaction", build_transaction) + # Replace only the proving step, so the managed build and submit halves run for real. + monkeypatch.setattr(BoundCall, "_prove_transaction", prove) def submit_transaction(raw: Any) -> str: - # The managed path submits JSON text; the plain path submits the result object. tx = json.loads(raw if isinstance(raw, str) else raw.to_json()) seen["submits"].append(tx) return str(tx["id"]) - monkeypatch.setattr(aleo.network, "submit_transaction", submit_transaction) + monkeypatch.setattr(aleo._client, "submit_transaction", submit_transaction) # the raw node client # The synthetic records cannot be authorized by snarkVM, so a prover-side # delegate (broadcast=False) is answered directly; the broadcasting call # still runs the real hook. @@ -145,6 +141,54 @@ def test_managed_transact_reserves_inputs_saves_the_transaction_and_journals_sub assert aleo.inventory.get_record_inventory(RecordAsset("credits.aleo")).available[0].id == "2group" +def test_build_and_submit_halves_are_managed_on_their_own(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, seen = setup(monkeypatch) + result = transfer(credits, aleo).build_transaction() + assert result.checkpoint is not None and result.checkpoint.is_prepared + entry = aleo.inventory.store.list(aleo.inventory.scope())[0] + assert (entry.status, entry.transaction_id) == ("prepared", "at1tx1") and entry.transaction == seen["builds"][0]["tx"] + assert seen["submits"] == [] + tx_id = aleo.network.submit_transaction(result) + assert tx_id == "at1tx1" and aleo.inventory.store.list(aleo.inventory.scope())[0].status == "submitted" + result.discard() # no effect after submission + assert aleo.inventory.store.list(aleo.inventory.scope())[0].status == "submitted" + + +def test_a_built_transaction_can_be_discarded_before_submission(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, _ = setup(monkeypatch) + result = transfer(credits, aleo).build_transaction() + assert aleo.inventory.reserved_record_ids() == {"1group"} + result.discard() + assert aleo.inventory.store.list(aleo.inventory.scope())[0].status == "cancelled" + assert aleo.inventory.reserved_record_ids() == set() + assert transfer(credits, aleo).transact() == "at1tx2" # the record is free again + + +def test_a_node_refusal_on_submit_releases_the_reservation(monkeypatch: pytest.MonkeyPatch) -> None: + from aleo import AleoNetworkError + aleo, _, credits, _ = setup(monkeypatch) + + def refuse(raw: Any) -> str: + raise AleoNetworkError("POST /transaction/broadcast returned 400", status=400) + + monkeypatch.setattr(aleo._client, "submit_transaction", refuse) + with pytest.raises(AleoNetworkError): + transfer(credits, aleo).transact() + assert aleo.inventory.store.list(aleo.inventory.scope())[0].status == "cancelled" + + +def test_a_proving_failure_releases_the_reservation(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, _ = setup(monkeypatch) + + def fail(self: BoundCall, acct: Any = None, **kw: Any) -> Any: + raise RuntimeError("proving failed") + + monkeypatch.setattr(BoundCall, "_prove_transaction", fail) + with pytest.raises(RuntimeError): + transfer(credits, aleo).build_transaction() + assert aleo.inventory.store.list(aleo.inventory.scope())[0].status == "cancelled" + + def test_managed_transact_refuses_a_reserved_input_before_proving(monkeypatch: pytest.MonkeyPatch) -> None: aleo, _, credits, seen = setup(monkeypatch) transfer(credits, aleo).transact() @@ -206,10 +250,10 @@ def test_delegate_without_broadcast_stays_unmanaged(monkeypatch: pytest.MonkeyPa def test_lost_broadcast_response_keeps_the_prepared_transaction(monkeypatch: pytest.MonkeyPatch) -> None: aleo, _, credits, _ = setup(monkeypatch) - def lost(raw: str) -> str: + def lost(raw: Any) -> str: raise ConnectionError("response lost") - monkeypatch.setattr(aleo.network, "submit_transaction", lost) + monkeypatch.setattr(aleo._client, "submit_transaction", lost) with pytest.raises(ConnectionError): transfer(credits, aleo).transact() entry = aleo.inventory.store.list(aleo.inventory.scope())[0] @@ -233,21 +277,22 @@ async def test_async_managed_transact_runs_the_same_hook(monkeypatch: pytest.Mon aleo.record_provider = AsyncProvider([record(str(account.address), 1_000_000, 1)]) submits: list[dict[str, Any]] = [] - async def build_transaction(self: AsyncBoundCall, acct: Any = None, **kw: Any) -> FakeResult: - return FakeResult.of(fake_transaction("at1async")) + async def prove(self: AsyncBoundCall, acct: Any = None, **kw: Any) -> FakeTransaction: + return FakeTransaction(fake_transaction("at1async")) - async def submit_transaction(raw: str) -> str: - submits.append(json.loads(raw)) + async def submit_transaction(raw: Any) -> str: + submits.append(json.loads(raw if isinstance(raw, str) else raw.to_json())) return "at1async" - monkeypatch.setattr(AsyncBoundCall, "build_transaction", build_transaction) - monkeypatch.setattr(aleo.network, "submit_transaction", submit_transaction) + monkeypatch.setattr(AsyncBoundCall, "_prove_transaction", prove) + monkeypatch.setattr(aleo.network_client, "submit_transaction", submit_transaction) from aleo.facade.async_client import AsyncProgram credits = AsyncProgram(aleo, RawProgram.from_source(CREDITS_SOURCE)) - call = credits.functions.transfer_private(aleo.record_provider.records[0]["record_plaintext"], str(account.address), 1) - assert await call.transact() == "at1async" + def call() -> Any: # the async program's callers are typed as the sync bound call + return credits.functions.transfer_private(aleo.record_provider.records[0]["record_plaintext"], str(account.address), 1) + assert await call().transact() == "at1async" assert [s["id"] for s in submits] == ["at1async"] entry = aleo.inventory.store.list(aleo.inventory.scope())[0] assert (entry.status, entry.records) == ("submitted", ("1group",)) with pytest.raises(RecordReservationConflictError): - await credits.functions.transfer_private(aleo.record_provider.records[0]["record_plaintext"], str(account.address), 1).transact() + await call().transact() diff --git a/shield-swap-sdk/python/aleo_shield_swap/_calls.py b/shield-swap-sdk/python/aleo_shield_swap/_calls.py index 65d4c6e1..d03fbd7f 100644 --- a/shield-swap-sdk/python/aleo_shield_swap/_calls.py +++ b/shield-swap-sdk/python/aleo_shield_swap/_calls.py @@ -14,7 +14,7 @@ from typing import Any, Callable, Generic, Optional, TypeVar -from aleo.inventory import RecordReservationConflictError, managed_inventory +from aleo.inventory import RecordReservationConflictError from ._core import record_nonce @@ -165,20 +165,17 @@ def transact(self, account: Any = None, *, wait: bool = False, before broadcast, so the result is complete without waiting for confirmation. Set ``wait=True`` to confirm and update an attached claim journal. - On a facade with record management enabled the record inputs are - reserved before proving and the proved transaction is saved before - broadcast. When another writer holds the selected record, a swap - that chose its record automatically moves to the next eligible one - before proving; an explicit record is never replaced. + On a facade with record management enabled the build reserves the + record inputs and saves the proved transaction, and the broadcast + records the submission. When another writer holds the selected + record, a swap that chose its record automatically moves to the next + eligible one before proving; an explicit record is never replaced. """ def submit(bound: Any) -> tuple[str, Any]: - if managed_inventory(self._aleo) is not None: - # The facade verb reserves, journals, and broadcasts; it hands - # back the built transaction for output harvesting. - return bound.transact_with_result(account, **fee_kwargs) + # The facade's own two halves: on a managed client the build + # reserves the inputs and the submit records the broadcast. tx = bound.build_transaction(account, **fee_kwargs) - # Same submit path the facade's BoundCall.transact uses. - self._aleo.network.submit_transaction(tx.raw) + self._aleo.network.submit_transaction(tx) return tx.id, tx tx_id, tx = self._submit(submit) diff --git a/shield-swap-sdk/python/aleo_shield_swap/async_client.py b/shield-swap-sdk/python/aleo_shield_swap/async_client.py index 0bf163c2..0319268d 100644 --- a/shield-swap-sdk/python/aleo_shield_swap/async_client.py +++ b/shield-swap-sdk/python/aleo_shield_swap/async_client.py @@ -19,7 +19,6 @@ from aleo.compliance import AsyncFreezeListProofs, ComplianceProofContext, async_freezelist_program from aleo.inventory import ( AsyncRecordInventory, InventoryPlan, InventoryResult, InventoryTarget, RecordAsset, RecordReservationConflictError, - managed_inventory, ) from . import _generated as g @@ -134,10 +133,8 @@ async def transact(self, account: Any = None, *, wait: bool = False, a reservation conflict; see :meth:`~aleo_shield_swap._calls.DexCall.transact`. """ async def submit(bound: Any) -> tuple[str, Any]: - if managed_inventory(self._aleo) is not None: - return await bound.transact_with_result(account, **fee_kwargs) tx = await bound.build_transaction(account, **fee_kwargs) - await self._aleo.network.submit_transaction(tx.raw) + await self._aleo.network.submit_transaction(tx) return tx.id, tx tx_id, tx = await self._submit(submit) diff --git a/shield-swap-sdk/tests/test_inventory.py b/shield-swap-sdk/tests/test_inventory.py index 5edbcf07..fa258cdf 100644 --- a/shield-swap-sdk/tests/test_inventory.py +++ b/shield-swap-sdk/tests/test_inventory.py @@ -123,19 +123,20 @@ def build(request: Any) -> PreparedTransaction: self._world.inventory.managed_write(self.program_id, self.function_name, self.args, account=account, builder=build) return payloads[0] - def transact_with_result(self, account: Any = None, **kw: Any) -> tuple[str, Any]: - built: list[Any] = [] - - def build(request: Any) -> PreparedTransaction: - result = self.build_transaction(account) - built.append(result) - return PreparedTransaction(result.id, json.loads(result.raw.to_json())) - - tx_id = self._world.inventory.managed_write(self.program_id, self.function_name, self.args, account=account, builder=build) - return tx_id, built[0] - def build_transaction(self, account: Any = None, **kw: Any) -> Any: - tx = self._prove() + # The facade contract: on a managed client the build reserves inputs + # and saves the proved transaction; the submit records the broadcast. + checkpoint = None + if self._world.record_management.enabled: + checkpoint = self._world.inventory.begin_spend(self.program_id, self.function_name, self.args, account=account) + try: + tx = self._prove() + if checkpoint is not None: + checkpoint.prepared(tx["id"], tx) + except BaseException: + if checkpoint is not None: + checkpoint.discard() + raise outer = self class _Result: @@ -148,7 +149,9 @@ def to_json() -> str: def decoded(self) -> list[dict[str, Any]]: return [{"program": outer.program_id, "function": outer.function_name, "outputs": [{"value": "77field"}]}] - return _Result() + result = _Result() + result.checkpoint = checkpoint # type: ignore[attr-defined] + return result class _ManagedFunctions: @@ -189,7 +192,18 @@ def __init__(self, record_list: list[dict[str, Any]], *, store: Any = None, enab self.prove_error: Exception | None = None self.submitted_json: list[dict[str, Any]] = [] original = self.network.submit_transaction - self.network.submit_transaction = lambda raw: (self.submitted_json.append(json.loads(raw)) or json.loads(raw)["id"]) if isinstance(raw, str) else original(raw) # type: ignore[method-assign] + + def submit(raw: Any) -> str: + checkpoint = getattr(raw, "checkpoint", None) + if isinstance(raw, str) or checkpoint is not None: + tx = json.loads(raw if isinstance(raw, str) else raw.raw.to_json()) + self.submitted_json.append(tx) + if checkpoint is not None: + checkpoint.submitted(tx["id"]) + return tx["id"] + return original(raw) + + self.network.submit_transaction = submit # type: ignore[method-assign] def dex_for(world: ManagedStubAleo) -> ShieldSwap: From b5ff5396a484f2bf60fac6d447aa530a6db94839 Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 10:59:01 -0400 Subject: [PATCH 07/11] fix: close the four review findings on record management - Inventory actions prove through the facade's unmanaged paths (_prove_transaction, delegate(broadcast=False)) so a managed client under local proving does not reserve its inputs twice. - The private fee record is folded into the same journal entry as the execution inputs (Veil's layout): a short-lived claim wins it atomically, then update(records, fee_record_id) moves it over. Every later checkpoint is one store update, so a crash cannot keep the proved transaction while releasing its fee record. - Dead-process recovery and reconcile(release_reserved=True) cancel only entries with account_type "local"; a Veil wallet's rpc entry is kept. - SpendCheckpoint tracks uncertain broadcast attempts: discard() is inert after any attempt, and a 4xx that follows an uncertain attempt keeps the reservation. The managed broadcast asks the raw client for single attempts (retry=False) and reports each one to the checkpoint before retrying with the usual backoff. --- README.md | 12 ++- sdk/Readme.md | 2 +- sdk/python/aleo/async_network_client.py | 11 +- sdk/python/aleo/facade/async_client.py | 15 ++- sdk/python/aleo/facade/call.py | 9 +- sdk/python/aleo/facade/network.py | 23 ++--- sdk/python/aleo/inventory/__init__.py | 4 +- sdk/python/aleo/inventory/_engine.py | 120 ++++++++++++++-------- sdk/python/aleo/inventory/client.py | 86 +++++++++++++--- sdk/python/aleo/inventory/store.py | 31 +++--- sdk/python/aleo/inventory/types.py | 8 +- sdk/python/aleo/network_client.py | 11 +- sdk/python/tests/test_facade_inventory.py | 63 ++++++++++-- sdk/python/tests/test_inventory_engine.py | 86 +++++++++++++--- sdk/python/tests/test_inventory_store.py | 17 ++- shield-swap-sdk/tests/test_inventory.py | 3 +- 16 files changed, 366 insertions(+), 135 deletions(-) diff --git a/README.md b/README.md index b6a9894e..7314b2c6 100644 --- a/README.md +++ b/README.md @@ -234,8 +234,10 @@ write each do their part of the journal work, the way snarkVM separates building a transaction from broadcasting it: `build_transaction` reserves the call's record inputs before proving and saves the proved transaction, and `submit_transaction` records the broadcast. `transact` is the two in sequence; -`delegate` does the same through the prover. A built transaction that will not -be sent is released with `result.discard()`. The example below reshapes the +`delegate` does the same through the prover. A built transaction that was never +sent is released with `result.discard()`; once a broadcast has been attempted, +even one that timed out, the reservation stays until +`reconcile_record_inventory()` settles it. The example below reshapes the account's private credits into four equal records of at least one credit each: ```python @@ -280,8 +282,10 @@ processes coordinate. | `JsonFileRecordInventoryStore(path)` | Yes | Processes on one machine | The journal should be readable in an editor; one JSON file under a file lock. | | `SqliteRecordInventoryStore(path)` | Yes | Processes on one machine | A long-running service, or a journal shared with a Veil client; same layout as Veil's store. | -Both file-backed stores release a reservation left by a process on the same -machine that exited before it proved anything. Any object with `list`, +Both file-backed stores release a reservation left by an SDK process on the +same machine that exited before it proved anything; an entry a Veil wallet +client wrote is kept, since the wallet may have broadcast before its answer +arrived. Any object with `list`, `acquire`, and `update` methods works as a store for other backends, such as a database shared across machines; `acquire` must reserve all of an entry's records or none. After an interruption, `reconcile_record_inventory()` checks diff --git a/sdk/Readme.md b/sdk/Readme.md index be8d2fa4..bce957a5 100644 --- a/sdk/Readme.md +++ b/sdk/Readme.md @@ -166,7 +166,7 @@ credits.functions.transfer_private(record, str(recipient.address), 1) \ ### Record inventory — join, split, plan, reserve, recover -One transaction spends whole records, so a large payment needs one record that covers it and several payments submitted together need one record each. `aleo.inventory` reads an asset's spendable records, plans the joins and splits that reach a target layout, and executes that plan one confirmed transaction at a time. Build the client with `record_management` enabled and the journal follows the two halves of a write: `build_transaction` reserves the record inputs before proving and saves the proved transaction, `submit_transaction` records the broadcast (`transact` is both in sequence, `delegate` does the same through the prover), and `result.discard()` releases a built transaction that will not be sent. Writers sharing the same store never spend the same record. +One transaction spends whole records, so a large payment needs one record that covers it and several payments submitted together need one record each. `aleo.inventory` reads an asset's spendable records, plans the joins and splits that reach a target layout, and executes that plan one confirmed transaction at a time. Build the client with `record_management` enabled and the journal follows the two halves of a write: `build_transaction` reserves the record inputs before proving and saves the proved transaction, `submit_transaction` records the broadcast (`transact` is both in sequence, `delegate` does the same through the prover), and `result.discard()` releases a built transaction that was never sent (after a broadcast attempt the reservation stays until reconciliation). Writers sharing the same store never spend the same record. ```python from aleo.inventory import InventoryTarget, RecordAsset, RecordManagementConfig diff --git a/sdk/python/aleo/async_network_client.py b/sdk/python/aleo/async_network_client.py index b96eefdb..e04680b2 100644 --- a/sdk/python/aleo/async_network_client.py +++ b/sdk/python/aleo/async_network_client.py @@ -311,6 +311,8 @@ async def _post( body: str, method_name: str, extra_headers: dict[str, str] | None = None, + *, + retry: bool = True, ) -> Any: hdrs = { **self._request_headers(method_name), @@ -327,7 +329,7 @@ async def _do() -> Any: ) return resp - return await async_retry_with_backoff(_do) + return await async_retry_with_backoff(_do) if retry else await _do() # ── JWT refresh ─────────────────────────────────────────────────────── @@ -903,7 +905,7 @@ async def get_deployment_transaction_for_program(self, program_id: str) -> Any: # ── POST endpoints ──────────────────────────────────────────────────── - async def submit_transaction(self, transaction: Any) -> str: + async def submit_transaction(self, transaction: Any, *, retry: bool = True) -> str: """Broadcast a proven transaction to the network. This spends the transaction's fee and is not reversible. Returning @@ -917,6 +919,9 @@ async def submit_transaction(self, transaction: Any) -> str: Args: transaction: The proven transaction; stringified before broadcast, so either a ``Transaction`` or its serialized form works. + retry: Retry transient node errors (5xx) with backoff. Pass + ``False`` to make exactly one attempt, when the caller needs + to see each outcome itself. Returns: The broadcast transaction ID (``at1…``). @@ -931,7 +936,7 @@ async def submit_transaction(self, transaction: Any) -> str: if self._verbose_errors else f"{self._host}/transaction/broadcast" ) - resp = await self._post(endpoint, tx_str, "submitTransaction") + resp = await self._post(endpoint, tx_str, "submitTransaction", retry=retry) return json.loads(resp.text) async def submit_solution(self, solution: str) -> str: diff --git a/sdk/python/aleo/facade/async_client.py b/sdk/python/aleo/facade/async_client.py index f0ffe37f..3f3706d6 100644 --- a/sdk/python/aleo/facade/async_client.py +++ b/sdk/python/aleo/facade/async_client.py @@ -694,15 +694,12 @@ async def submit_transaction(self, transaction: Any) -> str: """ checkpoint = getattr(transaction, "checkpoint", None) raw = getattr(transaction, "raw", transaction) - try: - tx_id = str(await self._nc().submit_transaction(raw)) - except BaseException as exc: - if checkpoint is not None: - checkpoint.broadcast_failed(exc) - raise - if checkpoint is not None: - checkpoint.submitted(tx_id) - return tx_id + if checkpoint is None: + return str(await self._nc().submit_transaction(raw)) + from ..inventory.client import async_broadcast_checkpointed + # One attempt per call so the checkpoint sees every outcome; see the + # sync NetworkModule.submit_transaction. + return await async_broadcast_checkpointed(lambda: self._nc().submit_transaction(raw, retry=False), checkpoint) send_raw_transaction = submit_transaction diff --git a/sdk/python/aleo/facade/call.py b/sdk/python/aleo/facade/call.py index 05ed31f8..7335bbf4 100644 --- a/sdk/python/aleo/facade/call.py +++ b/sdk/python/aleo/facade/call.py @@ -131,8 +131,11 @@ def raw(self) -> Any: def discard(self) -> None: """Release the record reservation of a built transaction that will not be submitted. - Only meaningful on a managed client; after ``submit_transaction`` it - does nothing, because the network may have spent the records. + Only meaningful on a managed client. It does nothing once + ``submit_transaction`` has been attempted, whether the node accepted + the transaction or the attempt ended without an answer, because the + network may have spent the records; use + ``reconcile_record_inventory`` to settle such a spend. """ if self.checkpoint is not None: self.checkpoint.discard() @@ -421,7 +424,7 @@ def build_transaction( chosen and reserved once the exact fee is known. Submit the result with ``aleo.network.submit_transaction(result)`` to record the broadcast, or call ``result.discard()`` to release the reservation of - a transaction that will not be sent. A record another writer holds + a transaction that was never sent. A record another writer holds raises :class:`~aleo.inventory.RecordReservationConflictError` before any proof is built. diff --git a/sdk/python/aleo/facade/network.py b/sdk/python/aleo/facade/network.py index d13959bb..9b649518 100644 --- a/sdk/python/aleo/facade/network.py +++ b/sdk/python/aleo/facade/network.py @@ -448,9 +448,11 @@ def submit_transaction(self, transaction: Any) -> str: A result built on a client with record management enabled carries its journal checkpoint: an accepted broadcast is recorded as - submitted, a transaction the node refuses with a 4xx releases its - reservation, and any other failure keeps the reservation for - reconciliation because the outcome is unknown. + submitted, a transaction the node refuses with a 4xx on the first + attempt releases its reservation, and any other failure keeps the + reservation for reconciliation because the outcome is unknown. Each + retry of a transient node error is reported to the checkpoint, so a + refusal after an uncertain attempt also keeps the reservation. Parameters ---------- @@ -465,15 +467,12 @@ def submit_transaction(self, transaction: Any) -> str: """ checkpoint = getattr(transaction, "checkpoint", None) raw = getattr(transaction, "raw", transaction) - try: - tx_id = str(self._nc().submit_transaction(raw)) - except BaseException as exc: - if checkpoint is not None: - checkpoint.broadcast_failed(exc) - raise - if checkpoint is not None: - checkpoint.submitted(tx_id) - return tx_id + if checkpoint is None: + return str(self._nc().submit_transaction(raw)) + from ..inventory.client import broadcast_checkpointed + # One attempt per call so the checkpoint sees every outcome; the + # helper applies the same backoff the raw client would. + return broadcast_checkpointed(lambda: self._nc().submit_transaction(raw, retry=False), checkpoint) # ``send_raw_transaction`` is the web3.py-parity alias — same callable. send_raw_transaction = submit_transaction diff --git a/sdk/python/aleo/inventory/__init__.py b/sdk/python/aleo/inventory/__init__.py index 4b61ef53..6687e934 100644 --- a/sdk/python/aleo/inventory/__init__.py +++ b/sdk/python/aleo/inventory/__init__.py @@ -46,6 +46,7 @@ managed_inventory as managed_inventory, prepared_from_payload as prepared_from_payload, prepared_from_result as prepared_from_result, + prepared_from_transaction as prepared_from_transaction, ) from .planner import build_inventory_plan as build_inventory_plan from .store import ( @@ -86,6 +87,7 @@ "ResolvedRecordAsset", "SpendCheckpoint", "SqliteRecordInventoryStore", "TokenJoinRouter", "build_inventory_plan", "decode_record", "held_record_ids", "holds_records", "inventory_output_commitments", "inventory_record_id", "managed_inventory", "parse_program_interface", - "prepared_from_payload", "prepared_from_result", "record_input_ids", "scope_for", "transaction_fee", + "prepared_from_payload", "prepared_from_result", "prepared_from_transaction", "record_input_ids", "scope_for", + "transaction_fee", "validate_asset", "validate_record_join_program", "validate_token_join_router", ] diff --git a/sdk/python/aleo/inventory/_engine.py b/sdk/python/aleo/inventory/_engine.py index bceee7a6..cb55bc92 100644 --- a/sdk/python/aleo/inventory/_engine.py +++ b/sdk/python/aleo/inventory/_engine.py @@ -105,9 +105,19 @@ class BuildTransaction: @dataclass(frozen=True) class Broadcast: - """Submit a proved transaction; the answer is the transaction id the node returned.""" + """Submit a proved transaction; the answer is the transaction id the node returned. + + Attributes: + transaction: The transaction as JSON data. + checkpoint: The journal checkpoint of a managed spend, when there is + one. A driver that retries the broadcast should report every + attempt's failure to ``checkpoint.broadcast_failed`` so a + refusal that follows an uncertain attempt does not release the + inputs; the engine reports the final outcome itself either way. + """ transaction: Mapping[str, Any] + checkpoint: "SpendCheckpoint | None" = None @dataclass(frozen=True) @@ -303,10 +313,16 @@ class SpendCheckpoint: the transaction outright. Under a private fee, :attr:`fee_record_selector` chooses and reserves the credits record once the exact fee is known. + One journal entry holds everything the spend reserves, the execution + inputs and the fee record alike, as in Veil. Every checkpoint is one + store update, so a crash between two of them can never leave the proved + transaction retained while its fee record is released. + Attributes: - entry: The journal entry holding the input reservation. - entries: Every entry this spend holds; a pinned fee record adds one. + entry: The journal entry holding the reservation. transaction_id: The id recorded at preparation or submission. + is_uncertain: A broadcast attempt ended without a definite answer, + so the network may hold the transaction; the reservation stays. """ def __init__( @@ -317,12 +333,12 @@ def __init__( self._scope = scope self._scope_now = scope_now self._fee_limit = fee_limit - self._base = entry + self._inputs = entry.records self.entry = entry - self.entries: list[RecordSpend] = [entry] self.transaction_id: str | None = None self.is_prepared = False self.is_submitted = False + self.is_uncertain = False self.is_closed = False self._fee_candidates: list[InventoryRecord] = [] self._fee_choice: InventoryRecord | None = None @@ -330,8 +346,8 @@ def __init__( @property def input_ids(self) -> tuple[str, ...]: - """The reserved execution input nonces.""" - return self._base.records + """The reserved execution input nonces, without the fee record.""" + return self._inputs def use_fee_candidates(self, available: Sequence[InventoryRecord], preferred: str | None = None) -> FeeRecordSelector: """Prepare private-fee selection from the credits records available now. @@ -339,10 +355,11 @@ def use_fee_candidates(self, available: Sequence[InventoryRecord], preferred: st The returned callable is handed to the prover as ``fee_record``. It runs once the execution is proved and the exact fee is known, picks the smallest sufficient record (the preferred nonce first when it is - eligible), and reserves it as its own journal entry so a rejected - execution can keep it held while releasing the inputs. + eligible), and adds it to this spend's journal entry as its + ``fee_record_id``, so a rejected execution keeps the fee record held + while releasing the inputs. """ - inputs = set(self._base.records) + inputs = set(self._inputs) self._fee_candidates = sorted( (item for item in available if item.id not in inputs and item.plaintext and item.tag), key=lambda item: (item.id != preferred, item.amount, item.id), @@ -356,11 +373,23 @@ def select(required: int) -> str: for candidate in self._fee_candidates: if candidate.amount < required: continue - fee_entry = self._store.acquire(replace(self._base, records=(candidate.id,), fee_record_id=candidate.id)) - if fee_entry is not None: - self.entries.append(fee_entry) - self._fee_choice = candidate - return candidate.plaintext + # The store only reserves through ``acquire``, so win the fee + # record with a short-lived entry first, then fold it into the + # entry that holds the inputs and cancel the short-lived one. + # Between the two updates both entries hold the record for this + # spend; a crash there leaves a duplicate hold, never a gap. + claim = self._store.acquire(replace(self.entry, records=(candidate.id,), fee_record_id=candidate.id)) + if claim is None: + continue + try: + self._store.update(self._scope, self.entry.id, { + "records": (*self._inputs, candidate.id), "fee_record_id": candidate.id, + }) + finally: + self._store.update(self._scope, claim.id, {"status": "cancelled"}) + self.entry = replace(self.entry, records=(*self._inputs, candidate.id), fee_record_id=candidate.id) + self._fee_choice = candidate + return candidate.plaintext raise InventoryError( "No available credits record covers the private fee separately from the transaction inputs; " "refresh records or fund private credits" @@ -398,51 +427,55 @@ def prepared(self, tx_id: str, transaction: Mapping[str, Any], fee_microcredits: raise InventoryError("Private fee exceeds the reserved credits record balance") if self._fee_limit is not None and fee > self._fee_limit: raise InventoryError("Proved transaction exceeds max_fee_microcredits") - for item in self.entries: - self._store.update(self._scope, item.id, { - "status": "prepared", "transaction_id": tx_id, "transaction": dict(transaction), "fee_microcredits": str(fee), - }) + self._store.update(self._scope, self.entry.id, { + "status": "prepared", "transaction_id": tx_id, "transaction": dict(transaction), "fee_microcredits": str(fee), + }) self.transaction_id = tx_id self.is_prepared = True return fee def submitted(self, tx_id: str) -> None: """Record that the network accepted the broadcast; the reservation now outlives any error.""" + if self.is_submitted and self.transaction_id == tx_id: + return # Mark before touching the store: a storage failure must not release # inputs the network may already have accepted. self.is_submitted = True self.transaction_id = tx_id - for item in self.entries: - self._store.update(self._scope, item.id, {"status": "submitted", "transaction_id": tx_id}) + self._store.update(self._scope, self.entry.id, {"status": "submitted", "transaction_id": tx_id}) def confirmed(self, tx_id: str) -> None: """Record acceptance and drop the saved transaction.""" - for item in self.entries: - self._store.update(self._scope, item.id, {"status": "confirmed", "transaction_id": tx_id, "transaction": None}) + self._store.update(self._scope, self.entry.id, {"status": "confirmed", "transaction_id": tx_id, "transaction": None}) def broadcast_failed(self, error: BaseException) -> None: - """Decide what a failed broadcast means for the reservation. - - A 4xx from the broadcast endpoint means the node refused the - transaction before admitting it, so nothing can reach the chain and - the inputs are released. Any other failure (timeout, lost response, - 5xx) is uncertain and keeps them for reconciliation. + """Decide what a failed broadcast attempt means for the reservation. + + A 4xx from the broadcast endpoint on the first attempt means the node + refused the transaction before admitting it, so nothing can reach the + chain and the inputs are released. Any other failure (timeout, lost + response, 5xx) is uncertain: the transaction may be in the node, so + the reservation is kept for reconciliation and stays kept even if a + later attempt is refused, since the earlier one may have gone through. """ status = getattr(error, "status", None) - if isinstance(error, AleoNetworkError) and isinstance(status, int) and 400 <= status < 500: + refused = isinstance(error, AleoNetworkError) and isinstance(status, int) and 400 <= status < 500 + if refused and not self.is_uncertain: self.discard() + else: + self.is_uncertain = True def discard(self) -> None: - """Release the reservation of a spend that was never submitted. + """Release the reservation of a spend that never reached the network. - Safe to call more than once. Does nothing after :meth:`submitted`, - because the network may have spent the records. + Safe to call more than once. Does nothing after :meth:`submitted` + or after a broadcast attempt with an unknown outcome, because the + network may have spent the records. """ - if self.is_submitted or self.is_closed: + if self.is_submitted or self.is_uncertain or self.is_closed: return self.is_closed = True - for item in self.entries: - self._store.update(self._scope, item.id, {"status": "cancelled"}) + self._store.update(self._scope, self.entry.id, {"status": "cancelled"}) def open_spend( @@ -665,7 +698,7 @@ def _spend( if observed is not None: observed.append(prepared.id) try: - tx_id = yield Broadcast(prepared.transaction) + tx_id = yield Broadcast(prepared.transaction, checkpoint) except BaseException as exc: checkpoint.broadcast_failed(exc) raise @@ -766,7 +799,7 @@ def check() -> None: tx_id = yield from self.split_record(plan.asset, inputs[0], step.amount, fee_limit=limit, observed=transaction_ids) if tx_id not in transaction_ids: transaction_ids.append(tx_id) - entry = next((item for item in store.list(plan.scope) if item.transaction_id == tx_id and item.fee_record_id is None), None) + entry = next((item for item in store.list(plan.scope) if item.transaction_id == tx_id), None) if entry is None: raise InventoryError("Submitted transaction is missing its reservation") fees += int(entry.fee_microcredits or "0") @@ -825,10 +858,13 @@ def reconcile_record_inventory(self, *, rebroadcast: bool = False, release_reser if entry.status in ("confirmed", "rejected", "cancelled"): continue if entry.transaction_id is None: - # A reserved entry never reached the prepared checkpoint, so - # nothing was broadcast. It is released only on request, because - # another process may still be proving against it. - if release_reserved and entry.status == "reserved": + # A reserved entry from an SDK-held key never reached the + # prepared checkpoint, so nothing was broadcast. It is released + # only on request, because another process may still be proving + # against it. A wallet client's entry (Veil's ``rpc`` account + # type) is kept: the wallet may have broadcast before its + # response was lost. + if release_reserved and entry.status == "reserved" and entry.account_type == "local": store.update(scope, entry.id, {"status": "cancelled"}) continue # Only a definitive chain result changes the reservation; a missing diff --git a/sdk/python/aleo/inventory/client.py b/sdk/python/aleo/inventory/client.py index b967f90e..64f8a2ba 100644 --- a/sdk/python/aleo/inventory/client.py +++ b/sdk/python/aleo/inventory/client.py @@ -14,6 +14,7 @@ from contextlib import contextmanager from typing import Any, Awaitable, Callable, Generator, Literal, Mapping, Sequence, cast +from .._client_common import async_retry_with_backoff, retry_with_backoff from .._scanner_common import net_module from ..facade.errors import ProgramNotFound, TransactionNotFound from ._engine import ( @@ -155,9 +156,62 @@ def add(program_id: str) -> None: add(program_id) +def prepared_from_transaction(transaction: Any) -> PreparedTransaction: + """A proved network ``Transaction`` that has not been broadcast, ready for the journal.""" + return PreparedTransaction(str(transaction.id), json.loads(str(transaction.to_json()))) + + def prepared_from_result(result: Any) -> PreparedTransaction: """The unbroadcast transaction behind a facade ``TransactionResult``, ready for the journal.""" - return PreparedTransaction(str(result.id), json.loads(str(result.raw.to_json()))) + return prepared_from_transaction(result.raw) + + +class _Submission: + """A serialised transaction paired with its checkpoint, in the shape ``network.submit_transaction`` reads.""" + + __slots__ = ("raw", "checkpoint") + + def __init__(self, raw: str, checkpoint: SpendCheckpoint | None) -> None: + self.raw = raw + self.checkpoint = checkpoint + + def __str__(self) -> str: + return self.raw + + +def broadcast_checkpointed(submit_once: Callable[[], Any], checkpoint: Any) -> str: + """Broadcast a managed transaction, reporting every attempt to its checkpoint. + + *submit_once* performs one broadcast without retrying. Transient node + errors (5xx) are retried here with the SDK's usual backoff, and the + checkpoint learns of each failed attempt first, so a refusal that follows + an attempt with an unknown outcome keeps the reservation instead of + releasing it. A successful answer is recorded as submitted. + """ + def attempt() -> str: + try: + return str(submit_once()) + except BaseException as exc: + checkpoint.broadcast_failed(exc) + raise + + tx_id = retry_with_backoff(attempt) + checkpoint.submitted(tx_id) + return tx_id + + +async def async_broadcast_checkpointed(submit_once: Callable[[], Awaitable[Any]], checkpoint: Any) -> str: + """Awaited twin of :func:`broadcast_checkpointed`.""" + async def attempt() -> str: + try: + return str(await submit_once()) + except BaseException as exc: + checkpoint.broadcast_failed(exc) + raise + + tx_id = str(await async_retry_with_backoff(attempt)) + checkpoint.submitted(tx_id) + return tx_id def _json_dict(value: Any) -> dict[str, Any]: @@ -188,10 +242,6 @@ def managed_inventory(aleo: Any) -> Any: return inventory if inventory is not None and getattr(inventory, "enabled", False) else None -_prepared_from_transaction = prepared_from_result -_prepared_from_payload = prepared_from_payload - - def _scanner_synced(status: Any) -> bool: report = _json_dict(status) return bool(report.get("ok")) and bool(_json_dict(report.get("data")).get("synced")) @@ -244,7 +294,9 @@ def _perform(self, request: Request, account: Any, builder: Builder | None) -> A if isinstance(request, BuildTransaction): return builder(request) if builder is not None else self._build(request, account) if isinstance(request, Broadcast): - return str(self._aleo.network.submit_transaction(json.dumps(request.transaction))) + # The facade's broadcast reports each attempt to the checkpoint; + # the engine records the final outcome itself for other drivers. + return str(self._aleo.network.submit_transaction(_Submission(json.dumps(request.transaction), request.checkpoint))) if isinstance(request, ConfirmedTransaction): try: return self._aleo.network.get_confirmed_transaction(request.tx_id) @@ -264,15 +316,19 @@ def _perform(self, request: Request, account: Any, builder: Builder | None) -> A def _build(self, request: BuildTransaction, account: Any) -> PreparedTransaction: self._register_programs([request.program, *request.imports]) bound = self._aleo.programs.get(request.program).functions[request.function](*request.inputs) + # The engine has already reserved this call's inputs, so prove through + # the facade's unmanaged paths: ``build_transaction`` would try to + # reserve them a second time, and ``delegate`` is only managed when it + # broadcasts. This is the Python form of Veil's ``withoutManagement``. if self.proving == "local": - result = bound.build_transaction( + transaction = bound._prove_transaction( # pyright: ignore[reportPrivateUsage] account, priority_fee=self.priority_fee, fee_record=request.fee_record, private_fee=request.private_fee, ) - return _prepared_from_transaction(result) + return prepared_from_transaction(transaction) payload = bound.delegate( account, broadcast=False, pay_own_fee=self.pay_own_fee, fee_record=request.fee_record, priority_fee=self.priority_fee, ) - return _prepared_from_payload(payload) + return prepared_from_payload(payload) def _register_programs(self, program_ids: Sequence[str]) -> None: if getattr(self._aleo, "process", None) is None: @@ -300,7 +356,8 @@ def begin_spend( once the exact fee is known (*fee_record* is preferred when eligible). Call ``checkpoint.prepared`` after proving, pass the result to ``aleo.network.submit_transaction`` to record the broadcast, or - ``checkpoint.discard()`` to release a spend that is never sent. + ``checkpoint.discard()`` to release a spend that was never sent; + after a broadcast attempt, ``discard`` does nothing. Raises: RecordReservationConflictError: An input is already held. @@ -575,7 +632,7 @@ async def _perform(self, request: Request, account: Any, builder: AsyncBuilder | built = builder(request) return await built if inspect.isawaitable(built) else built if isinstance(request, Broadcast): - return str(await self._aleo.network.submit_transaction(json.dumps(request.transaction))) + return str(await self._aleo.network.submit_transaction(_Submission(json.dumps(request.transaction), request.checkpoint))) if isinstance(request, ConfirmedTransaction): try: return await self._aleo.network.get_confirmed_transaction(request.tx_id) @@ -596,15 +653,16 @@ async def _build(self, request: BuildTransaction, account: Any) -> PreparedTrans await self._register_programs([request.program, *request.imports]) program = await self._aleo.programs.get(request.program) bound = program.functions[request.function](*request.inputs) + # Unmanaged proving paths only; see RecordInventory._build. if self.proving == "local": - result = await bound.build_transaction( + transaction = await bound._prove_transaction( # pyright: ignore[reportPrivateUsage] account, priority_fee=self.priority_fee, fee_record=request.fee_record, private_fee=request.private_fee, ) - return _prepared_from_transaction(result) + return prepared_from_transaction(transaction) payload = await bound.delegate( account, broadcast=False, pay_own_fee=self.pay_own_fee, fee_record=request.fee_record, priority_fee=self.priority_fee, ) - return _prepared_from_payload(payload) + return prepared_from_payload(payload) async def _register_programs(self, program_ids: Sequence[str]) -> None: if getattr(self._aleo, "process", None) is None: diff --git a/sdk/python/aleo/inventory/store.py b/sdk/python/aleo/inventory/store.py index d4ffc248..282839c3 100644 --- a/sdk/python/aleo/inventory/store.py +++ b/sdk/python/aleo/inventory/store.py @@ -16,9 +16,9 @@ Anything else that satisfies :class:`~aleo.inventory.types.RecordInventoryStore` can replace them, such as a database shared across machines. The two -file-backed stores also release reservations left by a process on the same -machine that exited before proving, since such a reservation cannot have -reached the network. +file-backed stores also release reservations left by an SDK process on the +same machine that exited before proving, since such a reservation cannot +have reached the network; reservations written by a wallet client are kept. """ from __future__ import annotations @@ -114,15 +114,7 @@ def update(self, scope: str, entry_id: str, patch: RecordSpendPatch) -> None: with self._lock: for entry in self._entries: if entry.scope == scope and entry.id == entry_id: - if "status" in patch: - entry.status = patch["status"] - if "transaction_id" in patch: - entry.transaction_id = patch["transaction_id"] - if "transaction" in patch: - # ``None`` removes a transaction that recovery no longer needs. - entry.transaction = copy.deepcopy(patch["transaction"]) - if "fee_microcredits" in patch: - entry.fee_microcredits = patch["fee_microcredits"] + _apply_patch(entry, patch) return raise KeyError("Unknown record reservation") @@ -141,11 +133,17 @@ def reserved_record_ids(store: RecordInventoryStore | None, scope: str) -> set[s def _apply_patch(entry: RecordSpend, patch: RecordSpendPatch) -> None: + """Copy every key present in *patch* onto *entry*; absent keys leave the field alone.""" if "status" in patch: entry.status = patch["status"] + if "records" in patch: + entry.records = tuple(patch["records"]) + if "fee_record_id" in patch: + entry.fee_record_id = patch["fee_record_id"] if "transaction_id" in patch: entry.transaction_id = patch["transaction_id"] if "transaction" in patch: + # ``None`` removes a transaction that recovery no longer needs. entry.transaction = copy.deepcopy(patch["transaction"]) if "fee_microcredits" in patch: entry.fee_microcredits = patch["fee_microcredits"] @@ -218,8 +216,11 @@ class _DurableStore: Each entry is stored with the process id and host that wrote it. On every read under *scope*, a ``reserved`` entry written on this host by a process - that no longer runs is cancelled: the prepared checkpoint is always - written before broadcast, so such an entry never reached the network. + that no longer runs is cancelled when its ``account_type`` is ``local``: + an SDK-held key always writes the prepared checkpoint before broadcast, + so such an entry never reached the network. An entry a wallet client + wrote (Veil's ``rpc`` account type) is kept, because the wallet may have + broadcast before its response was lost. """ def __init__(self) -> None: @@ -242,7 +243,7 @@ def _transaction(self) -> Any: def _recover(self, scope: str) -> list[RecordSpend]: entries: list[RecordSpend] = [] for entry, pid, host in self._rows(scope): - if entry.status == "reserved" and host == self._host and not _process_alive(pid): + if entry.status == "reserved" and entry.account_type == "local" and host == self._host and not _process_alive(pid): entry.status = "cancelled" self._store(entry) entries.append(entry) diff --git a/sdk/python/aleo/inventory/types.py b/sdk/python/aleo/inventory/types.py index beaf2202..c0258f8a 100644 --- a/sdk/python/aleo/inventory/types.py +++ b/sdk/python/aleo/inventory/types.py @@ -455,9 +455,14 @@ class RecordSpendPatch(TypedDict, total=False): A key that is present replaces the stored value; ``transaction: None`` removes a stored transaction once it is no longer needed for recovery. + ``records`` and ``fee_record_id`` change together when a private fee + record is folded into the entry that holds the execution inputs, so one + entry carries everything a crash must keep together. """ status: RecordSpendStatus + records: tuple[str, ...] + fee_record_id: str | None transaction_id: str transaction: dict[str, Any] | None fee_microcredits: str @@ -472,7 +477,8 @@ class RecordInventoryStore(Protocol): Two guarantees are required. ``acquire`` MUST reserve every nonce in the entry or none of them, atomically with respect to other callers. Each - mutation MUST be finished before the method returns. Entries whose status + mutation MUST be finished before the method returns, and ``update`` MUST + apply every key present in the patch in one step. Entries whose status is not ``rejected`` or ``cancelled`` keep their records excluded from selection; a rejected entry keeps only its ``fee_record_id`` excluded. """ diff --git a/sdk/python/aleo/network_client.py b/sdk/python/aleo/network_client.py index 919a89a3..00945f93 100644 --- a/sdk/python/aleo/network_client.py +++ b/sdk/python/aleo/network_client.py @@ -336,6 +336,8 @@ def _post( body: str, method_name: str, extra_headers: dict[str, str] | None = None, + *, + retry: bool = True, ) -> requests.Response: hdrs = { **self._request_headers(method_name), @@ -352,7 +354,7 @@ def _do() -> requests.Response: ) return resp - return retry_with_backoff(_do) + return retry_with_backoff(_do) if retry else _do() # ── JWT refresh ─────────────────────────────────────────────────────── @@ -941,7 +943,7 @@ def get_deployment_transaction_for_program(self, program_id: str) -> Any: # ── POST endpoints ──────────────────────────────────────────────────── - def submit_transaction(self, transaction: Any) -> str: + def submit_transaction(self, transaction: Any, *, retry: bool = True) -> str: """Broadcast a proven transaction to the network. This spends the transaction's fee and is not reversible. Returning @@ -955,6 +957,9 @@ def submit_transaction(self, transaction: Any) -> str: Args: transaction: The proven transaction; stringified before broadcast, so either a ``Transaction`` or its serialized form works. + retry: Retry transient node errors (5xx) with backoff. Pass + ``False`` to make exactly one attempt, when the caller needs + to see each outcome itself. Returns: The broadcast transaction ID (``at1…``). @@ -969,7 +974,7 @@ def submit_transaction(self, transaction: Any) -> str: if self._verbose_errors else f"{self._host}/transaction/broadcast" ) - resp = self._post(endpoint, tx_str, "submitTransaction") + resp = self._post(endpoint, tx_str, "submitTransaction", retry=retry) return json.loads(resp.text) def submit_solution(self, solution: str) -> str: diff --git a/sdk/python/tests/test_facade_inventory.py b/sdk/python/tests/test_facade_inventory.py index d86330f2..090cbbef 100644 --- a/sdk/python/tests/test_facade_inventory.py +++ b/sdk/python/tests/test_facade_inventory.py @@ -88,7 +88,7 @@ def prove(self: BoundCall, acct: Any = None, *, priority_fee: int = 0, fee_recor # Replace only the proving step, so the managed build and submit halves run for real. monkeypatch.setattr(BoundCall, "_prove_transaction", prove) - def submit_transaction(raw: Any) -> str: + def submit_transaction(raw: Any, **_: Any) -> str: tx = json.loads(raw if isinstance(raw, str) else raw.to_json()) seen["submits"].append(tx) return str(tx["id"]) @@ -168,7 +168,7 @@ def test_a_node_refusal_on_submit_releases_the_reservation(monkeypatch: pytest.M from aleo import AleoNetworkError aleo, _, credits, _ = setup(monkeypatch) - def refuse(raw: Any) -> str: + def refuse(raw: Any, **_: Any) -> str: raise AleoNetworkError("POST /transaction/broadcast returned 400", status=400) monkeypatch.setattr(aleo._client, "submit_transaction", refuse) @@ -213,21 +213,66 @@ def test_fee_ceiling_is_enforced_before_broadcast(monkeypatch: pytest.MonkeyPatc assert aleo.inventory.reserved_record_ids() == set() -def test_private_fee_record_is_chosen_after_proving_and_reserved_beside_the_inputs(monkeypatch: pytest.MonkeyPatch) -> None: +def test_private_fee_record_is_chosen_after_proving_and_held_in_the_same_entry_as_the_inputs(monkeypatch: pytest.MonkeyPatch) -> None: aleo, _, credits, seen = setup(monkeypatch) transfer(credits, aleo).transact(private_fee=True, priority_fee=20) build = seen["builds"][0] assert build["private_fee"] is True and callable(build["fee_record"]) - inputs_entry, fee_entry = aleo.inventory.store.list(aleo.inventory.scope()) - assert (inputs_entry.records, inputs_entry.fee_record_id, inputs_entry.fee_microcredits) == (("1group",), None, "120") - assert (fee_entry.records, fee_entry.fee_record_id, fee_entry.status) == (("3group",), "3group", "submitted") + entry = aleo.inventory.store.list(aleo.inventory.scope())[0] + assert (entry.records, entry.fee_record_id, entry.fee_microcredits, entry.status) == (("1group", "3group"), "3group", "120", "submitted") assert build["tx"]["fee"]["transition"]["inputs"][0]["tag"] == "3field" + assert aleo.inventory.reserved_record_ids() == {"1group", "3group"} def test_supplied_fee_record_is_preferred_when_eligible(monkeypatch: pytest.MonkeyPatch) -> None: aleo, _, credits, _ = setup(monkeypatch, records=[1_000_000, 500, 200]) transfer(credits, aleo).transact(fee_record=aleo.record_provider.records[1]["record_plaintext"]) - assert aleo.inventory.store.list(aleo.inventory.scope())[1].fee_record_id == "2group" + assert aleo.inventory.store.list(aleo.inventory.scope())[0].fee_record_id == "2group" + + +def test_inventory_actions_prove_locally_on_a_managed_client_without_reserving_twice(monkeypatch: pytest.MonkeyPatch) -> None: + """The engine reserves the inputs itself, so the facade's managed build must not run again underneath it.""" + aleo, _, credits, seen = setup(monkeypatch, records=[1_000_000]) + monkeypatch.setattr(aleo.inventory, "proving", "local") + monkeypatch.setattr(aleo.programs, "get", lambda program_id: credits) + record = aleo.inventory.get_record_inventory(RecordAsset("credits.aleo")).available[0] + tx_id = aleo.inventory.split_record(RecordAsset("credits.aleo"), record, 400_000) + assert tx_id == "at1tx1" and [s["id"] for s in seen["submits"]] == ["at1tx1"] + (entry,) = aleo.inventory.store.list(aleo.inventory.scope()) + assert (entry.function, entry.records, entry.status) == ("split", ("1group",), "submitted") + + +def test_discard_keeps_the_reservation_after_a_broadcast_attempt_with_no_answer(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, _ = setup(monkeypatch) + result = transfer(credits, aleo).build_transaction() + + def lost(raw: Any, **_: Any) -> str: + raise TimeoutError("no response") + + monkeypatch.setattr(aleo._client, "submit_transaction", lost) + with pytest.raises(TimeoutError): + aleo.network.submit_transaction(result) + result.discard() # a cleanup path after the timeout + entry = aleo.inventory.store.list(aleo.inventory.scope())[0] + assert entry.status == "prepared" and aleo.inventory.reserved_record_ids() == {"1group"} + + +def test_a_refusal_after_a_retried_node_error_keeps_the_reservation(monkeypatch: pytest.MonkeyPatch) -> None: + """The raw client retries 5xx; the first attempt may have been admitted, so the later 4xx is not a clean refusal.""" + from aleo import AleoNetworkError + aleo, _, credits, _ = setup(monkeypatch) + answers = [AleoNetworkError("POST returned 503", status=503), AleoNetworkError("POST returned 400: exists", status=400)] + attempts: list[bool] = [] + + def flaky(raw: Any, *, retry: bool = True) -> str: + attempts.append(retry) + raise answers[len(attempts) - 1] + + monkeypatch.setattr(aleo._client, "submit_transaction", flaky) + with pytest.raises(AleoNetworkError, match="400"): + transfer(credits, aleo).transact() + assert attempts == [False, False] # one node attempt per report + assert aleo.inventory.store.list(aleo.inventory.scope())[0].status == "prepared" def test_managed_delegate_proves_without_broadcast_and_submits_from_the_client(monkeypatch: pytest.MonkeyPatch) -> None: @@ -250,7 +295,7 @@ def test_delegate_without_broadcast_stays_unmanaged(monkeypatch: pytest.MonkeyPa def test_lost_broadcast_response_keeps_the_prepared_transaction(monkeypatch: pytest.MonkeyPatch) -> None: aleo, _, credits, _ = setup(monkeypatch) - def lost(raw: Any) -> str: + def lost(raw: Any, **_: Any) -> str: raise ConnectionError("response lost") monkeypatch.setattr(aleo._client, "submit_transaction", lost) @@ -280,7 +325,7 @@ async def test_async_managed_transact_runs_the_same_hook(monkeypatch: pytest.Mon async def prove(self: AsyncBoundCall, acct: Any = None, **kw: Any) -> FakeTransaction: return FakeTransaction(fake_transaction("at1async")) - async def submit_transaction(raw: Any) -> str: + async def submit_transaction(raw: Any, **_: Any) -> str: submits.append(json.loads(raw if isinstance(raw, str) else raw.to_json())) return "at1async" diff --git a/sdk/python/tests/test_inventory_engine.py b/sdk/python/tests/test_inventory_engine.py index 5bba58a9..72254295 100644 --- a/sdk/python/tests/test_inventory_engine.py +++ b/sdk/python/tests/test_inventory_engine.py @@ -129,8 +129,10 @@ class _Network: def __init__(self, world: "World") -> None: self.world = world - def submit_transaction(self, raw: str) -> str: - return self.world.submit(json.loads(raw)) + def submit_transaction(self, raw: Any) -> str: + # The facade accepts a built result, a transaction, or a string; the + # node client stringifies whatever it is given. + return self.world.submit(json.loads(str(raw))) def get_confirmed_transaction(self, tx_id: str) -> dict[str, Any]: if self.world.on_confirmed is not None: @@ -288,10 +290,10 @@ def journal(self) -> list[RecordSpend]: assert self.store is not None return self.store.list(self.scope) - def hold(self, *nonces: str, fee: str | None = None, status: str = "reserved") -> RecordSpend | None: + def hold(self, *nonces: str, fee: str | None = None, status: str = "reserved", account_type: str = "local") -> RecordSpend | None: assert self.store is not None return self.store.acquire(RecordSpend(id="", scope=self.scope, records=nonces, program="credits.aleo", function="join", - status=status, created_at=1, fee_record_id=fee)) # type: ignore[arg-type] + status=status, created_at=1, fee_record_id=fee, account_type=account_type)) # type: ignore[arg-type] def available_amounts(self, asset: RecordAsset | None = None) -> list[int]: return sorted(r.amount for r in self.inventory.get_record_inventory(asset or self.asset).available) @@ -753,26 +755,34 @@ def builder(request: BuildTransaction) -> PreparedTransaction: return w.inventory.managed_write("credits.aleo", "transfer_private", spend, private_fee=True, fee_record=fee_record, builder=builder) -def test_selects_and_journals_a_separate_fee_record_including_priority_fees() -> None: +def spends(w: World) -> list[RecordSpend]: + """Journal entries that are spends, without the short-lived claims that win a fee record.""" + return [entry for entry in w.journal() if entry.status != "cancelled" or entry.transaction_id is not None] + + +def test_selects_the_fee_record_and_journals_it_in_the_same_entry_as_the_inputs() -> None: w = private_fee_world() managed(w, priority=20) - inputs_entry, fee_entry = w.journal() - assert (inputs_entry.records, inputs_entry.fee_record_id, inputs_entry.fee_microcredits, inputs_entry.status) == (("1group",), None, "120", "submitted") - assert (fee_entry.records, fee_entry.fee_record_id, fee_entry.status) == (("3group",), "3group", "submitted") + (entry,) = spends(w) + # One entry holds inputs and fee, as Veil writes it, so a crash between + # checkpoints can never keep the transaction while releasing its fee record. + assert (entry.records, entry.fee_record_id, entry.fee_microcredits, entry.status) == (("1group", "3group"), "3group", "120", "submitted") assert "microcredits:" not in json.dumps([entry.to_dict() for entry in w.journal()]) + claims = [entry for entry in w.journal() if entry not in spends(w)] + assert [(claim.records, claim.status) for claim in claims] == [(("3group",), "cancelled")] def test_uses_a_supplied_fee_record_when_eligible_and_replaces_a_reserved_or_input_one() -> None: w = private_fee_world() managed(w, fee_record=w.records[2]["record_plaintext"]) - assert w.journal()[1].fee_record_id == "3group" + assert spends(w)[-1].fee_record_id == "3group" w = private_fee_world() w.hold("3group", status="prepared") managed(w, fee_record=w.records[2]["record_plaintext"]) - assert w.journal()[-1].fee_record_id == "2group" + assert spends(w)[-1].fee_record_id == "2group" w = private_fee_world() managed(w, fee_record=w.records[0]["record_plaintext"]) - assert w.journal()[-1].fee_record_id == "2group" + assert spends(w)[-1].fee_record_id == "2group" def test_chooses_disjoint_fee_records_when_independent_writes_race() -> None: @@ -793,7 +803,7 @@ def run() -> None: for thread in threads: thread.join() assert errors == [] - assert {entry.fee_record_id for entry in w.journal() if entry.fee_record_id} == {"1group", "2group"} + assert {entry.fee_record_id for entry in spends(w)} == {"1group", "2group"} def test_excludes_reserved_records_and_fails_before_proving_when_nothing_covers_the_fee() -> None: @@ -845,7 +855,30 @@ def test_preserves_both_reservations_when_the_broadcast_outcome_is_unknown() -> w.submit_mode = "offline" with pytest.raises(ConnectionError, match="offline"): managed(w) - assert [(entry.status, entry.records) for entry in w.journal()] == [("prepared", ("1group",)), ("prepared", ("2group",))] + assert [(entry.status, entry.records) for entry in spends(w)] == [("prepared", ("1group", "2group"))] + assert w.inventory.reserved_record_ids() == {"1group", "2group"} + + +def test_the_fee_record_is_never_held_apart_from_the_inputs_once_the_transaction_is_saved() -> None: + """Every checkpoint after fee selection is one store update (the reviewer's crash probe).""" + w = private_fee_world() + updates: list[tuple[str, dict[str, Any]]] = [] + store = w.store + assert store is not None + original = store.update + + def record_update(scope: str, entry_id: str, patch: Any) -> None: + updates.append((entry_id, dict(patch))) + original(scope, entry_id, patch) + + store.update = record_update # type: ignore[method-assign] + managed(w) + (entry,) = spends(w) + # From the prepared checkpoint on, only the spend's own entry is touched, + # and it already carries the fee record when the transaction is saved. + after_fee = [(entry_id, patch) for entry_id, patch in updates if "transaction" in patch or patch.get("status") == "submitted"] + assert after_fee and all(entry_id == entry.id for entry_id, _ in after_fee) + assert entry.records == ("1group", "2group") and entry.fee_record_id == "2group" def test_a_nested_conflict_after_acquisition_is_not_reported_as_retryable() -> None: @@ -904,7 +937,7 @@ async def find(self, account: Any = None, **kw: Any) -> list[dict[str, Any]]: return outer._fake.record_provider.find(account, **kw) class _N: - async def submit_transaction(self, raw: str) -> str: + async def submit_transaction(self, raw: Any) -> str: return outer._fake.network.submit_transaction(raw) async def get_confirmed_transaction(self, tx_id: str) -> dict[str, Any]: @@ -959,6 +992,31 @@ def test_reconcile_releases_reserved_entries_only_on_request() -> None: assert w.journal()[0].status == "cancelled" and w.inventory.reserved_record_ids() == set() +def test_reconcile_keeps_a_wallet_clients_reservation_even_on_request() -> None: + w = World([1_000_000]) + w.hold("1group", account_type="rpc") # a Veil wallet entry in a shared journal + w.inventory.reconcile_record_inventory(release_reserved=True) + # The wallet may have broadcast before its response was lost; only the + # SDK's own reserved entries provably never reached the network. + assert w.journal()[0].status == "reserved" and w.inventory.reserved_record_ids() == {"1group"} + + +def test_a_refusal_after_an_uncertain_broadcast_attempt_keeps_the_reservation() -> None: + from aleo import AleoNetworkError + w = World([1_000_000]) + record = w.inventory.get_record_inventory(CREDITS).available[0] + with w.inventory.reserve("credits.aleo", "split", [record.id]) as checkpoint: + checkpoint.prepared("at1x", w.build("credits.aleo", "split", (record.plaintext, "1u64"), None, 0)) + checkpoint.broadcast_failed(AleoNetworkError("POST returned 503", status=503)) + assert checkpoint.is_uncertain and not checkpoint.is_closed + checkpoint.broadcast_failed(AleoNetworkError("POST returned 400: already exists", status=400)) + checkpoint.discard() # cleanup after a timeout must not release it + assert w.journal()[0].status == "prepared" and w.inventory.reserved_record_ids() == {record.id} + with w.inventory.reserve("credits.aleo", "split", ["2000group"]) as fresh: + fresh.broadcast_failed(AleoNetworkError("POST returned 400", status=400)) + assert w.journal()[-1].status == "cancelled" # a first-attempt refusal still releases + + def test_rebalance_stops_before_proving_once_the_fee_budget_is_spent() -> None: w = World([100] * 3) plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=1)) diff --git a/sdk/python/tests/test_inventory_store.py b/sdk/python/tests/test_inventory_store.py index 6bdb2d25..83fd563a 100644 --- a/sdk/python/tests/test_inventory_store.py +++ b/sdk/python/tests/test_inventory_store.py @@ -75,6 +75,10 @@ def test_update_patches_fields_and_clears_transactions(make_store: Factory) -> N assert (stored.status, stored.transaction_id, stored.transaction, stored.fee_microcredits) == ("prepared", "at1x", {"id": "at1x"}, "100") store.update(SCOPE, saved.id, {"status": "confirmed", "transaction": None}) assert make_store().list(SCOPE)[0].transaction is None + # Folding a fee record into the entry changes both fields in one update. + store.update(SCOPE, saved.id, {"records": ("1group", "7group"), "fee_record_id": "7group"}) + folded = make_store().list(SCOPE)[0] + assert (folded.records, folded.fee_record_id) == (("1group", "7group"), "7group") with pytest.raises(KeyError): store.update(SCOPE, "99", {"status": "cancelled"}) @@ -168,11 +172,18 @@ def test_durable_stores_release_reservations_of_dead_local_processes(durable: tu prepared = live.acquire(entry("2group")) assert prepared is not None live.update(SCOPE, prepared.id, {"status": "prepared", "transaction_id": "at1y", "transaction": {"id": "at1y"}}) + wallet = make() + wallet._pid = dead_pid() # pyright: ignore[reportPrivateUsage] + rpc = entry("3group") + rpc.account_type = "rpc" # a Veil wallet client's entry + assert wallet.acquire(rpc) is not None journal = make().list(SCOPE) - # The dead writer's reserved entry never reached the network; a prepared - # entry from a dead writer would be kept, and anything from a live one is. - assert [(e.records, e.status) for e in journal] == [(("1group",), "cancelled"), (("2group",), "prepared")] + # The dead SDK writer's reserved entry never reached the network; a prepared + # entry from a dead writer is kept, anything from a live one is kept, and a + # wallet's reserved entry is kept because the wallet may have broadcast. + assert [(e.records, e.status) for e in journal] == [(("1group",), "cancelled"), (("2group",), "prepared"), (("3group",), "reserved")] assert make().acquire(entry("1group")) is not None + assert make().acquire(entry("3group")) is None def dead_pid() -> int: diff --git a/shield-swap-sdk/tests/test_inventory.py b/shield-swap-sdk/tests/test_inventory.py index fa258cdf..4f8c6c6e 100644 --- a/shield-swap-sdk/tests/test_inventory.py +++ b/shield-swap-sdk/tests/test_inventory.py @@ -196,7 +196,8 @@ def __init__(self, record_list: list[dict[str, Any]], *, store: Any = None, enab def submit(raw: Any) -> str: checkpoint = getattr(raw, "checkpoint", None) if isinstance(raw, str) or checkpoint is not None: - tx = json.loads(raw if isinstance(raw, str) else raw.raw.to_json()) + payload = raw if isinstance(raw, str) else raw.raw + tx = json.loads(payload if isinstance(payload, str) else payload.to_json()) self.submitted_json.append(tx) if checkpoint is not None: checkpoint.submitted(tx["id"]) From c553c9c5cbbed07d70d7830b9926225aef0d27b9 Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 11:19:47 -0400 Subject: [PATCH 08/11] fix: hold a reservation while its broadcast is in flight - SpendCheckpoint marks a broadcast attempt before the request leaves (begin_broadcast); discard() is inert until the attempt resolves, and a checkpoint that was already discarded refuses to broadcast, since its inputs may belong to another transaction by then. The engine and both facade broadcast helpers set the mark; a lock serialises the lifecycle flags against a discard from another thread. - Durable stores no longer chmod an existing parent directory; only a directory they create becomes owner-only. - The file lock and the JSON journal guard release what they took when acquisition or decoding fails, so a repaired journal works without a restart. - Root .gitignore excludes .shield-swap/ (the examples write claim secrets there when run from the repository root), and the Shield Swap README's concurrent-swap snippet attaches the swap journal. --- .gitignore | 2 + sdk/python/aleo/facade/call.py | 10 ++-- sdk/python/aleo/inventory/_engine.py | 68 +++++++++++++++++------ sdk/python/aleo/inventory/client.py | 7 ++- sdk/python/aleo/inventory/store.py | 54 +++++++++++++----- sdk/python/tests/test_facade_inventory.py | 58 +++++++++++++++++++ sdk/python/tests/test_inventory_engine.py | 16 ++++++ sdk/python/tests/test_inventory_store.py | 30 ++++++++++ shield-swap-sdk/README.md | 6 ++ 9 files changed, 213 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index 6db8505e..269e0d82 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ tmp/ .superpowers/ docs/superpowers/ .worktrees/ +# Shield Swap journals (claim secrets) written by the examples when run from the repo root +.shield-swap/ diff --git a/sdk/python/aleo/facade/call.py b/sdk/python/aleo/facade/call.py index 7335bbf4..837651b3 100644 --- a/sdk/python/aleo/facade/call.py +++ b/sdk/python/aleo/facade/call.py @@ -132,10 +132,12 @@ def discard(self) -> None: """Release the record reservation of a built transaction that will not be submitted. Only meaningful on a managed client. It does nothing once - ``submit_transaction`` has been attempted, whether the node accepted - the transaction or the attempt ended without an answer, because the - network may have spent the records; use - ``reconcile_record_inventory`` to settle such a spend. + ``submit_transaction`` has started, whether the attempt is still in + flight, the node accepted the transaction, or the attempt ended + without an answer, because the network may have spent the records; + use ``reconcile_record_inventory`` to settle such a spend. A result + that was discarded is refused by ``submit_transaction`` afterwards, + since its records may already belong to another transaction. """ if self.checkpoint is not None: self.checkpoint.discard() diff --git a/sdk/python/aleo/inventory/_engine.py b/sdk/python/aleo/inventory/_engine.py index cb55bc92..97ef476e 100644 --- a/sdk/python/aleo/inventory/_engine.py +++ b/sdk/python/aleo/inventory/_engine.py @@ -19,6 +19,7 @@ from __future__ import annotations import re +import threading import time from dataclasses import dataclass, replace from typing import Any, Callable, Generator, Literal, Mapping, Sequence, TypeVar, Union, cast @@ -321,6 +322,8 @@ class SpendCheckpoint: Attributes: entry: The journal entry holding the reservation. transaction_id: The id recorded at preparation or submission. + is_broadcasting: A broadcast attempt is in flight; the reservation + cannot be released until its outcome is known. is_uncertain: A broadcast attempt ended without a definite answer, so the network may hold the transaction; the reservation stays. """ @@ -337,9 +340,13 @@ def __init__( self.entry = entry self.transaction_id: str | None = None self.is_prepared = False + self.is_broadcasting = False self.is_submitted = False self.is_uncertain = False self.is_closed = False + # Serialises the lifecycle flags against a concurrent ``discard`` from + # another thread while a broadcast is in flight. + self._state = threading.RLock() self._fee_candidates: list[InventoryRecord] = [] self._fee_choice: InventoryRecord | None = None self.fee_record_selector: FeeRecordSelector | None = None @@ -434,15 +441,33 @@ def prepared(self, tx_id: str, transaction: Mapping[str, Any], fee_microcredits: self.is_prepared = True return fee + def begin_broadcast(self) -> None: + """Mark a broadcast attempt as in flight, before the request is sent. + + From here until :meth:`submitted` or :meth:`broadcast_failed`, + :meth:`discard` does nothing, so a concurrent cleanup cannot release + inputs the node may be accepting at that moment. + + Raises: + InventoryError: The spend was already discarded; its inputs may + belong to another transaction now, so it must not be sent. + """ + with self._state: + if self.is_closed: + raise InventoryError("Transaction was discarded before broadcast; build it again") + self.is_broadcasting = True + def submitted(self, tx_id: str) -> None: """Record that the network accepted the broadcast; the reservation now outlives any error.""" - if self.is_submitted and self.transaction_id == tx_id: - return - # Mark before touching the store: a storage failure must not release - # inputs the network may already have accepted. - self.is_submitted = True - self.transaction_id = tx_id - self._store.update(self._scope, self.entry.id, {"status": "submitted", "transaction_id": tx_id}) + with self._state: + if self.is_submitted and self.transaction_id == tx_id: + return + # Mark before touching the store: a storage failure must not release + # inputs the network may already have accepted. + self.is_submitted = True + self.is_broadcasting = False + self.transaction_id = tx_id + self._store.update(self._scope, self.entry.id, {"status": "submitted", "transaction_id": tx_id}) def confirmed(self, tx_id: str) -> None: """Record acceptance and drop the saved transaction.""" @@ -460,22 +485,25 @@ def broadcast_failed(self, error: BaseException) -> None: """ status = getattr(error, "status", None) refused = isinstance(error, AleoNetworkError) and isinstance(status, int) and 400 <= status < 500 - if refused and not self.is_uncertain: - self.discard() - else: - self.is_uncertain = True + with self._state: + self.is_broadcasting = False + if refused and not self.is_uncertain: + self.discard() + else: + self.is_uncertain = True def discard(self) -> None: """Release the reservation of a spend that never reached the network. - Safe to call more than once. Does nothing after :meth:`submitted` - or after a broadcast attempt with an unknown outcome, because the - network may have spent the records. + Safe to call more than once. Does nothing while a broadcast is in + flight, after :meth:`submitted`, or after a broadcast attempt with an + unknown outcome, because the network may have spent the records. """ - if self.is_submitted or self.is_uncertain or self.is_closed: - return - self.is_closed = True - self._store.update(self._scope, self.entry.id, {"status": "cancelled"}) + with self._state: + if self.is_broadcasting or self.is_submitted or self.is_uncertain or self.is_closed: + return + self.is_closed = True + self._store.update(self._scope, self.entry.id, {"status": "cancelled"}) def open_spend( @@ -698,6 +726,10 @@ def _spend( if observed is not None: observed.append(prepared.id) try: + # Marked in flight here as well as in the facade's broadcast, + # so a driver that performs the request itself gets the same + # protection against a concurrent discard. + checkpoint.begin_broadcast() tx_id = yield Broadcast(prepared.transaction, checkpoint) except BaseException as exc: checkpoint.broadcast_failed(exc) diff --git a/sdk/python/aleo/inventory/client.py b/sdk/python/aleo/inventory/client.py index 64f8a2ba..8fdaf91f 100644 --- a/sdk/python/aleo/inventory/client.py +++ b/sdk/python/aleo/inventory/client.py @@ -186,9 +186,13 @@ def broadcast_checkpointed(submit_once: Callable[[], Any], checkpoint: Any) -> s errors (5xx) are retried here with the SDK's usual backoff, and the checkpoint learns of each failed attempt first, so a refusal that follows an attempt with an unknown outcome keeps the reservation instead of - releasing it. A successful answer is recorded as submitted. + releasing it. The checkpoint is marked in flight before every attempt, + so ``discard`` cannot release the inputs while the node may be accepting + them, and a spend already discarded is refused rather than sent. A + successful answer is recorded as submitted. """ def attempt() -> str: + checkpoint.begin_broadcast() try: return str(submit_once()) except BaseException as exc: @@ -203,6 +207,7 @@ def attempt() -> str: async def async_broadcast_checkpointed(submit_once: Callable[[], Awaitable[Any]], checkpoint: Any) -> str: """Awaited twin of :func:`broadcast_checkpointed`.""" async def attempt() -> str: + checkpoint.begin_broadcast() try: return str(await submit_once()) except BaseException as exc: diff --git a/sdk/python/aleo/inventory/store.py b/sdk/python/aleo/inventory/store.py index 282839c3..8164b70d 100644 --- a/sdk/python/aleo/inventory/store.py +++ b/sdk/python/aleo/inventory/store.py @@ -170,12 +170,18 @@ def _process_alive(pid: int) -> bool: def _private_file(path: Path) -> None: - """Create *path* and its parent readable by the owner only, keeping an existing file's contents.""" - path.parent.mkdir(parents=True, exist_ok=True) - try: - path.parent.chmod(0o700) - except OSError: # pragma: no cover - not every filesystem honours modes - pass + """Create *path* readable by the owner only, keeping an existing file's contents. + + A parent directory this call creates is owner-only as well; an existing + directory keeps its permissions, since it may be the application's own + directory or the working directory. + """ + if not path.parent.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + try: + path.parent.chmod(0o700) + except OSError: # pragma: no cover - not every filesystem honours modes + pass if not path.exists(): fd = os.open(path, os.O_CREAT | os.O_WRONLY, 0o600) os.close(fd) @@ -190,13 +196,24 @@ def __init__(self, path: Path) -> None: def __enter__(self) -> None: self._thread_lock.acquire() - self._handle = open(self._path, "a+") - if sys.platform == "win32": # pragma: no cover - exercised on Windows only - import msvcrt - msvcrt.locking(self._handle.fileno(), msvcrt.LK_LOCK, 1) # type: ignore[attr-defined] - else: - import fcntl - fcntl.flock(self._handle, fcntl.LOCK_EX) + # ``__exit__`` never runs when ``__enter__`` raises, so release what + # was taken before an open or lock failure propagates; the next call + # then gets a fresh attempt instead of blocking on this thread lock. + try: + self._handle = open(self._path, "a+") + try: + if sys.platform == "win32": # pragma: no cover - exercised on Windows only + import msvcrt + msvcrt.locking(self._handle.fileno(), msvcrt.LK_LOCK, 1) # type: ignore[attr-defined] + else: + import fcntl + fcntl.flock(self._handle, fcntl.LOCK_EX) + except BaseException: + self._handle.close() + raise + except BaseException: + self._thread_lock.release() + raise def __exit__(self, *exc: object) -> None: try: @@ -311,8 +328,15 @@ def _transaction(self) -> Any: class _Guard: def __enter__(self) -> None: store._lock.__enter__() - text = store.path.read_text() or "[]" - store._cache = [dict(item) for item in json.loads(text)] + try: + text = store.path.read_text() or "[]" + store._cache = [dict(item) for item in json.loads(text)] + except BaseException: + # A journal that cannot be read must not leave the locks + # held; once the file is repaired the next call proceeds. + store._cache = None + store._lock.__exit__(None, None, None) + raise def __exit__(self, kind: object, exc: object, tb: object) -> None: try: diff --git a/sdk/python/tests/test_facade_inventory.py b/sdk/python/tests/test_facade_inventory.py index 090cbbef..ffe49786 100644 --- a/sdk/python/tests/test_facade_inventory.py +++ b/sdk/python/tests/test_facade_inventory.py @@ -257,6 +257,64 @@ def lost(raw: Any, **_: Any) -> str: assert entry.status == "prepared" and aleo.inventory.reserved_record_ids() == {"1group"} +def test_discard_during_an_in_flight_broadcast_does_not_release_the_inputs(monkeypatch: pytest.MonkeyPatch) -> None: + """Another thread's cleanup while the request is pending must not free inputs the node may be accepting.""" + aleo, _, credits, seen = setup(monkeypatch) + result = transfer(credits, aleo).build_transaction() + + def slow_node(raw: Any, **_: Any) -> str: + # The request is on the wire; a concurrent discard arrives now. + result.discard() + assert aleo.inventory.store.list(aleo.inventory.scope())[0].status == "prepared" + tx = json.loads(raw if isinstance(raw, str) else raw.to_json()) + seen["submits"].append(tx) + return str(tx["id"]) + + monkeypatch.setattr(aleo._client, "submit_transaction", slow_node) + assert aleo.network.submit_transaction(result) == "at1tx1" + entry = aleo.inventory.store.list(aleo.inventory.scope())[0] + assert entry.status == "submitted" and aleo.inventory.reserved_record_ids() == {"1group"} + with pytest.raises(RecordReservationConflictError): + transfer(credits, aleo).build_transaction() # the input is still held + + +def test_a_discarded_result_is_refused_by_submit(monkeypatch: pytest.MonkeyPatch) -> None: + aleo, _, credits, seen = setup(monkeypatch) + result = transfer(credits, aleo).build_transaction() + result.discard() + second = transfer(credits, aleo).build_transaction() # the record is free again + with pytest.raises(InventoryError, match="discarded"): + aleo.network.submit_transaction(result) # must not spend a record now held by `second` + assert seen["submits"] == [] and second.checkpoint is not None and second.checkpoint.is_prepared + + +@pytest.mark.asyncio +async def test_async_discard_during_an_in_flight_broadcast_does_not_release_the_inputs(monkeypatch: pytest.MonkeyPatch) -> None: + config = RecordManagementConfig(enabled=True, store=MemoryRecordInventoryStore()) + aleo = AsyncAleo(HTTPProvider(BASE), record_management=config) + account = aleo.account.from_private_key(PrivateKey.random()) + aleo.default_account = account + aleo.record_provider = AsyncProvider([record(str(account.address), 1_000_000, 1)]) + built: list[Any] = [] + + async def prove(self: AsyncBoundCall, acct: Any = None, **kw: Any) -> FakeTransaction: + return FakeTransaction(fake_transaction("at1async")) + + async def slow_node(raw: Any, **_: Any) -> str: + built[0].discard() # a concurrent task's cleanup mid-flight + return "at1async" + + monkeypatch.setattr(AsyncBoundCall, "_prove_transaction", prove) + monkeypatch.setattr(aleo.network_client, "submit_transaction", slow_node) + from aleo.facade.async_client import AsyncProgram + credits = AsyncProgram(aleo, RawProgram.from_source(CREDITS_SOURCE)) + call: Any = credits.functions.transfer_private(aleo.record_provider.records[0]["record_plaintext"], str(account.address), 1) + built.append(await call.build_transaction()) + assert await aleo.network.submit_transaction(built[0]) == "at1async" + entry = aleo.inventory.store.list(aleo.inventory.scope())[0] + assert entry.status == "submitted" and aleo.inventory.reserved_record_ids() == {"1group"} + + def test_a_refusal_after_a_retried_node_error_keeps_the_reservation(monkeypatch: pytest.MonkeyPatch) -> None: """The raw client retries 5xx; the first attempt may have been admitted, so the later 4xx is not a clean refusal.""" from aleo import AleoNetworkError diff --git a/sdk/python/tests/test_inventory_engine.py b/sdk/python/tests/test_inventory_engine.py index 72254295..0404f1e1 100644 --- a/sdk/python/tests/test_inventory_engine.py +++ b/sdk/python/tests/test_inventory_engine.py @@ -1017,6 +1017,22 @@ def test_a_refusal_after_an_uncertain_broadcast_attempt_keeps_the_reservation() assert w.journal()[-1].status == "cancelled" # a first-attempt refusal still releases +def test_a_checkpoint_in_flight_ignores_discard_and_a_discarded_one_refuses_to_broadcast() -> None: + w = World([1_000_000, 500]) + first, second = w.inventory.get_record_inventory(CREDITS).available + with w.inventory.reserve("credits.aleo", "split", [first.id]) as checkpoint: + checkpoint.prepared("at1x", w.build("credits.aleo", "split", (first.plaintext, "1u64"), None, 0)) + checkpoint.begin_broadcast() # the request is on the wire + checkpoint.discard() + assert checkpoint.is_broadcasting and not checkpoint.is_closed + checkpoint.submitted("at1x") + assert w.journal()[0].status == "submitted" and first.id in w.inventory.reserved_record_ids() + with w.inventory.reserve("credits.aleo", "split", [second.id]) as dropped: + pass # left the block: released + with pytest.raises(InventoryError, match="discarded"): + dropped.begin_broadcast() + + def test_rebalance_stops_before_proving_once_the_fee_budget_is_spent() -> None: w = World([100] * 3) plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=1)) diff --git a/sdk/python/tests/test_inventory_store.py b/sdk/python/tests/test_inventory_store.py index 83fd563a..9a0ab133 100644 --- a/sdk/python/tests/test_inventory_store.py +++ b/sdk/python/tests/test_inventory_store.py @@ -218,6 +218,36 @@ def test_sqlite_layout_matches_veil(tmp_path: Path) -> None: assert entries[1].records == ("9group",) and entries[1].transaction_id == "at1veil" +def test_durable_stores_keep_an_existing_directory_s_permissions(tmp_path: Path) -> None: + if sys.platform == "win32": + pytest.skip("POSIX permission bits") + existing = tmp_path / "app" + existing.mkdir(mode=0o755) + SqliteRecordInventoryStore(existing / "inventory.sqlite").close() + assert (existing.stat().st_mode & 0o777) == 0o755, "an application's own directory is left alone" + JsonFileRecordInventoryStore(existing / "nested" / "journal" / "inventory.json") + assert ((existing / "nested" / "journal").stat().st_mode & 0o077) == 0, "a directory the store creates is owner-only" + + +def test_json_store_releases_its_locks_when_the_journal_cannot_be_read(tmp_path: Path) -> None: + path = tmp_path / "inventory.json" + store = JsonFileRecordInventoryStore(path) + path.write_text("{not json") + with pytest.raises(json.JSONDecodeError): + store.list(SCOPE) + path.write_text("[]") + assert store.list(SCOPE) == [], "the repaired journal is usable without a restart" + # Nor does a failure to take the lock itself leave the thread lock held. + lock = store._lock # pyright: ignore[reportPrivateUsage] + lock_path = lock._path # pyright: ignore[reportPrivateUsage] + lock_path.unlink() + lock_path.mkdir() # opening a directory for append fails + with pytest.raises(OSError): + store.list(SCOPE) + assert lock._thread_lock.acquire(blocking=False) # pyright: ignore[reportPrivateUsage] + lock._thread_lock.release() # pyright: ignore[reportPrivateUsage] + + def test_json_file_is_plain_json_with_writer_metadata(tmp_path: Path) -> None: path = tmp_path / "inventory.json" store = JsonFileRecordInventoryStore(path) diff --git a/shield-swap-sdk/README.md b/shield-swap-sdk/README.md index 7dffcf22..4177a341 100644 --- a/shield-swap-sdk/README.md +++ b/shield-swap-sdk/README.md @@ -203,12 +203,18 @@ plans the joins and splits that reshape the token's underlying records, and time through `aleo.inventory`: ```python +from pathlib import Path + from aleo.inventory import InventoryTarget, RecordManagementConfig +from aleo_shield_swap.journal import Journal aleo = Aleo(Aleo.HTTPProvider("https://edge.provable.com/api", network="testnet"), record_management=RecordManagementConfig(enabled=True)) aleo.default_account = account dex = ShieldSwap(aleo) +# One journal per account and network, outside version control: it holds the +# claim secrets and hands each concurrent swap its own blinded identity. +dex.journal = Journal(Path.home() / ".shield-swap" / "testnet" / str(account.address) / "swaps.jsonl") plan = dex.plan_inventory("USDCx", InventoryTarget(records=4, min_record_amount=100_000, distribution="balanced")) result = dex.rebalance_inventory("USDCx", InventoryTarget(records=4, min_record_amount=100_000, distribution="balanced")) From 16c057e359e40701dcb45811a235cb1499b516d9 Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 12:06:59 -0400 Subject: [PATCH 09/11] fix: split whole lots when halving would strand value below the minimum The planner halved the largest record on every split, so a record small relative to the minimum could reject a reachable target: [9, 3] towards four lots of at least 3 went 9 -> [4, 5] -> [2, 3] and raised, and 20,003 microcredits could not fund three lots of 1 although two 10,000 deductions leave exactly enough. When the two halves could no longer yield the lots still required, the split now cuts a whole number of lots instead, which keeps every lot the record can fund. Halving still wins whenever it can finish, so Veil's known-answer plans are unchanged; only plans Veil rejects differ. Closes #83. --- sdk/python/aleo/inventory/planner.py | 25 +++++++++++++++++++--- sdk/python/tests/test_inventory_planner.py | 23 +++++++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/sdk/python/aleo/inventory/planner.py b/sdk/python/aleo/inventory/planner.py index 3c665a90..cd2d6d4c 100644 --- a/sdk/python/aleo/inventory/planner.py +++ b/sdk/python/aleo/inventory/planner.py @@ -4,8 +4,10 @@ produce the same steps, so a plan built for review can be rebuilt at execution time and compared. It follows one bounded strategy: keep records that already fit, split large records directly into the missing lots when -the amounts divide exactly, and otherwise consolidate and then split. Every -amount stays within the asset's declared width. +the amounts divide exactly, and otherwise consolidate and then split. A split +halves the record unless halving would strand value below the minimum, in +which case it cuts whole lots. Every amount stays within the asset's +declared width. """ from __future__ import annotations @@ -187,6 +189,11 @@ def push(kind: StepKind, consumed: Sequence[PlannedRecord], amounts: Sequence[in kind="join", inputs=tuple(item.id for item in consumed), outputs=(output,), join_program=helper, )) working = working[batch:] + [output] + def lots(amount: int) -> int: + # The most records of at least `minimum` that `amount` can still be + # split into, counting the deduction each further split costs. + return (amount + deduction) // (minimum + deduction) + # Split the largest remaining record and leave enough in its remainder # for any dependent splits, including their credits deductions. while len(working) < count: @@ -195,7 +202,19 @@ def push(kind: StepKind, consumed: Sequence[PlannedRecord], amounts: Sequence[in outputs_needed = count - len(working) + 1 if repack else 2 usable = record.amount - deduction * (outputs_needed - 1) amount = usable // outputs_needed - push("split", [record], [amount, record.amount - amount - deduction], amount) + remainder = record.amount - amount - deduction + # Halving a record that is small relative to the minimum can strand + # value below it: [9, 3] towards four lots of at least 3 would go + # 9 -> [4, 5] -> [2, 3]. When the two halves could no longer yield + # the lots still required, cut whole lots instead; a first output + # of exactly j lots keeps every lot the record can fund. (Veil's + # planner halves unconditionally and rejects these targets.) + required = count - sum(lots(item.amount) for item in working[1:]) + if lots(amount) + lots(remainder) < required: + whole = max(1, lots(record.amount) // 2) + amount = whole * (minimum + deduction) - deduction + remainder = record.amount - amount - deduction + push("split", [record], [amount, remainder], amount) if any(item.amount < minimum for item in working): raise InventoryPlanError("Insufficient balance after split deductions") return finish() diff --git a/sdk/python/tests/test_inventory_planner.py b/sdk/python/tests/test_inventory_planner.py index c3e69ed2..bda814e5 100644 --- a/sdk/python/tests/test_inventory_planner.py +++ b/sdk/python/tests/test_inventory_planner.py @@ -2,7 +2,9 @@ _source: ProvableHQ/veil packages/core/test/inventory/inventory.test.ts ("inventory planning") and configuration/autojoin suites, PR #172 @ 2e324d5. -Amounts and step shapes are copied as-is; the planner is a line-for-line port. +Amounts and step shapes are copied as-is; the planner is a line-for-line port +with one extension, the whole-lot split (python-sdk#83), which only changes +plans Veil would reject. """ from __future__ import annotations @@ -61,6 +63,25 @@ def test_preserves_exact_balanced_lots_and_repacks_small_records() -> None: assert [record.amount for record in result.steps[-1].outputs] == [2, 2] +def test_splits_whole_lots_when_halving_would_strand_value_below_the_minimum() -> None: + """Python extends Veil here: the same inputs raise in Veil's planner (python-sdk#83).""" + result = build_inventory_plan( + scope=SCOPE, asset=ARC20, inputs=[PlannedRecord("1group", 9), PlannedRecord("2group", 3)], + target=InventoryTarget(records=4, min_record_amount=3, distribution="preserve"), + ) + assert [[o.amount for o in step.outputs] for step in result.steps] == [[3, 6], [3, 3]] + # Credits: 20,003 microcredits fund two 10,000 deductions and three lots of 1, + # but a first halving would leave neither half able to pay the second split. + credits = build_inventory_plan( + scope=SCOPE, asset=CREDITS, inputs=[PlannedRecord("1group", 20_003)], + target=InventoryTarget(records=3, min_record_amount=1, distribution="preserve"), + ) + assert [[o.amount for o in step.outputs] for step in credits.steps] == [[1, 10_002], [1, 1]] + assert credits.deduction == 20_000 + # Halving still wins whenever it can finish, so existing plans are unchanged. + assert [[o.amount for o in step.outputs] for step in plan([100], 2, ARC20, "preserve").steps] == [[50, 50]] + + def test_rejects_impossible_counts_deductions_overflow_and_operation_limits() -> None: with pytest.raises(InventoryPlanError, match="infeasible"): plan([10_001], 2, CREDITS) From af3b2c7032a709db35ea9b9430cfc1d8ff300575 Mon Sep 17 00:00:00 2001 From: Michael Turner Date: Fri, 23 Oct 2026 12:44:04 -0400 Subject: [PATCH 10/11] fix: count overlapping broadcast attempts before releasing a reservation The in-flight mark was one boolean, so when the same built result was submitted twice at once, a 4xx on the second request (a duplicate, refused because the node had admitted the first) cleared the mark and cancelled the reservation while the first request was still pending. SpendCheckpoint now counts unresolved attempts: a refusal releases the inputs only when no other attempt of the spend is outstanding, success resolves them all, and anything uncertain still keeps the reservation. The engine's attempt wraps the facade's, so the count also covers that nesting. --- sdk/python/aleo/inventory/_engine.py | 27 +++++++++++++++++------ sdk/python/tests/test_facade_inventory.py | 26 ++++++++++++++++++++++ sdk/python/tests/test_inventory_engine.py | 23 +++++++++++++++++++ 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/sdk/python/aleo/inventory/_engine.py b/sdk/python/aleo/inventory/_engine.py index 97ef476e..e9d047cd 100644 --- a/sdk/python/aleo/inventory/_engine.py +++ b/sdk/python/aleo/inventory/_engine.py @@ -322,8 +322,8 @@ class SpendCheckpoint: Attributes: entry: The journal entry holding the reservation. transaction_id: The id recorded at preparation or submission. - is_broadcasting: A broadcast attempt is in flight; the reservation - cannot be released until its outcome is known. + is_broadcasting: At least one broadcast attempt is in flight; the + reservation cannot be released until every attempt has resolved. is_uncertain: A broadcast attempt ended without a definite answer, so the network may hold the transaction; the reservation stays. """ @@ -340,10 +340,13 @@ def __init__( self.entry = entry self.transaction_id: str | None = None self.is_prepared = False - self.is_broadcasting = False self.is_submitted = False self.is_uncertain = False self.is_closed = False + # Unresolved broadcast attempts. The same result may be submitted + # twice at once, and the engine's attempt wraps the facade's, so this + # is a count rather than a flag: the last attempt to resolve decides. + self._in_flight = 0 # Serialises the lifecycle flags against a concurrent ``discard`` from # another thread while a broadcast is in flight. self._state = threading.RLock() @@ -351,6 +354,11 @@ def __init__( self._fee_choice: InventoryRecord | None = None self.fee_record_selector: FeeRecordSelector | None = None + @property + def is_broadcasting(self) -> bool: + """Whether any broadcast attempt of this spend is still unresolved.""" + return self._in_flight > 0 + @property def input_ids(self) -> tuple[str, ...]: """The reserved execution input nonces, without the fee record.""" @@ -455,7 +463,7 @@ def begin_broadcast(self) -> None: with self._state: if self.is_closed: raise InventoryError("Transaction was discarded before broadcast; build it again") - self.is_broadcasting = True + self._in_flight += 1 def submitted(self, tx_id: str) -> None: """Record that the network accepted the broadcast; the reservation now outlives any error.""" @@ -465,7 +473,7 @@ def submitted(self, tx_id: str) -> None: # Mark before touching the store: a storage failure must not release # inputs the network may already have accepted. self.is_submitted = True - self.is_broadcasting = False + self._in_flight = 0 self.transaction_id = tx_id self._store.update(self._scope, self.entry.id, {"status": "submitted", "transaction_id": tx_id}) @@ -482,13 +490,18 @@ def broadcast_failed(self, error: BaseException) -> None: response, 5xx) is uncertain: the transaction may be in the node, so the reservation is kept for reconciliation and stays kept even if a later attempt is refused, since the earlier one may have gone through. + A refusal while another attempt of the same spend is still pending + releases nothing either: the pending one may be the request the node + admitted (a duplicate is refused for exactly that reason), so the + last attempt to resolve decides. """ status = getattr(error, "status", None) refused = isinstance(error, AleoNetworkError) and isinstance(status, int) and 400 <= status < 500 with self._state: - self.is_broadcasting = False + self._in_flight = max(0, self._in_flight - 1) if refused and not self.is_uncertain: - self.discard() + if self._in_flight == 0: + self.discard() else: self.is_uncertain = True diff --git a/sdk/python/tests/test_facade_inventory.py b/sdk/python/tests/test_facade_inventory.py index ffe49786..495a4a27 100644 --- a/sdk/python/tests/test_facade_inventory.py +++ b/sdk/python/tests/test_facade_inventory.py @@ -278,6 +278,32 @@ def slow_node(raw: Any, **_: Any) -> str: transfer(credits, aleo).build_transaction() # the input is still held +def test_a_duplicate_refusal_does_not_release_inputs_while_the_first_submission_is_pending(monkeypatch: pytest.MonkeyPatch) -> None: + """Two submits of one result overlap; the node refuses the second as a duplicate because it admitted the first.""" + from aleo import AleoNetworkError + aleo, _, credits, seen = setup(monkeypatch) + result = transfer(credits, aleo).build_transaction() + answers: list[str] = [] + + def node(raw: Any, **_: Any) -> str: + if answers: # the overlapping second request + raise AleoNetworkError("POST returned 400: transaction already exists", status=400) + answers.append("first") + with pytest.raises(AleoNetworkError): + aleo.network.submit_transaction(result) # arrives while the first is still pending + assert aleo.inventory.store.list(aleo.inventory.scope())[0].status == "prepared" + assert aleo.inventory.reserved_record_ids() == {"1group"} + tx = json.loads(raw if isinstance(raw, str) else raw.to_json()) + seen["submits"].append(tx) + return str(tx["id"]) + + monkeypatch.setattr(aleo._client, "submit_transaction", node) + assert aleo.network.submit_transaction(result) == "at1tx1" + assert aleo.inventory.store.list(aleo.inventory.scope())[0].status == "submitted" + with pytest.raises(RecordReservationConflictError): + transfer(credits, aleo).build_transaction() + + def test_a_discarded_result_is_refused_by_submit(monkeypatch: pytest.MonkeyPatch) -> None: aleo, _, credits, seen = setup(monkeypatch) result = transfer(credits, aleo).build_transaction() diff --git a/sdk/python/tests/test_inventory_engine.py b/sdk/python/tests/test_inventory_engine.py index 0404f1e1..d2328df4 100644 --- a/sdk/python/tests/test_inventory_engine.py +++ b/sdk/python/tests/test_inventory_engine.py @@ -1033,6 +1033,29 @@ def test_a_checkpoint_in_flight_ignores_discard_and_a_discarded_one_refuses_to_b dropped.begin_broadcast() +def test_overlapping_attempts_are_counted_and_the_last_to_resolve_decides() -> None: + from aleo import AleoNetworkError + w = World([1_000_000, 500]) + first, second = w.inventory.get_record_inventory(CREDITS).available + refusal = AleoNetworkError("POST returned 400: transaction already exists", status=400) + with w.inventory.reserve("credits.aleo", "split", [first.id]) as checkpoint: + checkpoint.prepared("at1x", w.build("credits.aleo", "split", (first.plaintext, "1u64"), None, 0)) + checkpoint.begin_broadcast() # request A on the wire + checkpoint.begin_broadcast() # request B overlaps it + checkpoint.broadcast_failed(refusal) # B is refused as a duplicate: A was admitted + assert checkpoint.is_broadcasting and not checkpoint.is_closed + checkpoint.submitted("at1x") # A succeeds + assert w.journal()[0].status == "submitted" + with w.inventory.reserve("credits.aleo", "split", [second.id]) as both_refused: + both_refused.begin_broadcast() + both_refused.begin_broadcast() + both_refused.broadcast_failed(refusal) + assert not both_refused.is_closed + both_refused.broadcast_failed(refusal) # nothing left pending: never admitted + assert both_refused.is_closed + assert w.journal()[-1].status == "cancelled" + + def test_rebalance_stops_before_proving_once_the_fee_budget_is_spent() -> None: w = World([100] * 3) plan = w.inventory.plan_record_inventory(CREDITS, InventoryTarget(records=1)) From 57e804402ab3bb39349cb837760359b85c05eba2 Mon Sep 17 00:00:00 2001 From: Mike Turner Date: Fri, 23 Oct 2026 12:06:17 -0500 Subject: [PATCH 11/11] chore: bump every package to 0.8.0 (#84) aleo-sdk, aleo-contract-abi-generator, shield-swap-sdk, and aleo-bridge-sdk move from 0.7.0 to 0.8.0 together, as before. shield-swap-sdk now imports aleo.inventory at module load, so its aleo-sdk floor rises to 0.8.0 and the import guard probes aleo.inventory instead of aleo.compliance. The bridge floor stays at 0.7.0; nothing there needs the new module. --- bridge-sdk/pyproject.toml | 2 +- bridge-sdk/python/aleo_bridge/__init__.py | 2 +- bridge-sdk/tests/test_client_lifecycle.py | 2 +- bridge-sdk/tests/test_package.py | 4 ++-- sdk-abi/Cargo.lock | 2 +- sdk-abi/Cargo.toml | 2 +- sdk-abi/pyproject.toml | 2 +- sdk/Cargo.lock | 2 +- sdk/Cargo.toml | 2 +- sdk/docs/source/conf.py | 2 +- sdk/pyproject.toml | 2 +- shield-swap-sdk/README.md | 2 +- shield-swap-sdk/pyproject.toml | 4 ++-- .../python/aleo_shield_swap/__init__.py | 14 +++++++------- shield-swap-sdk/tests/test_package.py | 8 ++++---- 15 files changed, 26 insertions(+), 26 deletions(-) diff --git a/bridge-sdk/pyproject.toml b/bridge-sdk/pyproject.toml index deed5020..b721516e 100644 --- a/bridge-sdk/pyproject.toml +++ b/bridge-sdk/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "aleo-bridge-sdk" -version = "0.7.0" +version = "0.8.0" description = "Bridge Aleo, Ethereum, Arc, Base, Arbitrum and Solana assets with Hyperlane, Circle xReserve and CCTP" readme = "README.md" requires-python = ">=3.10" diff --git a/bridge-sdk/python/aleo_bridge/__init__.py b/bridge-sdk/python/aleo_bridge/__init__.py index 175cefea..4bc73771 100644 --- a/bridge-sdk/python/aleo_bridge/__init__.py +++ b/bridge-sdk/python/aleo_bridge/__init__.py @@ -8,7 +8,7 @@ from pathlib import Path -__version__ = "0.7.0" +__version__ = "0.8.0" _ALEO_SDK_MINIMUM = "0.7.0" diff --git a/bridge-sdk/tests/test_client_lifecycle.py b/bridge-sdk/tests/test_client_lifecycle.py index 45341ace..ade27af2 100644 --- a/bridge-sdk/tests/test_client_lifecycle.py +++ b/bridge-sdk/tests/test_client_lifecycle.py @@ -205,7 +205,7 @@ def test_public_exports_pin_pre_existing_set_and_add_lifecycle_names(): assert hasattr(aleo_bridge, name), name assert set(PRE_EXISTING_EXPORTS) <= set(aleo_bridge.__all__) assert set(NEW_EXPORTS) <= set(aleo_bridge.__all__) - assert aleo_bridge.__version__ == "0.7.0" + assert aleo_bridge.__version__ == "0.8.0" assert aleo_bridge.lifecycle is lifecycle assert aleo_bridge.prepare is lifecycle.prepare assert aleo_bridge.bridge_tools is agent.bridge_tools diff --git a/bridge-sdk/tests/test_package.py b/bridge-sdk/tests/test_package.py index cbc48569..32a65527 100644 --- a/bridge-sdk/tests/test_package.py +++ b/bridge-sdk/tests/test_package.py @@ -13,7 +13,7 @@ def test_version_is_pinned_in_lockstep(): pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text()) - assert pyproject["project"]["version"] == "0.7.0" == aleo_bridge.__version__ + assert pyproject["project"]["version"] == "0.8.0" == aleo_bridge.__version__ assert pyproject["project"]["name"] == "aleo-bridge-sdk" deps = pyproject["project"]["dependencies"] for base in ("aleo-sdk", "pynacl", "web3", "eth-account", "solders", "solana", "privy-client", "dynamic-wallet-sdk"): @@ -105,7 +105,7 @@ def test_import_without_optional_extras(monkeypatch): monkeypatch.delitem(sys.modules, name) # reverted at teardown — a fresh reimport below must not # leak new module/class objects into tests that run after this one pkg = importlib.import_module("aleo_bridge") - assert pkg.__version__ == "0.7.0" + assert pkg.__version__ == "0.8.0" assert issubclass(pkg.RouteNotFoundError, pkg.BridgeError) diff --git a/sdk-abi/Cargo.lock b/sdk-abi/Cargo.lock index 89b19798..0f330384 100644 --- a/sdk-abi/Cargo.lock +++ b/sdk-abi/Cargo.lock @@ -19,7 +19,7 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aleo-abi" -version = "0.7.0" +version = "0.8.0" dependencies = [ "anyhow", "leo-abi", diff --git a/sdk-abi/Cargo.toml b/sdk-abi/Cargo.toml index 88a66ae4..7bd35f7f 100644 --- a/sdk-abi/Cargo.toml +++ b/sdk-abi/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "aleo-abi" -version = "0.7.0" +version = "0.8.0" edition = "2024" license = "GPL-3.0-or-later" description = "Python bindings for ABI generation from Aleo bytecode (via Leo's leo-abi crate)" diff --git a/sdk-abi/pyproject.toml b/sdk-abi/pyproject.toml index db444cda..b64c6045 100644 --- a/sdk-abi/pyproject.toml +++ b/sdk-abi/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "aleo-contract-abi-generator" -version = "0.7.0" +version = "0.8.0" description = "Python bindings for ABI generation from Aleo bytecode" readme = "README.md" license = {text = "GPL-3.0-or-later"} diff --git a/sdk/Cargo.lock b/sdk/Cargo.lock index 71584690..28ea9d1d 100644 --- a/sdk/Cargo.lock +++ b/sdk/Cargo.lock @@ -37,7 +37,7 @@ dependencies = [ [[package]] name = "aleo" -version = "0.7.0" +version = "0.8.0" dependencies = [ "anyhow", "hex", diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index d806cf57..f8a34c65 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "aleo" authors = ["Konstantin Pandl", "Mike Turner", "Roman Proskuryakov"] -version = "0.7.0" +version = "0.8.0" description = "A Python sdk for zero-knowledge cryptography based on Aleo" edition = "2021" license = "GPL-3.0-or-later" diff --git a/sdk/docs/source/conf.py b/sdk/docs/source/conf.py index 1c353ccc..04468ca7 100644 --- a/sdk/docs/source/conf.py +++ b/sdk/docs/source/conf.py @@ -11,7 +11,7 @@ copyright = '2019-2026, Provable Inc.' author = 'kpp' -version = '0.7.0' +version = '0.8.0' # -- General configuration diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index 7006747e..33c1f338 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "aleo-sdk" description = "Python SDK for building zero-knowledge apps and DeFi on the Aleo network" -version = "0.7.0" +version = "0.8.0" readme = "Readme.md" license = {file = "LICENSE.md"} authors = [ diff --git a/shield-swap-sdk/README.md b/shield-swap-sdk/README.md index 4177a341..57ab64f7 100644 --- a/shield-swap-sdk/README.md +++ b/shield-swap-sdk/README.md @@ -43,7 +43,7 @@ pip install -e "shield-swap-sdk[async]" # + AsyncShieldSwap (httpx) pip install -e "shield-swap-sdk[mcp]" # + the MCP server ``` -Requires `aleo-sdk>=0.5.1` (this repo's SDK; imports as `aleo`) and Python 3.10+. +Requires `aleo-sdk>=0.8.0` (this repo's SDK; imports as `aleo`) and Python 3.10+. ## First swap example diff --git a/shield-swap-sdk/pyproject.toml b/shield-swap-sdk/pyproject.toml index d617b8a5..fa435f3e 100644 --- a/shield-swap-sdk/pyproject.toml +++ b/shield-swap-sdk/pyproject.toml @@ -1,10 +1,10 @@ [project] name = "shield-swap-sdk" -version = "0.7.0" +version = "0.8.0" description = "Python SDK for the shield swap AMM Dex on Aleo" readme = "README.md" requires-python = ">=3.10" -dependencies = ["aleo-sdk>=0.7.0", "requests>=2"] # 0.7.0: aleo.compliance (freeze-list proofs) +dependencies = ["aleo-sdk>=0.8.0", "requests>=2"] # 0.8.0: aleo.inventory (record management) [project.optional-dependencies] async = ["httpx>=0.27"] diff --git a/shield-swap-sdk/python/aleo_shield_swap/__init__.py b/shield-swap-sdk/python/aleo_shield_swap/__init__.py index 3492d059..66d8871a 100644 --- a/shield-swap-sdk/python/aleo_shield_swap/__init__.py +++ b/shield-swap-sdk/python/aleo_shield_swap/__init__.py @@ -14,18 +14,18 @@ amount_in=10**9).delegate() out = dex.claim_swap_output(handle).delegate() """ -_ALEO_SDK_MINIMUM = "0.7.0" +_ALEO_SDK_MINIMUM = "0.8.0" def _require_aleo_sdk() -> None: - """Fail at import, naming the fix, when the installed aleo-sdk predates ``aleo.compliance``. + """Fail at import, naming the fix, when the installed aleo-sdk predates ``aleo.inventory``. - Freeze-list proofs come from ``aleo.compliance``, which first shipped in aleo-sdk 0.7.0. An - older install would otherwise fail on the first import of the client with a bare - ``ModuleNotFoundError``. + Record management comes from ``aleo.inventory``, which first shipped in aleo-sdk 0.8.0 + (freeze-list proofs, from ``aleo.compliance``, arrived in 0.7.0). An older install would + otherwise fail on the first import of the client with a bare ``ModuleNotFoundError``. """ try: - import aleo.compliance # noqa: F401 + import aleo.inventory # noqa: F401 except ImportError as exc: from importlib.metadata import PackageNotFoundError, version try: @@ -118,7 +118,7 @@ def agent_guide() -> str: return files(__name__).joinpath("AGENTS.md").read_text() -__version__ = "0.7.0" +__version__ = "0.8.0" __all__ = [ "ShieldSwap", "AsyncShieldSwap", "ApiClient", "AsyncApiClient", diff --git a/shield-swap-sdk/tests/test_package.py b/shield-swap-sdk/tests/test_package.py index 8d8452a5..5344b05f 100644 --- a/shield-swap-sdk/tests/test_package.py +++ b/shield-swap-sdk/tests/test_package.py @@ -2,7 +2,7 @@ def test_version(): - assert aleo_shield_swap.__version__ == "0.7.0" + assert aleo_shield_swap.__version__ == "0.8.0" def test_lifecycle_exports(): @@ -32,7 +32,7 @@ def test_import_guard_names_the_aleo_sdk_upgrade(monkeypatch): import pytest import aleo_shield_swap - aleo_shield_swap._require_aleo_sdk() # the installed sdk has aleo.compliance - monkeypatch.setitem(sys.modules, "aleo.compliance", None) # simulate an aleo-sdk without it - with pytest.raises(ImportError, match=r'needs aleo-sdk>=0\.7\.0 \(found .*\); run: pip install -U "aleo-sdk>=0\.7\.0"'): + aleo_shield_swap._require_aleo_sdk() # the installed sdk has aleo.inventory + monkeypatch.setitem(sys.modules, "aleo.inventory", None) # simulate an aleo-sdk without it + with pytest.raises(ImportError, match=r'needs aleo-sdk>=0\.8\.0 \(found .*\); run: pip install -U "aleo-sdk>=0\.8\.0"'): aleo_shield_swap._require_aleo_sdk()