diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index 355ee130..09d18e36 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -1,27 +1,33 @@ -name: Base Std Smoke Tests (Vibenet) +name: Base Std Smoke Tests -# Advisory, MANUALLY-triggered smoke run of the FULL b20 suite against a LIVE Base network -# (Vibenet by default). +# Advisory, MANUALLY-triggered smoke run of the FULL b20 suite against a LIVE Base network. # -# workflow_dispatch ONLY — deliberately not pull_request / merge_group / schedule. Vibenet is a -# live chain, manually deployed per hardfork; CI cannot guarantee which precompile set is live, so -# an automated run would flap. This job never gates merges: it is a bring-up check you run by hand -# (e.g. after a hardfork ships to Vibenet) to confirm the b20 surface behaves against the real Rust -# precompiles. It always runs the whole suite against the ref you dispatch from (latest = main) — -# there is no journey picker and no fork/ref selector (branch-per-fork: dispatch from the branch/tag -# that matches the fork you want). Each journey self-skips when the live chain is not yet on the fork -# that ships its surface (e.g. the ERC-8056 multiplier journey on a pre-Cobalt chain). +# workflow_dispatch ONLY — deliberately not pull_request / merge_group / schedule. Live chains are +# manually deployed per hardfork; CI cannot guarantee which precompile set is live, so an automated +# run would flap. This job never gates merges: it is a bring-up check you run by hand (e.g. after a +# hardfork ships to a network) to confirm the b20 surface behaves against the real Rust precompiles. +# It always runs the whole suite against the ref you dispatch from (latest = main) — there is no +# journey picker and no fork/ref selector (branch-per-fork: dispatch from the branch/tag that matches +# the fork you want). Each journey self-skips when the live chain is not yet on the fork that ships +# its surface (e.g. the ERC-8056 multiplier journey on a pre-Cobalt chain). +# +# Networks: the `network` input names a GitHub Environment. Each environment holds ONE secret, +# SMOKE_CONFIG, a JSON blob with everything network-specific (RPC URL, both private keys, optional +# faucet). Nothing network-specific or sensitive lives in git. See script/smoke/README.md +# ("Advisory CI") for the blob shape and how to add a network. on: workflow_dispatch: inputs: - rpc_url: - description: "JSON-RPC endpoint of the live chain" + network: + description: "Network to test = name of the GitHub Environment holding its SMOKE_CONFIG secret" required: true - default: https://rpc.vibes.base.org/ + default: vibenet +# One run at a time per network: runs on the same chain share a deployer key (nonce collisions), +# runs on different chains are independent. concurrency: - group: ${{ github.workflow }}-${{ github.run_id }} + group: smoke-${{ inputs.network }} cancel-in-progress: false permissions: @@ -29,9 +35,10 @@ permissions: jobs: smoke: - name: Vibenet smoke + name: Smoke (${{ inputs.network }}) runs-on: ubuntu-latest timeout-minutes: 30 + environment: ${{ inputs.network }} steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 @@ -56,36 +63,71 @@ jobs: - name: Set up smoke runner venv shell: bash - # setup-python makes python3 == 3.13; make python-check enforces it. + # `make smoke-setup` (one-time venv + web3). setup-python makes python3 == 3.13, so point the + # Makefile at it; make python-check still enforces 3.13. run: make smoke-setup PYTHON=python3 - - name: Build contracts (interface ABIs the harness binds to) + - name: Load network config shell: bash - run: forge build + # Unpack the environment's SMOKE_CONFIG JSON into the env vars the harness reads (RPC_URL, + # DEPLOYER_PK, USER2_PK, optional FAUCET_*). Every value is masked BEFORE it is exported so + # it can never appear in a log. An empty secret means the environment is missing/unconfigured + # (GitHub silently creates an empty environment for an unknown name), so fail fast. + env: + SMOKE_CONFIG: ${{ secrets.SMOKE_CONFIG }} + NETWORK: ${{ inputs.network }} + run: | + set -euo pipefail + if [ -z "$SMOKE_CONFIG" ]; then + echo "::error::No SMOKE_CONFIG secret for environment '$NETWORK'. Check the network name, or add the secret (see script/smoke/README.md)." + exit 1 + fi + if ! echo "$SMOKE_CONFIG" | jq -e 'type == "object"' >/dev/null 2>&1; then + echo "::error::SMOKE_CONFIG for '$NETWORK' is not a valid JSON object." + exit 1 + fi + for key in rpc_url deployer_pk user2_pk; do + if ! echo "$SMOKE_CONFIG" | jq -e --arg k "$key" '(.[$k] // "") | type == "string" and length > 0' >/dev/null 2>&1; then + echo "::error::SMOKE_CONFIG for '$NETWORK' is missing required field '$key'." + exit 1 + fi + done - - name: Run the full Vibenet smoke suite + export_field() { # + local value + value=$(echo "$SMOKE_CONFIG" | jq -r --arg k "$1" '.[$k] // empty') + [ -n "$value" ] || return 0 + echo "::add-mask::$value" + echo "$2=$value" >> "$GITHUB_ENV" + } + export_field rpc_url RPC_URL + export_field deployer_pk DEPLOYER_PK + export_field user2_pk USER2_PK + export_field faucet_url FAUCET_URL + export_field faucet_network FAUCET_NETWORK + + - name: Run the full smoke suite id: smoke shell: bash - # RPC_URL comes from the dispatch input; the two keys are repo secrets (never echoed). `-k` - # (keep-going) runs every journey and prints a summary, so one journey's failure/skip never + # Same as the local runbook: `make smoke-all KEEP_GOING=1` (which runs `forge build` first to + # produce the interface ABIs). The Makefile sources .env when present, but existing env wins, + # so the values exported by "Load network config" are used and no .env is written to disk. + # KEEP_GOING runs every journey and prints a summary, so one journey's failure/skip never # masks the rest; it always exits 0, so the (advisory) job stays green and the summary below # reports the real per-journey status. continue-on-error still guards against an infra crash. continue-on-error: true - env: - RPC_URL: ${{ inputs.rpc_url }} - DEPLOYER_PK: ${{ secrets.SMOKE_DEPLOYER_PK }} - USER2_PK: ${{ secrets.SMOKE_USER2_PK }} run: | set -o pipefail - PYTHONPATH=script script/smoke/.venv/bin/python -m smoke all -k \ - 2>&1 | tee "$RUNNER_TEMP/smoke-output.txt" + make smoke-all KEEP_GOING=1 2>&1 | tee "$RUNNER_TEMP/smoke-output.txt" - name: Summarize smoke results if: always() shell: bash + env: + NETWORK: ${{ inputs.network }} run: | output="$RUNNER_TEMP/smoke-output.txt" - # Vibenet chain id, as logged by the harness preflight ("preflight ok — chain= ..."). + # Chain id, as logged by the harness preflight ("preflight ok — chain= ..."). chain_id=$(grep -oE 'chain=[0-9]+' "$output" 2>/dev/null | head -1 | cut -d= -f2) chain_id=${chain_id:-unknown} @@ -96,12 +138,12 @@ jobs: skipped=$(echo "$summary" | grep -oE '[0-9]+ skipped' | grep -oE '[0-9]+'); skipped=${skipped:-0} { - echo "## Vibenet Smoke Results" + echo "## Smoke Results" echo "" echo "| Field | Value |" echo "|---|---|" echo "| Ref | \`${{ github.ref_name }}\` |" - echo "| RPC endpoint | \`${{ inputs.rpc_url }}\` |" + echo "| Network | \`${NETWORK}\` |" echo "| Chain id | \`${chain_id}\` |" echo "| Passed / Failed / Skipped | ${passed} / ${failed} / ${skipped} |" echo "" @@ -112,6 +154,6 @@ jobs: elif [ "$passed" -ne 0 ]; then echo "✅ **${passed} passed** (${skipped} skipped) — the b20 surface behaved against the live precompiles." else - echo "⏭️ **All ${skipped} skipped** — Vibenet is not on the expected fork yet (feature inactive or pre-fork). Chain/fork state, not a defect." + echo "⏭️ **All ${skipped} skipped** — the network is not on the expected fork yet (feature inactive or pre-fork). Chain/fork state, not a defect." fi } >> "$GITHUB_STEP_SUMMARY" diff --git a/script/smoke/README.md b/script/smoke/README.md index f4b923bb..45ba52d3 100644 --- a/script/smoke/README.md +++ b/script/smoke/README.md @@ -88,19 +88,53 @@ PYTHONPATH=script python -m smoke asset policy -k # a subset, keep-going | `FAUCET_URL` / `FAUCET_NETWORK` | no | — | Optional deployer top-up when underfunded. | | `FAUCET_AMOUNT` / `FAUCET_MIN_ETHER` | no | `0.05` / `0.02` | Faucet amount and balance floor. | -### Advisory CI against Vibenet +### Advisory CI against a live network `.github/workflows/smoke-tests.yml` runs the **full suite** against a live Base network on demand. It is **`workflow_dispatch` only** — deliberately not wired to pull requests, merge groups, or a schedule. -Vibenet is a live chain that is manually deployed per hardfork, so CI can't guarantee which precompile -set is live; an automated run would flap. The workflow is **advisory and never gates merges**: run it -by hand (Actions → *Base Std Smoke Tests (Vibenet)* → *Run workflow*) after a hardfork ships. Its only -input is the RPC endpoint (default `https://rpc.vibes.base.org/`); it always runs every journey (`-k`) -against the ref you dispatch from — latest is `main`, and to run an older fork you dispatch from the -matching branch/tag (branch-per-fork; there is no journey picker and no fork/ref selector). It exports -`RPC_URL` from the input and `DEPLOYER_PK` / `USER2_PK` from repo secrets (`SMOKE_DEPLOYER_PK` / -`SMOKE_USER2_PK`), then reports per-journey **passed / failed / skipped** plus the chain id in the run -summary. A journey whose surface the live chain does not yet ship is reported as *skipped*, not failed. +Live chains are manually deployed per hardfork, so CI can't guarantee which precompile set is live; an +automated run would flap. The workflow is **advisory and never gates merges**: run it by hand (Actions → +*Base Std Smoke Tests* → *Run workflow*) after a hardfork ships. Its only input is `network`, the name of +a GitHub Environment (default `vibenet`); it runs `make smoke-all KEEP_GOING=1` against the ref you +dispatch from — latest is `main`, and to run an older fork you dispatch from the matching branch/tag +(branch-per-fork; there is no journey picker and no fork/ref selector). It reports per-journey +**passed / failed / skipped** plus the network name and chain id in the run summary. A journey whose +surface the live chain does not yet ship is reported as *skipped*, not failed. + +#### Network config lives in GitHub, not in git + +Each network is a **GitHub Environment** (Settings → Environments) holding one secret, `SMOKE_CONFIG`, +a JSON object with everything network-specific. Nothing network-specific or sensitive is committed; the +workflow unpacks the blob into `RPC_URL` / `DEPLOYER_PK` / `USER2_PK` / `FAUCET_*` and masks every value +in the logs. The run summary never prints the RPC URL (hosted RPC URLs often embed an API key). + +```json +{ + "rpc_url": "https://...", + "deployer_pk": "0x...", + "user2_pk": "0x...", + "faucet_url": "https://...", + "faucet_network": "..." +} +``` + +`rpc_url`, `deployer_pk` and `user2_pk` are required; `faucet_url` / `faucet_network` are optional (both +must be set for the deployer top-up, see the table above). A missing/empty secret or a missing required +field fails the run immediately with a clear error. + +**Adding a network** + +1. `cast wallet new` twice (deployer + user2) and fund the deployer. **Use throwaway keys that hold only + testnet funds for this network — never a key that controls real value.** +2. Create a GitHub Environment named after the network (e.g. `sepolia`); optionally require reviewers. +3. Save the JSON above as a secret without putting it in shell history or git, e.g. + `gh secret set SMOKE_CONFIG --env sepolia < config.json` (then delete `config.json`), or paste it in + the environment's settings page. +4. Dispatch: `gh workflow run smoke-tests.yml --ref main -f network=sepolia`. + +Runs are serialized per network (they share a deployer nonce); different networks run in parallel. A +mistyped `network` makes GitHub create an empty environment, which the workflow rejects as "No +SMOKE_CONFIG secret". ## What it checks