From 9e7b8aa9d79da8fb9bb1ae2a58923915b039ed81 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 23 Oct 2026 14:10:55 +0200 Subject: [PATCH 1/5] ci: tidy CI workflows and Python tooling after multi-aspect review - harden: hashed root requirements lock, env-passed inputs instead of interpolated expressions, abort job no longer discards a publish whose outcome is unknown, cert audit failures are no longer swallowed - modernize: pyproject.toml with ruff/mypy/pytest config, actionlint workflow, CodeQL actions language and re-pinned action, dependabot covers the hashed CI requirements, job summaries - simplify: single Python pin, shared setup-ci-scripts action, ESP-IDF activation moved to a script, per-event gate decisions, trimmed comments and dead modes --- .changes/ci-repository-server-only-deploy.md | 3 +- .../actions/build-compilationdb/action.yml | 28 +- .github/actions/build-firmware/action.yml | 42 +- .github/actions/build-frontend/action.yml | 27 +- .github/actions/build-staticfs/action.yml | 5 +- .github/actions/package-firmware/action.yml | 4 +- .../actions/repo-server-publish/action.yml | 20 +- .github/actions/repo-server-token/action.yml | 11 +- .github/actions/setup-ci-scripts/action.yml | 15 + .github/actions/setup-esp-idf/action.yml | 95 +-- .github/dependabot.yml | 34 +- .github/scripts/.python-version | 1 - .github/scripts/activate_idf.sh | 32 + .github/scripts/ci_build_gate.py | 163 ++--- .github/scripts/{get-vars.py => get_vars.py} | 164 ++--- .github/scripts/gha.py | 45 +- .github/scripts/github_api.py | 32 + .github/scripts/repo_server_policy.py | 51 +- .github/scripts/repo_server_publish.py | 168 ++--- .github/scripts/repo_server_token.py | 16 +- .github/scripts/requirements-dev.txt | 355 +++++++++- .github/scripts/requirements.txt | 15 +- .github/scripts/tests/conftest.py | 10 - .github/scripts/tests/support.py | 5 + .github/scripts/tests/test_build_cli.py | 4 +- .github/scripts/tests/test_build_scripts.py | 10 +- .github/scripts/tests/test_build_variants.py | 2 +- .github/scripts/tests/test_ci_build_gate.py | 29 +- .github/scripts/tests/test_get_vars.py | 41 +- .github/scripts/tests/test_merge_image.py | 3 +- .github/scripts/tests/test_package_boards.py | 36 +- .../scripts/tests/test_repo_server_policy.py | 1 - .../scripts/tests/test_repo_server_publish.py | 38 +- .../scripts/tests/test_repo_server_token.py | 20 + .github/scripts/tests/test_stamp_board.py | 15 +- .github/workflows/check-changes.yml | 4 +- .github/workflows/ci-build.yml | 129 ++-- .github/workflows/codeql.yml | 31 +- .github/workflows/cpp-linter.yml | 30 +- .github/workflows/frontend-checks.yml | 4 +- .github/workflows/host-tests.yml | 20 +- .github/workflows/nightly.yml | 4 +- .github/workflows/pinned-cert-audit.yml | 53 +- .github/workflows/pr-check-comment.yml | 5 +- .github/workflows/release.yml | 29 +- .github/workflows/script-tests.yml | 34 +- .github/workflows/workflow-lint.yml | 32 + CMakeLists.txt | 29 +- README.md | 2 +- certificates/cert_bundle.py | 367 +++++----- pyproject.toml | 32 + requirements.in | 5 + requirements.txt | 655 +++++++++++++++++- scripts/build.py | 91 +-- scripts/fetch_flatc.py | 13 +- scripts/gen_dep_graph.py | 38 +- scripts/gen_env_header.py | 63 +- scripts/gen_staticfs.py | 89 ++- scripts/generate_schemas.py | 76 +- scripts/merge_image.py | 7 +- scripts/utils/boards.py | 92 ++- scripts/utils/espimage.py | 37 +- scripts/utils/partitions.py | 16 +- scripts/utils/stamping.py | 5 +- 64 files changed, 2212 insertions(+), 1320 deletions(-) create mode 100644 .github/actions/setup-ci-scripts/action.yml delete mode 100644 .github/scripts/.python-version create mode 100755 .github/scripts/activate_idf.sh rename .github/scripts/{get-vars.py => get_vars.py} (58%) create mode 100644 .github/scripts/github_api.py delete mode 100644 .github/scripts/tests/conftest.py create mode 100644 .github/scripts/tests/test_repo_server_token.py create mode 100644 .github/workflows/workflow-lint.yml create mode 100644 pyproject.toml create mode 100644 requirements.in diff --git a/.changes/ci-repository-server-only-deploy.md b/.changes/ci-repository-server-only-deploy.md index 18a3489d0..2ec17ca82 100644 --- a/.changes/ci-repository-server-only-deploy.md +++ b/.changes/ci-repository-server-only-deploy.md @@ -1,11 +1,12 @@ --- kind: chore +pr: 438 --- Deploy firmware through the repository server only, and drop CI's direct CDN access ## Release Note CI no longer uploads firmware to the CDN or advances the channel pointer files itself. The whole deployment is now three calls to the repository server: start a release, upload every board's artifacts, publish, and the run fails on the first one that does not succeed. Where artifacts are stored, which hostname serves them and the moment a channel starts offering a version are decisions the server makes behind one API call, so CI no longer holds a storage credential and can no longer leave objects, pointer files and the release index disagreeing about what is live. -What remains of that path moved out of YAML into `.github/scripts/repo_server_publish.py` and `.github/scripts/repo_server_token.py`, joining `get-vars.py` and the build scripts. All three now share `.github/scripts/gha.py` for workflow commands, step outputs and the interpreter floor. HTTP and multipart are `requests`, and `get-vars.py` parses tags with the `semver` package instead of a local regex and comparison class. Dependencies are pinned by digest in `.github/scripts/requirements.txt` and installed with `--require-hashes`, so the job holding the publish credential cannot pick up whatever the index happens to serve. The staged release is a resource with a lifetime, and a `try/finally` expresses that directly, where the four YAML steps it replaces had to coordinate through `GITHUB_ENV` and an `always()` cleanup step to get the same guarantee. +What remains of that path moved out of YAML into `.github/scripts/repo_server_publish.py` and `.github/scripts/repo_server_token.py`, joining `get_vars.py` and the build scripts. All three now share `.github/scripts/gha.py` for workflow commands, step outputs and the interpreter floor. HTTP and multipart are `requests`, and `get_vars.py` parses tags with the `semver` package instead of a local regex and comparison class. Dependencies are pinned by digest in `.github/scripts/requirements.txt` and installed with `--require-hashes`, so the job holding the publish credential cannot pick up whatever the index happens to serve. The staged release is a resource with a lifetime, and a `try`/`except` that discards on failure expresses that directly, where the four YAML steps it replaces had to coordinate through `GITHUB_ENV` and an `always()` cleanup step to get the same guarantee. The `cdn-deploy` and `cdn-bump` jobs are gone, along with the `cdn-upload-firmware`, `cdn-upload-version-info`, `cdn-bump-version` and `sftp-mirror` composite actions and the `cdn` environment they read their Bunny credentials from. Publishing is now driven by the deploy gate instead of the `OPENSHOCK_REPO_SERVER_PUBLISH` variable, which is retired: a release tag or a nightly/manual develop deploy publishes, and every other build does the same init-upload-discard dry run as before. The staged-draft-release check that guarded the CDN upload now guards the publish. The GitHub release and the API announcement still run last, in that order, each chained on the publish succeeding. diff --git a/.github/actions/build-compilationdb/action.yml b/.github/actions/build-compilationdb/action.yml index 41c14763e..8d79c5e07 100644 --- a/.github/actions/build-compilationdb/action.yml +++ b/.github/actions/build-compilationdb/action.yml @@ -1,44 +1,32 @@ name: build-compilationdb description: >- Emit build/OpenShock-Core-V2/compile_commands.json with native ESP-IDF for the - canonical OpenShock-Core-V2 config, configuring only or also compiling. + canonical OpenShock-Core-V2 config. inputs: version: required: true description: 'Current firmware version' - mode: + target: required: false - default: configure + default: reconfigure description: >- - configure - CMake only, which writes compile_commands.json. - build - also compile, for tools that trace the compiler (CodeQL). + The idf.py target: reconfigure runs CMake only, which writes compile_commands.json; + app also compiles, for tools that trace the compiler (CodeQL). runs: using: composite steps: - uses: ./.github/actions/setup-esp-idf - - name: Validate mode - shell: bash - run: | - set -euo pipefail - case "${{ inputs.mode }}" in - configure|build) ;; - *) echo "::error::mode must be 'configure' or 'build', got '${{ inputs.mode }}'"; exit 1 ;; - esac - # ccache stays off: it would hide compiler invocations from CodeQL's tracer. - - name: Generate compilation database (OpenShock-Core-V2 / esp32s3) + - name: Generate compilation database (OpenShock-Core-V2) shell: bash run: | set -euo pipefail - if [ "${{ inputs.mode }}" = "build" ]; then - python3 scripts/build.py OpenShock-Core-V2 app - else - python3 scripts/build.py OpenShock-Core-V2 reconfigure - fi + python scripts/build.py OpenShock-Core-V2 "$TARGET" test -f build/OpenShock-Core-V2/compile_commands.json \ || { echo "::error::compile_commands.json was not generated"; exit 1; } env: + TARGET: ${{ inputs.target }} OPENSHOCK_FW_VERSION: ${{ inputs.version }} OPENSHOCK_FW_GIT_COMMIT: ${{ github.sha }} diff --git a/.github/actions/build-firmware/action.yml b/.github/actions/build-firmware/action.yml index 22b42846e..8bf7247f6 100644 --- a/.github/actions/build-firmware/action.yml +++ b/.github/actions/build-firmware/action.yml @@ -5,7 +5,7 @@ description: >- inputs: variant: required: true - description: 'Build variant id, as get-vars.py emits it (e.g. esp32s3-6832dbd5)' + description: 'Build variant id, as get_vars.py emits it (e.g. esp32s3-6832dbd5)' version: required: true description: 'Current firmware version' @@ -18,13 +18,21 @@ runs: # One store per variant and IDF version. Entries are immutable, so the key ends in the run and # restore-keys picks the newest store; only develop saves, so every other ref restores develop's. + - name: Compute ccache key + id: ccache-key + shell: bash + env: + IDF_VERSION: ${{ steps.idf.outputs.version }} + VARIANT: ${{ inputs.variant }} + run: echo "key=ccache-v2-$RUNNER_OS-$IDF_VERSION-$VARIANT-" >> "$GITHUB_OUTPUT" + - name: Restore ccache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/ccache - key: ccache-v2-${{ runner.os }}-${{ steps.idf.outputs.version }}-${{ inputs.variant }}-${{ github.run_id }}-${{ github.run_attempt }} + key: ${{ steps.ccache-key.outputs.key }}${{ github.run_id }}-${{ github.run_attempt }} restore-keys: | - ccache-v2-${{ runner.os }}-${{ steps.idf.outputs.version }}-${{ inputs.variant }}- + ${{ steps.ccache-key.outputs.key }} # Code targets only: the static0 image is the staticfs job's. - name: Build firmware @@ -33,8 +41,9 @@ runs: set -euo pipefail export IDF_CCACHE_ENABLE=1 ccache --zero-stats >/dev/null 2>&1 || true - python3 scripts/build.py --variant "${{ inputs.variant }}" app bootloader partition-table + python scripts/build.py --variant "$VARIANT" app bootloader partition-table env: + VARIANT: ${{ inputs.variant }} OPENSHOCK_FW_VERSION: ${{ inputs.version }} # Fixed: the build directory is on every compile command line, so it is part of each ccache hash. OPENSHOCK_BUILD_DIR: build/firmware @@ -47,7 +56,16 @@ runs: # A near-zero hit rate on an unchanged tree means something per-commit is being force-included. - name: ccache statistics shell: bash - run: ccache --show-stats --verbose || ccache -s + env: + VARIANT: ${{ inputs.variant }} + run: | + ccache --show-stats --verbose || ccache -s + { + echo "### ccache ($VARIANT)" + echo '```' + ccache --show-stats || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" # develop only: another ref's entry is restorable by that ref alone and would only spend the 10 GB. - name: Save ccache @@ -55,20 +73,20 @@ runs: uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/ccache - key: ccache-v2-${{ runner.os }}-${{ steps.idf.outputs.version }}-${{ inputs.variant }}-${{ github.run_id }}-${{ github.run_attempt }} + key: ${{ steps.ccache-key.outputs.key }}${{ github.run_id }}-${{ github.run_attempt }} # dist// so the package job can merge every variant into one tree; app.bin is the OTA name. - name: Stage partition images shell: bash env: - B: build/firmware - OUT: dist/${{ inputs.variant }} + BUILD_DIR: build/firmware + OUT_DIR: dist/${{ inputs.variant }} run: | set -euo pipefail - mkdir -p "$OUT" - cp "$B/firmware.bin" "$OUT/app.bin" - cp "$B/bootloader/bootloader.bin" "$OUT/bootloader.bin" - cp "$B/partition_table/partition-table.bin" "$OUT/partition-table.bin" + mkdir -p "$OUT_DIR" + cp "$BUILD_DIR/firmware.bin" "$OUT_DIR/app.bin" + cp "$BUILD_DIR/bootloader/bootloader.bin" "$OUT_DIR/bootloader.bin" + cp "$BUILD_DIR/partition_table/partition-table.bin" "$OUT_DIR/partition-table.bin" # variant_*, not firmware_*: repo-server-publish downloads firmware_* and must not get unstamped images. - name: Upload partition images diff --git a/.github/actions/build-frontend/action.yml b/.github/actions/build-frontend/action.yml index 2c3cd93ec..6388f62ce 100644 --- a/.github/actions/build-frontend/action.yml +++ b/.github/actions/build-frontend/action.yml @@ -15,16 +15,16 @@ runs: with: node-version-file: ./frontend/package.json - - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 - name: Install pnpm + - name: Install pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 with: - cache: true - cache_dependency_path: ./frontend/pnpm-lock.yaml - package_json_file: ./frontend/package.json - run_install: | - - recursive: true - cwd: ./frontend - args: [--frozen-lockfile, --strict-peer-dependencies] + cache: true + cache_dependency_path: ./frontend/pnpm-lock.yaml + package_json_file: ./frontend/package.json + run_install: | + - recursive: true + cwd: ./frontend + args: [--frozen-lockfile, --strict-peer-dependencies] - name: Check if: ${{ inputs.checks == 'true' }} @@ -32,17 +32,12 @@ runs: shell: bash run: pnpm run check - # FIXME: Playwirght is not working: Error: browserType.launch: Executable doesn't exist at /home/runner/.cache/ms-playwright/chromium_headless_shell-1161/chrome-linux/headless_shell - # - name: Install playwright - # working-directory: ./frontend - # shell: bash - # run: pnpx playwright install --with-deps - + # TODO: run e2e tests once playwright installs on the runner - name: Test if: ${{ inputs.checks == 'true' }} working-directory: ./frontend shell: bash - run: pnpm test:unit # TODO: Change back to pnpm test when playwright is working + run: pnpm test:unit - name: Build working-directory: ./frontend diff --git a/.github/actions/build-staticfs/action.yml b/.github/actions/build-staticfs/action.yml index 7e302bdee..bd9a99244 100644 --- a/.github/actions/build-staticfs/action.yml +++ b/.github/actions/build-staticfs/action.yml @@ -8,13 +8,12 @@ runs: steps: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version-file: .github/scripts/.python-version + python-version-file: .python-version cache: 'pip' - # gen_staticfs.py needs only littlefs-python; the rest of requirements.txt is for the merge. - name: Install Python dependencies shell: bash - run: pip install "$(grep -E '^littlefs-python==' requirements.txt)" + run: pip install --require-hashes -r requirements.txt # Sized from partitions/ota_4mb.csv, the table every board uses. - name: Build static filesystem image diff --git a/.github/actions/package-firmware/action.yml b/.github/actions/package-firmware/action.yml index 85b6bb11e..6ceac4711 100644 --- a/.github/actions/package-firmware/action.yml +++ b/.github/actions/package-firmware/action.yml @@ -15,12 +15,12 @@ runs: steps: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version-file: .github/scripts/.python-version + python-version-file: .python-version cache: 'pip' - name: Install Python dependencies shell: bash - run: pip install -r requirements.txt + run: pip install --require-hashes -r requirements.txt # Each variant artifact holds one / directory, so merging them keeps variants apart. - name: Download variant builds diff --git a/.github/actions/repo-server-publish/action.yml b/.github/actions/repo-server-publish/action.yml index 60a6c2a38..70b34c6b7 100644 --- a/.github/actions/repo-server-publish/action.yml +++ b/.github/actions/repo-server-publish/action.yml @@ -1,13 +1,15 @@ name: repo-server-publish description: >- Drives one phase of a firmware release on the repository server. - `init` opens the release and returns its id; `publish` and `dry-run` upload every board's artifacts and then promote or discard; `abort` closes a release whose build never got that far. - Nothing before the ending differs between `publish` and `dry-run`, so a dry run exercises the whole publish path. + `init` opens the release and returns its id; `publish` uploads every board's artifacts and promotes it; `abort` closes a release whose build never got that far. outputs: release-id: - description: 'The staged release id, set by `init` and passed to a later `publish`, `dry-run` or `abort`.' + description: 'The staged release id, set by `init` and passed to a later `publish` or `abort`.' value: ${{ steps.run.outputs.release-id }} + outcome-unknown: + description: "'true' when `publish` got no response, so the release may be live and must not be aborted." + value: ${{ steps.run.outputs.outcome-unknown }} inputs: server-url: @@ -23,22 +25,21 @@ inputs: description: 'Release channel: stable, beta or develop. Required by `init`.' required: false boards: - description: 'Newline-separated boards to publish. Required by `init`, `publish` and `dry-run`.' + description: 'Newline-separated boards to publish. Required by `init` and `publish`.' required: false release-id: - description: 'Release opened by an earlier `init`. Required by `publish`, `dry-run` and `abort`.' + description: 'Release opened by an earlier `init`. Required by `publish` and `abort`.' required: false mode: - description: '`init` opens the release; `publish` promotes it; `dry-run` aborts it, leaving nothing staged; `abort` closes one the build never reached.' - required: false - default: 'dry-run' + description: '`init` opens the release; `publish` uploads and promotes it; `abort` closes one the build never reached.' + required: true runs: using: composite steps: - name: Download firmware artifacts - if: ${{ inputs.mode == 'publish' || inputs.mode == 'dry-run' }} + if: ${{ inputs.mode == 'publish' }} uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: path: artifacts @@ -56,5 +57,4 @@ runs: BOARDS: ${{ inputs.boards }} RELEASE_ID: ${{ inputs.release-id }} MODE: ${{ inputs.mode }} - ARTIFACTS_DIR: artifacts run: python .github/scripts/repo_server_publish.py diff --git a/.github/actions/repo-server-token/action.yml b/.github/actions/repo-server-token/action.yml index 2086fbe89..7d01f87fd 100644 --- a/.github/actions/repo-server-token/action.yml +++ b/.github/actions/repo-server-token/action.yml @@ -16,16 +16,7 @@ outputs: runs: using: composite steps: - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version-file: .github/scripts/.python-version - cache: 'pip' - cache-dependency-path: .github/scripts/requirements.txt - - # Same pinned set as repo-server-publish; pip no-ops when the job already has them. - - name: Install script dependencies - shell: bash - run: pip install --require-hashes -r .github/scripts/requirements.txt + - uses: ./.github/actions/setup-ci-scripts - id: mint shell: bash diff --git a/.github/actions/setup-ci-scripts/action.yml b/.github/actions/setup-ci-scripts/action.yml new file mode 100644 index 000000000..14a194471 --- /dev/null +++ b/.github/actions/setup-ci-scripts/action.yml @@ -0,0 +1,15 @@ +name: setup-ci-scripts +description: Installs Python and the hash-pinned dependencies of the scripts in .github/scripts. + +runs: + using: composite + steps: + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version-file: .python-version + cache: 'pip' + cache-dependency-path: .github/scripts/requirements.txt + + - name: Install script dependencies + shell: bash + run: pip install --require-hashes -r .github/scripts/requirements.txt diff --git a/.github/actions/setup-esp-idf/action.yml b/.github/actions/setup-esp-idf/action.yml index f7188a2fe..ae23ad308 100644 --- a/.github/actions/setup-esp-idf/action.yml +++ b/.github/actions/setup-esp-idf/action.yml @@ -18,6 +18,14 @@ runs: steps: # Restore and save are split so the prune below lands between them. Missing paths are skipped. # No restore-keys: a partial match would run the installer over a stale tree and save the layering. + # Bump the trailing number to invalidate. + - name: Compute cache key + id: key + shell: bash + env: + IDF_VERSION: ${{ inputs.idf-version }} + run: echo "key=esp-idf-$RUNNER_OS-$IDF_VERSION-6" >> "$GITHUB_OUTPUT" + - name: Restore ESP-IDF install id: idf-cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -26,8 +34,7 @@ runs: /tmp/esp /opt/esp ~/.espressif - # Bump the trailing number to invalidate; the save step below repeats this key. - key: esp-idf-${{ runner.os }}-${{ inputs.idf-version }}-6 + key: ${{ steps.key.outputs.key }} # Cache miss only: the action re-runs apt and re-prepares the clone even over a complete tree. # v1 is a moving branch (no release tags), pinned by SHA. @@ -41,77 +48,10 @@ runs: - name: Activate cached ESP-IDF if: ${{ steps.idf-cache.outputs.cache-hit == 'true' }} shell: bash - run: | - set -euo pipefail - # Guard with -f: an unmatched glob stays unexpanded. - act="" - for c in /tmp/esp /opt/esp "$HOME/.espressif" "$HOME/.espressif/tools"; do - for f in "$c"/activate_idf_*.sh; do - if [ -f "$f" ]; then - act="$f" - break 2 - fi - done - done - if [ -z "$act" ]; then - echo "::error::ESP-IDF cache was restored but no activate_idf_*.sh was found in it." - exit 1 - fi - echo "activating $act" - - # -e prints KEY=VALUE lines instead of mutating the shell; sourcing it refuses to run - # inside a step ($0 is not a shell name) and trips set -u on $ZSH_VERSION. - env_dump="$(sh "$act" -e)" - if [ -z "$env_dump" ]; then - echo "::error::$act produced no environment." - exit 1 - fi - - # PATH= holds only the directories to prepend; SYSTEM_PATH is the caller's own PATH. - printf '%s\n' "$env_dump" | while IFS= read -r line; do - case "$line" in - PATH=*) - printf '%s\n' "${line#PATH=}" | tr ':' '\n' | grep -v '^$' >> "$GITHUB_PATH" - ;; - SYSTEM_PATH=*|'') ;; - *=*) - echo "$line" >> "$GITHUB_ENV" - ;; - esac - done - - idf_path="$(printf '%s\n' "$env_dump" | sed -n 's/^IDF_PATH=//p' | head -1)" - if [ -z "$idf_path" ]; then - echo "::error::$act did not report IDF_PATH." - exit 1 - fi - # idf.py is in $IDF_PATH/tools, which the script does not emit; scripts/build.py prefers it. - echo "$idf_path/tools" >> "$GITHUB_PATH" - echo "activated ESP-IDF at $idf_path" - - # What the fresh install weighs per directory, to check the prune list against. - - name: Measure ESP-IDF install - if: ${{ steps.idf-cache.outputs.cache-hit != 'true' }} - shell: bash - run: | - set -uo pipefail - for root in /tmp/esp /opt/esp "$HOME/.espressif"; do - [ -d "$root" ] || continue - echo "::group::$root" - du -sh "$root" 2>/dev/null || true - du -sh "$root"/* 2>/dev/null | sort -rh | head -20 || true - if [ -d "$root/tools" ]; then - echo "-- tools --" - du -sh "$root"/tools/* 2>/dev/null | sort -rh | head -20 || true - fi - echo "::endgroup::" - done + run: "$GITHUB_ACTION_PATH/../../scripts/activate_idf.sh" - # Trim what no CI build uses before caching: download archives (already extracted), esp-clang - # (the firmware builds with gcc), the GDBs and OpenOCD (no hardware), and the ULP toolchain. - # No board enables the ULP; one that does must take esp32ulp-elf off this list. - # Never prune the gcc toolchains (xtensa-esp-elf, riscv32-esp-elf). Removal is by exact name so - # xtensa-esp-elf-gdb cannot take xtensa-esp-elf with it; runners use /, EIM /tools/. + # Drops what no CI build uses (archives, clang, GDBs, OpenOCD, ULP) by exact name, never the gcc toolchains. + # A board that enables the ULP must take esp32ulp-elf off this list. - name: Prune unused toolchains before caching if: ${{ steps.idf-cache.outputs.cache-hit != 'true' }} shell: bash @@ -138,8 +78,7 @@ runs: done echo "pruned $((total / 1024)) MB in total" - # ccache is the one build prerequisite the IDF tree lacks, so a pinned static build is cached with it - # and a warm run never touches apt. + # A pinned static ccache, cached with the install so a warm run never touches apt. - name: Bundle ccache into the install if: ${{ steps.idf-cache.outputs.cache-hit != 'true' }} shell: bash @@ -156,11 +95,7 @@ runs: tar -xJf "$RUNNER_TEMP/ccache.tar.xz" -C /tmp/esp/ccache --strip-components=1 "${name}/ccache" /tmp/esp/ccache/ccache --version | head -1 - # Only develop saves: a cache saved on another ref is restorable by that ref alone, and each one - # would be another ~1.8 GB of the repository's 10 GB. Concurrent misses race; the losers only warn. - # The cost, accepted for that limit: after an eviction (7 days unused) or a key bump, every tag - # and PR build installs cold and saves nothing until a develop push next runs this action. - # The alternative, if that bites, is to let one nominated job save on a miss from any ref. + # Only develop saves: a cache saved on another ref is restorable by that ref alone, and costs ~1.8 GB of the 10 GB. - name: Save ESP-IDF install if: >- ${{ steps.idf-cache.outputs.cache-hit != 'true' @@ -171,7 +106,7 @@ runs: /tmp/esp /opt/esp ~/.espressif - key: esp-idf-${{ runner.os }}-${{ inputs.idf-version }}-6 + key: ${{ steps.key.outputs.key }} - name: Put ccache on PATH shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 55e1e994d..a76a7c8a6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,7 +5,7 @@ version: 2 updates: - # Daily checks for updates in Github Actions (CI/CD workflows) + # Weekly checks for updates in Github Actions (CI/CD workflows) - package-ecosystem: 'github-actions' directory: '/' schedule: @@ -16,44 +16,32 @@ updates: ci-cd: patterns: - '*' - update-types: - - 'major' - - 'minor' - - 'patch' - # Daily checks for npm package updates in CI-CD scripts - - package-ecosystem: 'npm' - directory: '/.github/scripts' + # Weekly checks for pip package updates in build system scripts + - package-ecosystem: 'pip' + directory: '/' schedule: interval: 'weekly' day: monday time: '06:00' groups: - ci-cd: + build-system: patterns: - '*' - update-types: - - 'major' - - 'minor' - - 'patch' - # Daily checks for pip package updates in build system scripts + # Weekly checks for pip package updates in CI-CD scripts - package-ecosystem: 'pip' - directory: '/' + directory: '/.github/scripts' schedule: interval: 'weekly' day: monday time: '06:00' groups: - build-system: + ci-cd: patterns: - '*' - update-types: - - 'major' - - 'minor' - - 'patch' - # Daily checks for npm package updates in frontend + # Weekly checks for npm package updates in frontend - package-ecosystem: 'npm' directory: '/frontend' schedule: @@ -64,7 +52,3 @@ updates: frontend: patterns: - '*' - update-types: - - 'major' - - 'minor' - - 'patch' diff --git a/.github/scripts/.python-version b/.github/scripts/.python-version deleted file mode 100644 index e4fba2183..000000000 --- a/.github/scripts/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.12 diff --git a/.github/scripts/activate_idf.sh b/.github/scripts/activate_idf.sh new file mode 100755 index 000000000..14c66a3e9 --- /dev/null +++ b/.github/scripts/activate_idf.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Exports a restored ESP-IDF install (EIM's activate_idf_.sh) to later steps of this job. +set -euo pipefail + +act="$(compgen -G '/tmp/esp/activate_idf_*.sh' || compgen -G '/opt/esp/activate_idf_*.sh' \ + || compgen -G "$HOME/.espressif/activate_idf_*.sh" || compgen -G "$HOME/.espressif/tools/activate_idf_*.sh" || true)" +act="$(printf '%s\n' "$act" | head -1)" +if [ -z "$act" ]; then + echo '::error::ESP-IDF cache was restored but no activate_idf_*.sh was found in it.' + exit 1 +fi +echo "activating $act" + +# -e prints KEY=VALUE lines; sourcing the script refuses to run inside a step. +env_dump="$(sh "$act" -e)" +if [ -z "$env_dump" ]; then + echo "::error::$act produced no environment." + exit 1 +fi + +# PATH= holds only the directories to prepend; SYSTEM_PATH is the caller's own PATH. +sed -n 's/^PATH=//p' <<< "$env_dump" | tr ':' '\n' | grep -v '^$' >> "$GITHUB_PATH" || true +grep -E '^[^=]+=' <<< "$env_dump" | grep -vE '^(PATH|SYSTEM_PATH)=' >> "$GITHUB_ENV" || true + +idf_path="$(sed -n 's/^IDF_PATH=//p' <<< "$env_dump" | head -1)" +if [ -z "$idf_path" ]; then + echo "::error::$act did not report IDF_PATH." + exit 1 +fi +# idf.py lives in $IDF_PATH/tools, which the script does not emit. +echo "$idf_path/tools" >> "$GITHUB_PATH" +echo "activated ESP-IDF at $idf_path" diff --git a/.github/scripts/ci_build_gate.py b/.github/scripts/ci_build_gate.py index 0309f4ce6..f2c61ee4f 100644 --- a/.github/scripts/ci_build_gate.py +++ b/.github/scripts/ci_build_gate.py @@ -1,24 +1,17 @@ #!/usr/bin/env python3 """Decide what a ci-build run does: build, publish, which server, which boards. -build governs the build matrix. -publish governs whether the build reaches a repository server at all, and server names which one. Building is not publishing: a pull request compiles and goes no further. - -Reads the event from the GITHUB_* environment and the event payload, and asks the GitHub API only for a nightly: when this branch last built, and whether anything since then can change the firmware. -Those two calls sit behind the Api protocol so the tests can answer them without a network. +The GitHub API is queried only for a nightly (when this branch last built, and what changed since); those calls sit behind the Api protocol so tests need no network. """ import json import re from dataclasses import dataclass -from datetime import datetime from typing import Protocol -import requests - -from gha import env, notice, require_env, set_output, warn -from repo_server_policy import API, API_VERSION +from gha import env, notice, require_env, set_output, summary, warn +from github_api import github_get FULL_BUILD_LABEL = 'ci:full-build' CANARY_BOARD = 'OpenShock-Core-V2' @@ -59,20 +52,10 @@ def compare_files(self, base: str, head: str) -> list[dict]: class GitHubApi: def __init__(self, repo: str, token: str) -> None: self.repo = repo - self.session = requests.Session() - self.session.headers.update( - { - 'Authorization': f'Bearer {token}', - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': API_VERSION, - } - ) + self.token = token def _get(self, path: str, params: dict) -> dict: - resp = self.session.get(f'{API}/repos/{self.repo}{path}', params=params, timeout=(10, 30)) - if resp.status_code != 200: - raise RuntimeError(f'HTTP {resp.status_code}: {(resp.text or "").strip()[:300]}') - return resp.json() + return github_get(f'/repos/{self.repo}{path}', self.token, params) def latest_successful_run(self, event: str, branch: str) -> dict | None: data = self._get( @@ -91,26 +74,17 @@ class Decision: build: bool = False publish: bool = False server: str = 'dev' - # Empty builds every board. The build variants these need are get-vars.py's to work out. boards: str = '' -def _created_at(run: dict) -> datetime: - return datetime.fromisoformat(run['created_at']) - - def last_build_sha(api: Api, branch: str) -> str: - """HEAD of this branch's last successful nightly or manual run. + """HEAD of this branch's last successful manual or nightly dispatch. - 'schedule' covers the develop nightlies from before nightly.yml dispatched them. Returns '' on error so a hiccup rebuilds rather than wrongly skipping. """ try: - runs = [r for r in (api.latest_successful_run(e, branch) for e in ('schedule', 'workflow_dispatch')) if r] - if not runs: - return '' - runs.sort(key=_created_at, reverse=True) - return runs[0].get('head_sha') or '' + run = api.latest_successful_run('workflow_dispatch', branch) + return (run or {}).get('head_sha') or '' except Exception as err: warn(f'Could not read the last {branch} build: {err}') return '' @@ -139,65 +113,68 @@ def source_changed_since(api: Api, base: str, sha: str) -> bool: return True -def decide(event_name: str, ref: str, sha: str, payload: dict, api: Api) -> Decision: - d = Decision() - branch = ref.removeprefix('refs/heads/') if ref.startswith('refs/heads/') else '' - inputs = payload.get('inputs') or {} - # The payload carries a boolean input as either true or 'true', depending on who dispatched it. - nightly = event_name == 'workflow_dispatch' and inputs.get('nightly') in (True, 'true') +def _decide_pull_request(payload: dict) -> Decision: + # A pull request compiles and stops: merging changes nothing on any server. + label = (payload.get('label') or {}).get('name') + if payload.get('action') == 'labeled' and label != FULL_BUILD_LABEL: + notice(f'Label "{label}" does not affect the build.') + return Decision() + d = Decision(build=True) + labels = (payload.get('pull_request') or {}).get('labels') or [] + if not any(lbl.get('name') == FULL_BUILD_LABEL for lbl in labels): + d.boards = CANARY_BOARD + notice(f'Canary build of {CANARY_BOARD} only; add the {FULL_BUILD_LABEL} label to build every board.') + return d - if event_name == 'pull_request': - label = (payload.get('label') or {}).get('name') - if payload.get('action') == 'labeled' and label != FULL_BUILD_LABEL: - notice(f'Label "{label}" does not affect the build.') - else: - d.build = True - labels = (payload.get('pull_request') or {}).get('labels') or [] - if not any(lbl.get('name') == FULL_BUILD_LABEL for lbl in labels): - d.boards = CANARY_BOARD - notice(f'Canary build of {CANARY_BOARD} only; add the {FULL_BUILD_LABEL} label to build every board.') - elif event_name == 'push': - # Only release tags trigger a push run. - d.build = True - elif event_name == 'workflow_dispatch': - if nightly: - if branch not in NIGHTLY_BRANCHES: - warn(f'Nightly requested on {ref}; only develop, beta and master have nightlies.') - else: - base = last_build_sha(api, branch) - if base == sha: - notice(f'{branch} unchanged since its last build ({sha}); nothing to build.') - elif base != '' and not source_changed_since(api, base, sha): - notice(f'No firmware sources changed on {branch} since its last build ({base}); nothing to build.') - else: - d.build = True - else: - d.build = True + +def _decide_nightly(ref: str, sha: str, api: Api) -> Decision: + # Only the develop nightly publishes (to prod); beta and master are published from tags. + branch = ref.removeprefix('refs/heads/') if ref.startswith('refs/heads/') else '' + d = Decision(server='prod') + if branch not in NIGHTLY_BRANCHES: + warn(f'Nightly requested on {ref}; only develop, beta and master have nightlies.') + return d + base = last_build_sha(api, branch) + if base == sha: + notice(f'{branch} unchanged since its last build ({sha}); nothing to build.') + elif base != '' and not source_changed_since(api, base, sha): + notice(f'No firmware sources changed on {branch} since its last build ({base}); nothing to build.') else: d.build = True + d.publish = d.build and branch == 'develop' + return d + + +def _decide_dispatch(ref: str, inputs: dict) -> Decision: + # A manual dispatch always publishes; the channel and the ref are chosen by hand. + d = Decision(build=True, publish=True, server='prod' if inputs.get('server') == 'prod' else 'dev') + # The ref picker offers tags as well as branches, so prod still means running against the tag being released. + if d.server == 'prod' and not ref.startswith('refs/tags/'): + warn(f'prod was requested on {ref}; only a tag may dispatch to prod, publishing to dev instead.') + d.server = 'dev' + return d - # Publishing is never a side effect of a build. - # A pull request compiles and stops, so merging changes nothing on any server until someone says to. - # The develop nightly and a release tag publish to prod, because both are already the deliberate act. - # The beta and master nightlies only build: those channels are published from tags. - # Everything else reaches a server only by being dispatched, which is where the channel and the ref are chosen by hand. + +def _decide_push(ref: str) -> Decision: + # The workflow only triggers on release tags, which publish to prod. is_tag = ref.startswith('refs/tags/') + return Decision(build=True, publish=is_tag, server='prod' if is_tag else 'dev') - if nightly: - d.publish = d.build and branch == 'develop' - d.server = 'prod' - elif event_name == 'workflow_dispatch': - d.publish = True - d.server = 'prod' if inputs.get('server') == 'prod' else 'dev' - # The ref picker offers tags as well as branches, so reaching prod by hand still means running against the tag being released. - if d.server == 'prod' and not is_tag: - warn(f'prod was requested on {ref}; only a tag may dispatch to prod, publishing to dev instead.') - d.server = 'dev' - elif event_name == 'push' and is_tag: - d.publish = True - d.server = 'prod' - return d +def decide(event_name: str, ref: str, sha: str, payload: dict, api: Api) -> Decision: + inputs = payload.get('inputs') or {} + # The payload carries a boolean input as either true or 'true', depending on who dispatched it. + nightly = event_name == 'workflow_dispatch' and inputs.get('nightly') in (True, 'true') + + if event_name == 'pull_request': + return _decide_pull_request(payload) + if nightly: + return _decide_nightly(ref, sha, api) + if event_name == 'workflow_dispatch': + return _decide_dispatch(ref, inputs) + if event_name == 'push': + return _decide_push(ref) + raise SystemExit(f'error: unsupported event {event_name!r}') def main() -> int: @@ -209,17 +186,13 @@ def main() -> int: api = GitHubApi(env('GITHUB_REPOSITORY'), env('GITHUB_TOKEN')) d = decide(env('GITHUB_EVENT_NAME'), env('GITHUB_REF'), env('GITHUB_SHA'), payload, api) - def flag(value: bool) -> str: - return 'true' if value else 'false' - - set_output('build', flag(d.build)) - set_output('publish', flag(d.publish)) + set_output('build', str(d.build).lower()) + set_output('publish', str(d.publish).lower()) set_output('server', d.server) set_output('boards', d.boards) - print( - f'decision: build={flag(d.build)} publish={flag(d.publish)} ' - f'server={d.server} boards={d.boards or "all"}' - ) + decision = f'build={str(d.build).lower()} publish={str(d.publish).lower()} server={d.server} boards={d.boards or "all"}' + print(f'decision: {decision}') + summary(f'### Gate\n\n`{decision}`') return 0 diff --git a/.github/scripts/get-vars.py b/.github/scripts/get_vars.py similarity index 58% rename from .github/scripts/get-vars.py rename to .github/scripts/get_vars.py index 7a4dcf18b..d041ba578 100644 --- a/.github/scripts/get-vars.py +++ b/.github/scripts/get_vars.py @@ -1,30 +1,31 @@ #!/usr/bin/env python3 """Derive the CI build variables: firmware version, release channel, build-variant matrix. -Python port of the former get-vars.js (no node/pnpm toolchain; semver is its one dependency). Reads the git ref/sha from the GITHUB_* env, the pending version from release-tool (RELEASE_NEXT_VERSION / RELEASE_SKIP), the repo tags via git, and the buildable boards from boards/*.defaults, then writes GitHub Action outputs. The boards are grouped into build variants (scripts/utils/boards.py): ci-build compiles once per variant and stamps each board into a copy. """ import json -import os import re import subprocess import sys from pathlib import Path import semver - -from gha import fail, notice, set_output +from gha import env, fail, notice, require_env, set_output, summary from repo_server_policy import CHANNELS -# The one parser for boards/.defaults lives with the build helpers. ci-build's plan job checks out a -# sparse tree, so scripts/utils is listed there explicitly. +# ci-build's plan job checks out a sparse tree, so scripts/utils is listed there explicitly. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'scripts')) from utils.boards import all_boards, group_variants # noqa: E402 +CHANNEL_BY_PRERELEASE = {'': 'stable', 'stable': 'stable', 'rc': 'beta', 'beta': 'beta', 'dev': 'develop', 'develop': 'develop'} +CHANNEL_BY_BRANCH = {'master': 'stable', 'beta': 'beta', 'develop': 'develop'} + + def parse_version(tag: str) -> semver.Version | None: - """Strict semver, except for the historical `v` prefix - the repo still carries a `v0.8.1` tag that the spec (and therefore the parser) rejects. + """Strict semver, except for the historical `v` prefix (the repo still carries a `v0.8.1` tag). + Returns None rather than raising, since scanning the tag list is best-effort. """ try: @@ -34,32 +35,77 @@ def parse_version(tag: str) -> semver.Version | None: def prerelease_id(version: semver.Version) -> str: - """First dot-separated prerelease identifier: `rc` out of `1.5.0-rc.2`, '' when the version is final. - This is what names the channel. - """ + """First dot-separated prerelease identifier (`rc` of `1.5.0-rc.2`, '' when final); it names the channel.""" return (version.prerelease or '').split('.')[0] -def is_stable(version: semver.Version) -> bool: - return prerelease_id(version) in ('', 'stable') +def sanitize(name: str) -> str: + return re.sub(r'[^a-zA-Z0-9]+', '-', name).strip('-') + + +def resolve_version(is_git_tag: bool, head_ref: str, short_sha: str, latest: semver.Version) -> str: + next_version = env('RELEASE_NEXT_VERSION') + + # Branch and PR builds use release-tool's next version, labelled as a pre-release of it. + if not is_git_tag and next_version and env('RELEASE_SKIP') != 'true': + version = next_version + else: + version = str(latest.finalize_version()) + + if not is_git_tag: + suffix = sanitize(head_ref.split('/')[-1]) + if suffix: + version += f'-{suffix}' + version += f'+{short_sha}' + else: + if latest.prerelease: + version += f'-{latest.prerelease}' + if latest.build: + version += f'+{latest.build}' + return version -def is_beta(version: semver.Version) -> bool: - return prerelease_id(version) in ('rc', 'beta') +def resolve_channel(is_git_tag: bool, head_ref: str, latest: semver.Version) -> str: + if is_git_tag: + channel = CHANNEL_BY_PRERELEASE.get(prerelease_id(latest)) + if channel is None: + fail(f'Tag "{head_ref}" has an unrecognized prerelease channel') + else: + channel = CHANNEL_BY_BRANCH.get(head_ref) or sanitize(head_ref) + # A manual dispatch may name the channel; derive and a nightly keep the one the ref maps to. + channel_input = env('CHANNEL_INPUT') + if channel_input not in ('', 'derive') and env('NIGHTLY_INPUT') != 'true': + if channel_input not in CHANNELS: + fail(f'Channel "{channel_input}" is not one of: {", ".join(CHANNELS)}') + if channel_input != channel: + notice(f'Channel overridden by dispatch: {channel} -> {channel_input}') + return channel_input + if channel not in CHANNELS: + # A feature branch's own channel name is not one the server knows. + notice(f'Channel "{channel}" is not one the server knows; using develop.') + return 'develop' + return channel -def is_dev(version: semver.Version) -> bool: - return prerelease_id(version) in ('dev', 'develop') +def select_boards() -> list[str]: + """Every boards/.defaults fragment, narrowed to BOARDS_INPUT when set (the pull request canary builds one board).""" + boards = all_boards() + if not boards: + fail('No boards/*.defaults files found in "boards"') -def sanitize(name: str) -> str: - return re.sub(r'^-+|-+$', '', re.sub(r'-+', '-', re.sub(r'[^a-zA-Z0-9-]', '-', name))) + boards_input = [b.strip() for b in env('BOARDS_INPUT').replace(',', '\n').split('\n') if b.strip()] + if boards_input: + unknown = sorted(set(boards_input) - set(boards)) + if unknown: + fail(f'Unknown board(s) requested: {", ".join(unknown)}') + boards = [b for b in boards if b in boards_input] + return boards def main() -> int: - git_ref = os.environ.get('GITHUB_REF') - if git_ref is None: - fail('Environment variable "GITHUB_REF" not found') + require_env('GITHUB_REF', 'GITHUB_SHA') + git_ref = env('GITHUB_REF') is_git_tag = git_ref.startswith('refs/tags/') is_git_branch = git_ref.startswith('refs/heads/') @@ -67,12 +113,9 @@ def main() -> int: if not (is_git_tag or is_git_branch or is_git_pr): fail(f'Git ref "{git_ref}" is not a valid branch, tag or pull request') - git_sha = os.environ.get('GITHUB_SHA') - if git_sha is None: - fail('Environment variable "GITHUB_SHA" not found') - short_sha = git_sha[:8] + short_sha = env('GITHUB_SHA')[:8] - head_ref = os.environ.get('GITHUB_HEAD_REF') if is_git_pr else git_ref.split('/', 2)[2] + head_ref = env('GITHUB_HEAD_REF') if is_git_pr else git_ref.split('/', 2)[2] if not head_ref: fail('Failed to get git head ref name') @@ -97,68 +140,11 @@ def main() -> int: else: latest = releases[0] if releases else semver.Version(0, 0, 0) - # Version: release-tool's next version for branch/PR builds (labelled as a pre-release of the upcoming version), else the latest tag base. - next_version = os.environ.get('RELEASE_NEXT_VERSION') - release_skip = os.environ.get('RELEASE_SKIP') == 'true' - - if not is_git_tag and next_version and not release_skip: - version = next_version - else: - version = str(latest.finalize_version()) - - if not is_git_tag: - suffix = sanitize(head_ref.split('/')[-1]) - if suffix: - version += f'-{suffix}' - version += f'+{short_sha}' - else: - if latest.prerelease: - version += f'-{latest.prerelease}' - if latest.build: - version += f'+{latest.build}' - - # Release channel. - if head_ref == 'master' or (is_git_tag and is_stable(latest)): - channel = 'stable' - elif head_ref == 'beta' or (is_git_tag and is_beta(latest)): - channel = 'beta' - elif head_ref == 'develop' or (is_git_tag and is_dev(latest)): - channel = 'develop' - elif is_git_tag: - fail(f'Tag "{head_ref}" has an unrecognized prerelease channel') - else: - channel = sanitize(head_ref) - - # A dispatch may name the channel; otherwise the ref decides. - # Resolving it here rather than at publish time keeps one answer for the whole run, so what a build reports about itself is what it is published as. - channel_input = os.environ.get('CHANNEL_INPUT', '').strip() - if channel_input: - if channel_input not in CHANNELS: - fail(f'Channel "{channel_input}" is not one of: {", ".join(CHANNELS)}') - if channel_input != channel: - notice(f'Channel overridden by dispatch: {channel} -> {channel_input}') - channel = channel_input - elif channel not in CHANNELS: - notice(f'Channel "{channel}" is not one the server knows; using develop.') - channel = 'develop' - - # Boards: one per boards/.defaults fragment. A feature branch's own channel name is not one the - # server knows, which is why it fell back to develop above. - boards = all_boards() - if not boards: - fail('No boards/*.defaults files found in "boards"') - - # A caller may narrow the matrix (the pull request canary builds one board); empty means all. - boards_input = [b.strip() for b in os.environ.get('BOARDS_INPUT', '').replace(',', '\n').split('\n') if b.strip()] - if boards_input: - unknown = sorted(set(boards_input) - set(boards)) - if unknown: - fail(f'Unknown board(s) requested: {", ".join(unknown)}') - boards = [b for b in boards if b in boards_input] + version = resolve_version(is_git_tag, head_ref, short_sha, latest) + channel = resolve_channel(is_git_tag, head_ref, latest) + boards = select_boards() - # One matrix entry per build variant those boards need, so the variants follow from the boards asked for: - # the pull request canary compiles its own variant only, and a full build every variant there is. - # `include` alone is the whole matrix - there are no other axes to combine it with. + # `include` alone is the whole matrix: one entry per build variant the boards need. # The board helpers report a bad fragment by raising SystemExit; route it through fail() for the annotation. try: variants = group_variants(boards) @@ -174,6 +160,10 @@ def main() -> int: for variant in variants: print(f' {variant.id} {", ".join(variant.boards)}') + summary( + f'### Plan\n\n- Version: `{version}`\n- Channel: `{channel}`\n- Boards: {", ".join(boards)}\n' + f'- Variants: {len(variants)}' + ) set_output('version', version) set_output('release-channel', channel) set_output('board-list', '\n'.join(boards)) diff --git a/.github/scripts/gha.py b/.github/scripts/gha.py index 6c22445d4..5039b215a 100644 --- a/.github/scripts/gha.py +++ b/.github/scripts/gha.py @@ -1,35 +1,33 @@ #!/usr/bin/env python3 """Shared GitHub Actions plumbing: workflow commands, step outputs, version check. -Importing applies the version check. -Stdlib only, and conservative in syntax: this has to import on an interpreter too old to run its callers, or the check fails as a SyntaxError instead of the sentence explaining what to install. +Stdlib only and old-syntax safe, so the version check can run before a too-old interpreter hits a SyntaxError. """ import os import sys from typing import NoReturn -# ESP-IDF's own tooling wants 3.9+; this floor is higher because these scripts use 3.12 typing syntax. -# The runners ship well past it - the check is here for the person running a script locally, not for CI. -MIN_PYTHON = (3, 12, 3) +# The scripts use 3.10+ syntax, as ESP-IDF's own tooling does. +MIN_PYTHON = (3, 10) if sys.version_info < MIN_PYTHON: raise SystemExit( - '%s requires Python >= %s, got %s' - % ( - os.path.basename(sys.argv[0]) or 'this script', - '.'.join(map(str, MIN_PYTHON)), - sys.version.split()[0], - ) + f'{os.path.basename(sys.argv[0]) or "this script"} requires Python >= ' + f'{".".join(map(str, MIN_PYTHON))}, got {sys.version.split()[0]}' ) def fail(msg: str) -> NoReturn: """Emit an Actions error annotation and stop the step.""" - print(f'::error::{msg}', flush=True) + error(msg) sys.exit(1) +def error(msg: str) -> None: + print(f'::error::{msg}', flush=True) + + def warn(msg: str) -> None: print(f'::warning::{msg}', flush=True) @@ -39,19 +37,12 @@ def notice(msg: str) -> None: def mask(value: str) -> None: - """Register a value as a secret, so the runner redacts it from the log. - - Call before the value can reach any other output. - """ + """Register a value as a secret so the runner redacts it; call before it reaches any other output.""" print(f'::add-mask::{value}', flush=True) def set_output(name: str, value: str, *, secret: bool = False) -> None: - """Write a step output, using the heredoc form when the value spans lines. - - Falls back to printing when GITHUB_OUTPUT is unset, so a script run locally says what it would have produced instead of failing on a missing environment. - Pass secret=True for a credential: the local fallback then reports the name only, since there is no runner to apply the mask outside CI. - """ + """Write a step output (heredoc form for multi-line values); prints instead when GITHUB_OUTPUT is unset.""" gh_output = os.environ.get('GITHUB_OUTPUT') if not gh_output: print(f"[output] {name}={'' if secret else value}") @@ -64,10 +55,16 @@ def set_output(name: str, value: str, *, secret: bool = False) -> None: f.write(f'{name}={value}\n') +def summary(markdown: str) -> None: + """Append to the job summary shown on the run page; a no-op outside Actions.""" + path = os.environ.get('GITHUB_STEP_SUMMARY') + if path: + with open(path, 'a', encoding='utf-8') as f: + f.write(markdown + '\n') + + def env(name: str, default: str = '') -> str: - """An environment variable, stripped. - Actions inputs arrive with stray whitespace often enough - a YAML folded scalar leaves a trailing newline - that reading them raw is a bug waiting for the one input somebody wrote across two lines. - """ + """An environment variable, stripped (a YAML folded scalar leaves a trailing newline).""" return os.environ.get(name, default).strip() diff --git a/.github/scripts/github_api.py b/.github/scripts/github_api.py new file mode 100644 index 000000000..e441a7c24 --- /dev/null +++ b/.github/scripts/github_api.py @@ -0,0 +1,32 @@ +"""GitHub REST API reads shared by the gate and the publishing policy.""" + +from typing import Any + +import requests + +API = 'https://api.github.com' +# Pinned so a new default cannot change what the checks see; the 400 for an unknown version lists the supported ones. +API_VERSION = '2026-03-10' + + +class GitHubError(Exception): + def __init__(self, status: int, detail: str) -> None: + super().__init__(f'HTTP {status}: {detail}') + self.status = status + + +def github_get(path: str, token: str, params: dict | None = None) -> Any: + """GET a JSON document; raises GitHubError on a non-200 and requests.RequestException on transport errors.""" + resp = requests.get( + f'{API}{path}', + params=params, + headers={ + 'Authorization': f'Bearer {token}', + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': API_VERSION, + }, + timeout=(10, 30), + ) + if resp.status_code != 200: + raise GitHubError(resp.status_code, (resp.text or '').strip()[:300]) + return resp.json() diff --git a/.github/scripts/repo_server_policy.py b/.github/scripts/repo_server_policy.py index 3e5cc1631..09f445e39 100644 --- a/.github/scripts/repo_server_policy.py +++ b/.github/scripts/repo_server_policy.py @@ -1,28 +1,20 @@ #!/usr/bin/env python3 -"""Refuse a run that may not publish the version and channel get-vars resolved. +"""Refuse a run that may not publish the version and channel get_vars resolved. -get-vars settles what this run is before anything builds, so nothing here overrides it - a build that reports one version must not be published as another. - -These are the repository's rules, not the server's. The server rejects an unregistered repo, a bad hash or an unknown board on its own, and checking those here would only give them a second place to be wrong. What it cannot know is that a stable release descends from master, or that a maintainer staged a draft for it, because those are facts about this repository. - -Everything here applies to prod only. A dev publish reaches no device, so constraining it buys nothing and costs the ability to test. +These are the repository's rules, not the server's: that a stable release descends from master and that a maintainer staged a draft for it. +They apply to prod only; a dev publish reaches no device. """ import time +from typing import Any import requests - from gha import env, fail, notice, require_env +from github_api import GitHubError, github_get -API = 'https://api.github.com' - -# Pinned rather than left to the default, so a new default cannot change what these checks see. -# GitHub names its supported versions in the 400 it returns for an unknown one, which is how to find the next. -API_VERSION = '2026-03-10' # The channels the repository server's enum knows. A feature branch derives its own name, which is not one of them. CHANNELS = ('stable', 'beta', 'develop') -# The branch a commit must be contained in to publish to the channel on prod: the one each channel is cut from. -# develop's entry is met by construction, since the nightly on develop is the only thing publishing that channel. +# The branch a commit must be contained in to publish to the channel on prod. CHANNEL_BRANCH = {'stable': 'master', 'beta': 'beta', 'develop': 'develop'} # Channels whose prod publishes also need a staged draft release (develop builds have no release). DRAFT_CHANNELS = ('stable', 'beta') @@ -31,33 +23,19 @@ DRAFT_INTERVAL = 5 -def get(path: str, *, allow_404: bool = False) -> dict | None: - """One GET against the API, or None when allow_404 and the thing is not there.""" +def get(path: str) -> Any: try: - resp = requests.get( - f'{API}{path}', - headers={ - 'Authorization': f"Bearer {env('GITHUB_TOKEN')}", - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': API_VERSION, - }, - timeout=(10, 30), - ) + return github_get(path, env('GITHUB_TOKEN')) except requests.RequestException as err: fail(f'Could not reach the GitHub API for {path}: {err}') - - if resp.status_code == 404 and allow_404: - return None - if resp.status_code != 200: - fail(f'GitHub returned HTTP {resp.status_code} for {path}: {(resp.text or "").strip()[:300]}') - return resp.json() + except GitHubError as err: + fail(f'GitHub returned {err} for {path}') def require_branch_contains(repo: str, channel: str, sha: str) -> None: """A prod publish onto beta or stable has to come from the branch that channel is cut from. - A tag is not a branch, so the test is containment rather than equality: compare reports head relative to base, so an ancestor of the branch reads as behind or identical. - Without this, tagging any commit would put arbitrary work on a channel the fleet follows. + A tag is not a branch, so the test is containment: compare reports an ancestor of the branch as behind or identical. """ branch = CHANNEL_BRANCH[channel] status = get(f'/repos/{repo}/compare/{branch}...{sha}')['status'] @@ -67,11 +45,10 @@ def require_branch_contains(repo: str, channel: str, sha: str) -> None: def require_staged_draft(repo: str, version: str) -> None: - """release.yml stages a draft for the version before it may go live. + """release.yml stages a draft for the version before it may go live; a stray tag must not reach a channel devices follow. - Otherwise a stray tag could put firmware on a channel devices follow. - A draft is not reachable by tag name, since GitHub only associates a release with its tag once published, so this reads the list. - release.yml pushes the tag that triggers this run and only then stages the draft, so poll rather than failing on the window between the two. + A draft is not reachable by tag name, so this reads the release list. + release.yml pushes the tag that triggers this run and only then stages the draft, so poll through that window. """ for attempt in range(1, DRAFT_ATTEMPTS + 1): releases = get(f'/repos/{repo}/releases?per_page=100') or [] diff --git a/.github/scripts/repo_server_publish.py b/.github/scripts/repo_server_publish.py index b844ccc61..7721047db 100644 --- a/.github/scripts/repo_server_publish.py +++ b/.github/scripts/repo_server_publish.py @@ -1,13 +1,7 @@ #!/usr/bin/env python3 -"""Drive one phase of a firmware release on the repository server. +"""Drive one phase of a firmware release on the repository server, picked by `MODE`. -CI's whole contract: submit each blob with the hash it computed and believe the server about everything else. - -`MODE` picks the phase, because the phases run in different jobs. -`init` opens the release before anything is built, so the server learns a release is coming while the build is still starting and refuses it now rather than half an hour from now. -`publish` and `dry-run` upload every board and then promote or discard; `abort` discards a release the build never got to. - -The staged release is a resource whose lifetime now outlives this process: init hands the id to the workflow, and ci-build's abort job is the `finally` that closes it. +`init` stages the release before the build, `publish` uploads every board and promotes it, `abort` discards a release the build never got to. """ import hashlib @@ -16,15 +10,13 @@ from contextlib import ExitStack from datetime import datetime, timezone from pathlib import Path +from typing import NoReturn import requests - -from gha import env, fail, require_env, set_output, warn +from gha import env, error, fail, require_env, set_output, warn from repo_server_policy import CHANNELS -VALID_MODES = ('init', 'publish', 'dry-run', 'abort') -# The server parses this into an enum; anything else is a 400 several steps later, after artifacts have been downloaded. -# Branch builds derive a channel from the branch name, so this catches a feature branch being published by accident. +VALID_MODES = ('init', 'publish', 'abort') # Server field name -> the artifact it carries. # app and staticfs are what an OTA update needs; the rest are for the flashtool, which writes a whole chip rather than updating one. @@ -43,26 +35,27 @@ def session_for(token: str) -> requests.Session: - """One session for the whole run, so the bearer token is attached in a single place and the connection to the server is reused across every board. - """ + """One session per run: attaches the bearer token once and reuses the connection.""" s = requests.Session() s.headers.update({'Authorization': f'Bearer {token}', 'Accept': 'application/json'}) return s def show_response(resp: requests.Response) -> None: - """Print the server's problem details. - It names its own causes better than any guess made from the status code alone, so the body is worth the log lines. - """ + """Print the server's problem details; they name the cause better than the status code.""" text = (resp.text or '').strip() if text: print(text, flush=True) +def die(message: str, resp: requests.Response) -> NoReturn: + error(message) + show_response(resp) + raise SystemExit(1) + + def extract_changelog(path: Path) -> str: - """The newest `# ` section, which is the release being built on a tag and the previous release on a branch. - Either way it is a real changelog going through the real parser, so a grammar the server rejects surfaces on an ordinary build. - """ + """The newest `# ` section: the release being built on a tag, the previous release on a branch.""" if not path.is_file(): fail(f"Changelog file '{path}' not found.") lines = path.read_text(encoding='utf-8').splitlines() @@ -90,38 +83,27 @@ def sha256_of(path: Path) -> str: def resolve_artifacts(artifacts: Path, board: str) -> dict[str, Path]: - """Every artifact the server wants for one board, or a hard failure naming the first one that is missing. - - ci-build's package job writes every board's images into the one firmware_boards artifact, a directory per board. - The app is that board's stamped image; the device's OTA refuses one whose descriptor names another board. - """ + """Every artifact the server wants for one board, or a failure naming the first one missing.""" build = artifacts / 'firmware_boards' / board - found: dict[str, Path] = {} - - for field, filename in BUILD_ARTIFACTS.items(): - found[field] = build / filename - found[STATICFS_FIELD] = artifacts / 'firmware_staticfs' / 'staticfs.bin' - # The package job merges every board at once, so the merged binaries share a directory and are told apart by the board in their filename. - merged_glob = f'OpenShock_{board}_*.bin' + merged_glob = f'OpenShock_{board}_[0-9]*.bin' merged = sorted((artifacts / 'firmware_merged').glob(merged_glob)) - if merged: - found[MERGED_FIELD] = merged[0] - - for field in (*BUILD_ARTIFACTS, STATICFS_FIELD, MERGED_FIELD): - path = found.get(field) - if path is None or not path.is_file(): - expected = path if path else f'{artifacts}/firmware_merged/{merged_glob}' - fail(f'{board} is missing its {field} artifact (expected {expected})') + if not merged: + fail(f'{board} is missing its {MERGED_FIELD} artifact (expected {artifacts}/firmware_merged/{merged_glob})') + + found = {field: build / filename for field, filename in BUILD_ARTIFACTS.items()} + found[STATICFS_FIELD] = artifacts / 'firmware_staticfs' / 'staticfs.bin' + found[MERGED_FIELD] = merged[0] + for field, path in found.items(): + if not path.is_file(): + fail(f'{board} is missing its {field} artifact (expected {path})') return found def upload_board(session: requests.Session, server: str, release_id: str, board: str, artifacts: Path) -> None: - """One request per board, carrying that board's artifacts and a sha256 manifest the server re-computes before writing anything. + """PUT one board's artifacts with a sha256 manifest. - Retried on transport errors and 5xx: the request is a whole-board PUT, so a repeat replaces the same artifacts rather than appending anything. - init and publish below are deliberately not retried - they are POSTs whose effect a timeout leaves unknown. - The file handles are reopened per attempt, since a retry has to send the body from the start and the previous attempt consumed the streams. + Retried on transport errors and 5xx, since a repeated PUT replaces rather than appends; init and publish are POSTs and are not retried. """ paths = resolve_artifacts(artifacts, board) manifest = {field: sha256_of(path) for field, path in paths.items()} @@ -157,17 +139,13 @@ def upload_board(session: requests.Session, server: str, release_id: str, board: time.sleep(UPLOAD_BACKOFF_SECONDS) continue - print(f'::error::Upload failed for {board} (HTTP {resp.status_code})', flush=True) - show_response(resp) - raise SystemExit(1) + die(f'Upload failed for {board} (HTTP {resp.status_code})', resp) -def init_release(session: requests.Session, server: str, *, version: str, channel: str, boards: list[str], changelog: str, nofail: bool) -> str: - """Stage the release. - There is no preflight: an unregistered repository, a missing scope and an unknown board are all non-success responses from here, before a single artifact is uploaded, and each names its own cause. - """ +def init_release(session: requests.Session, server: str, *, version: str, channel: str, boards: list[str], changelog: str) -> str: + """Stage the release; the server rejects a bad repository, scope or board here, before any upload.""" resp = session.post( - f'{server}/2/firmware/releases' + ('?nofail' if nofail else ''), + f'{server}/2/firmware/releases', json={ 'version': version, 'channel': channel, @@ -178,9 +156,7 @@ def init_release(session: requests.Session, server: str, *, version: str, channe timeout=(30, 60), ) if resp.status_code != 201: - print(f'::error::Init failed (HTTP {resp.status_code})', flush=True) - show_response(resp) - raise SystemExit(1) + die(f'Init failed (HTTP {resp.status_code})', resp) data = resp.json() release_id = data['id'] @@ -200,41 +176,39 @@ def promote(session: requests.Session, server: str, release_id: str) -> None: timeout=(30, 120), ) except requests.RequestException as e: - print(f'::error::Publish request for {release_id} got no response ({e}); the release may already be live', flush=True) + error(f'Publish request for {release_id} got no response ({e}); the release may already be live') raise PublishOutcomeUnknown(release_id) from e if resp.status_code != 201: - print(f'::error::Publish failed (HTTP {resp.status_code})', flush=True) - show_response(resp) - raise SystemExit(1) + die(f'Publish failed (HTTP {resp.status_code})', resp) print(f'Published {release_id}', flush=True) -def discard(session: requests.Session, server: str, release_id: str, mode: str) -> None: - """A staged release that is never aborted holds its version against a later attempt (init returns ReleaseAlreadyStaging) until the server's TTL job reaps it, so a failed run that left one behind would block the retry that was meant to fix it. +def discard(session: requests.Session, server: str, release_id: str) -> str: + """Delete a staged release, which otherwise blocks a retry of its version until the server's TTL job reaps it. - Never raises: this runs from a finally, where the failure being cleaned up after is the one worth reporting. + Returns 'discarded', 'already_closed' or 'failed' (warned about here). Never raises: the failure being cleaned up after is the one worth reporting. """ try: resp = session.delete(f'{server}/2/firmware/releases/{release_id}', timeout=(30, 60)) except requests.RequestException as err: warn(f'Could not reach the server to discard release {release_id} ({err}). It will be reaped by the server TTL job.') - return + return 'failed' if resp.status_code == 204: - if mode == 'dry-run': - print('Dry run complete: release staged, verified and discarded. Nothing was published.', flush=True) - else: - print('The run failed; the staged release was discarded so a retry can reuse this version.', flush=True) - return - - # The workflow's abort job is a backstop for a publish that never ran, so it also fires after one that ran and cleaned up after itself. - # Finding the release already gone is that job succeeding, not failing. - if mode == 'abort' and resp.status_code in (404, 409): - print(f'Release {release_id} was already closed; nothing to abort.', flush=True) - return - # Not fatal on its own - the TTL job reaps abandoned releases - but it means the next attempt at this version will be refused until that happens, so say so loudly. + return 'discarded' + # The abort job also fires after a publish that already cleaned up, so a release that is gone is success. + if resp.status_code in (404, 409): + return 'already_closed' warn(f'Could not discard release {release_id} (HTTP {resp.status_code}). It will be reaped by the server TTL job.') show_response(resp) + return 'failed' + + +def report_discard(status: str, release_id: str) -> None: + if status == 'discarded': + print('The run failed; the staged release was discarded so a retry can reuse this version.', flush=True) + elif status == 'already_closed': + print(f'Release {release_id} was already closed; nothing to discard.', flush=True) def read_boards() -> list[str]: @@ -242,10 +216,7 @@ def read_boards() -> list[str]: def do_init(session: requests.Session, server: str) -> int: - """Open the release and hand its id back, leaving it staged on purpose. - - Nothing is uploaded here, so every check the server makes at init - semver, channel, a version already published, a concurrent release holding the same one, an unknown board - lands before the build burns its half hour. - """ + """Open the release and leave it staged, so the server's checks fail before the build runs.""" require_env('VERSION', 'CHANNEL') version = env('VERSION') channel = env('CHANNEL') @@ -262,61 +233,52 @@ def do_init(session: requests.Session, server: str) -> int: version=version, channel=channel, boards=boards, - changelog=extract_changelog(Path(env('CHANGELOG_FILE', 'CHANGELOG.md'))), - nofail=env('NOFAIL').lower() == 'true', + changelog=extract_changelog(Path('CHANGELOG.md')), ) set_output('release-id', release_id) return 0 -def do_upload(session: requests.Session, server: str, mode: str) -> int: - """Upload every board into the release init opened, then promote or discard it.""" +def do_upload(session: requests.Session, server: str) -> int: + """Upload every board into the release init opened, then promote it; discard it on any failure.""" require_env('RELEASE_ID') release_id = env('RELEASE_ID') boards = read_boards() - artifacts = Path(env('ARTIFACTS_DIR', 'artifacts')) + artifacts = Path('artifacts') if not boards: fail('BOARDS is empty; there is nothing to upload.') - published = False - outcome_unknown = False try: for board in boards: upload_board(session, server, release_id, board, artifacts) - if mode == 'publish': - promote(session, server, release_id) - published = True + promote(session, server, release_id) except PublishOutcomeUnknown: # Discarding now could delete a release that did go live; leave it for a human to check. - outcome_unknown = True - print(f'::error::Not discarding {release_id}: verify on the repository server whether it was published.', flush=True) - raise SystemExit(1) - finally: - if not published and not outcome_unknown: - discard(session, server, release_id, mode) + set_output('outcome-unknown', 'true') + fail(f'Not discarding {release_id}: verify on the repository server whether it was published.') + except BaseException: + report_discard(discard(session, server, release_id), release_id) + raise return 0 def do_abort(session: requests.Session, server: str) -> int: - """Close a release whose build never reached the upload. - - Runs from the workflow's always() job, so it reports rather than fails: the build failure it is cleaning up after is the one worth surfacing. - """ + """Close a release whose build never reached the upload; reports rather than fails.""" release_id = env('RELEASE_ID') if not release_id: print('No release was staged, so there is nothing to abort.', flush=True) return 0 - discard(session, server, release_id, 'abort') + report_discard(discard(session, server, release_id), release_id) return 0 def main() -> int: require_env('SERVER_URL', 'TOKEN') server = env('SERVER_URL').rstrip('/') - mode = env('MODE', 'dry-run') + mode = env('MODE') if mode not in VALID_MODES: fail(f"MODE must be one of {'/'.join(VALID_MODES)}, got '{mode}'") @@ -327,7 +289,7 @@ def main() -> int: return do_init(session, server) if mode == 'abort': return do_abort(session, server) - return do_upload(session, server, mode) + return do_upload(session, server) if __name__ == '__main__': diff --git a/.github/scripts/repo_server_token.py b/.github/scripts/repo_server_token.py index d17556cf7..264deeb73 100644 --- a/.github/scripts/repo_server_token.py +++ b/.github/scripts/repo_server_token.py @@ -1,19 +1,13 @@ #!/usr/bin/env python3 -"""Mint a GitHub Actions OIDC token for the repository server. - -Every call into the server goes through a token from here, so there is one definition of the audience and one place where a malformed mint is caught. -""" +"""Mint a GitHub Actions OIDC token for the repository server.""" import base64 import json import requests - from gha import env, fail, mask, require_env, set_output -# The claims the server keys off. -# Printed so a later rejection can be read against what was actually sent, rather than what the workflow assumes it sent. -# The token itself is masked; these five claims are not secret. +# Printed so a later rejection can be read against what was sent; the token itself is masked, these claims are not secret. ECHOED_CLAIMS = ('aud', 'repository', 'repository_owner', 'ref', 'run_id') @@ -31,15 +25,15 @@ def main() -> int: require_env('AUDIENCE') audience = env('AUDIENCE') - # id-token: write is granted per job, not per action, so a caller that forgets it gets an unset request URL rather than a failed request. - # Say which it was. + # id-token: write is granted per job, so a caller that forgets it gets an unset URL rather than a failed request. request_url = env('ACTIONS_ID_TOKEN_REQUEST_URL') if not request_url: fail('No OIDC request URL. The calling job needs permissions: id-token: write.') try: resp = requests.get( - f'{request_url}&audience={audience}', + request_url, + params={'audience': audience}, headers={ 'Authorization': f"bearer {env('ACTIONS_ID_TOKEN_REQUEST_TOKEN')}", 'Accept': 'application/json', diff --git a/.github/scripts/requirements-dev.txt b/.github/scripts/requirements-dev.txt index 2e0046804..4809ba12a 100644 --- a/.github/scripts/requirements-dev.txt +++ b/.github/scripts/requirements-dev.txt @@ -1,4 +1,4 @@ -# Test dependencies for the CI scripts in this directory, on top of the runtime set. +# Test and lint dependencies for the CI scripts in this directory and the build scripts, on top of the runtime set. # # Kept out of requirements.txt so the jobs that hold the repository-server publish # credential do not install a test runner they never use. Hashed for the same reason @@ -16,6 +16,80 @@ pytest==9.1.1 \ --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c \ --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 +mypy==2.4.0 \ + --hash=sha256:058165f564ccf559c68c70fec2091fca5891110480210c22594635e3f6683437 \ + --hash=sha256:0bb95cf34899e4619c61ab0a8667804e139e580b30d5df12af2102dfe44d0c97 \ + --hash=sha256:13fa24f439c0e48a290a3922fa14ccd22f2762bae99d2142931b3e40a9055080 \ + --hash=sha256:172e30b8fea631fe310f0c665477f52d9ea40bb4e99e0c81dc30118563b13710 \ + --hash=sha256:1dc0f64b0a92ae27a49d2175f0bacfa56e15bdc5b420cf92e0c1f79292219cbb \ + --hash=sha256:20e9a5cd875837520c43db98dea0b6d0c2197833d95c30127d8f570fb9b1f00b \ + --hash=sha256:2106b55105ba5ea9be4f53a24517fc5fa927ff1585edc9bc1a975abb72caef89 \ + --hash=sha256:236e0d68f6941992b0811128e652590f590db444ab29ad8f1324765b9298b946 \ + --hash=sha256:29243242cf72582b65f9582ad9e56e8cb281566ed3519f4cd70bb8b9f2977e90 \ + --hash=sha256:295ecf2e57542cd836ca537486951289678c8c7d1ee6ad74ebe29b2168a003cf \ + --hash=sha256:29eb0b9427a6b11b992e452f6cceb8af724f4dceb47e779d0b35e405e996ea5e \ + --hash=sha256:3011537be6cf1de4511c0255a324362a812b58184bbe61e15f59c8b31033bd74 \ + --hash=sha256:3adef556a19eb3b630bf86a79c29d7da3d61e541472d0d897ca01b171c0abf8f \ + --hash=sha256:3bd0e340f0ebe65c548210f53be3fd8192e83964760caf0c28bef368e68b0d37 \ + --hash=sha256:4209da39d85cf240f762af622d8180fcdfcb4727d021f44ade62d613a1a43324 \ + --hash=sha256:4a378fc15fb33e321f04652c166ce73eeb8833a97c3d218132844e938cd93220 \ + --hash=sha256:502b94b0b331f7dafe32fd6b151797ddbb4f32385b362e722c783a025e5954a3 \ + --hash=sha256:528c8744b8b5e3ecb8774f86af38d2376216816e9908317ad055f3c9c2d74799 \ + --hash=sha256:5786ef987b3767e51aaa53f20aec104c0252b42ecda7aef8e8b4cbae279b05c5 \ + --hash=sha256:5d20e6c7c35fcbf2a0ebdd0eaeacfbc243009dfd33ab7822d54e213912e6dbbd \ + --hash=sha256:615b03922d40e186fd1df73156473db0ba525c639bb4fd1cf28e1694879c9b23 \ + --hash=sha256:6306086b87cf7f8a29aa618d9fd9bffb56c59247166b9660fdb54d86d7714ecd \ + --hash=sha256:6be721bd4bd57576193653b75b4af3461c9d0bf7dd8b528f782e9be210dc75bb \ + --hash=sha256:720434d48542ecfe84d32d287b727569d3fc8f5769acd39051130e490a5c295c \ + --hash=sha256:77bdaebd452f43fcfc4cc3ba94352a3ea537cd01e3f2d0879f48673d2ec00d6e \ + --hash=sha256:7c4f8f8d1d1c0e2832d8ee7113dd08f6df6c7aad9e863fcbed9f25832be0b8c4 \ + --hash=sha256:7da85fbcff6dac1abcc636707bed38b45598131fb7a605d9719c70b5cc733af8 \ + --hash=sha256:7f38f57d344f8b6accb40e01c3d83cfc590498231724d16c07ffb7940f157818 \ + --hash=sha256:82d0f94c8587ccb472622ee7795280aaa38a06640d5f45b3f16909d6dd86a989 \ + --hash=sha256:86d616fe84c6eab8026f8c50ab5bcb90db780d2ccd233d971e34e92bede9b359 \ + --hash=sha256:9279488933040b638c0ab739084c0ca100efeea6db581bf5d7628d8e89de53fe \ + --hash=sha256:970b221ed5842213d98e3c480c08f795ace4b1f81fb21e1b126bd0476bce1c34 \ + --hash=sha256:9f03a7828cca2b0adcd6662aee8f2711ff8830e1027641fdea3ab0b787483566 \ + --hash=sha256:9f459f0b4f0596d9d51fe7716b404b35287b99e77da98a7af90a65dd5fd61141 \ + --hash=sha256:9fa247e02b505a45a2775f69df38d360d197e3790bc60f717595db9eda358b6e \ + --hash=sha256:a3f86fd1313dd69d013e265f1fdcd12ea7a9d606f9875b2a3db946cd334555f3 \ + --hash=sha256:a6e851b82c0661f69f1630fc16172c68787a6a9cf0991e7c6437d60976cdcd76 \ + --hash=sha256:a96b07a49b7b1d025ce59c1b3acbcf24bead9a83da4523c4a6bde1bb94e7a0e1 \ + --hash=sha256:afa89837d9be67e0cadfa33bca3bb7efdda98c3b07e74dc3b635ebfb1c8a926a \ + --hash=sha256:ba05652540bf12828e52abae807b024b09ca144ff4f75e2450a81d69c376425b \ + --hash=sha256:bc378bdad4e9f12b5bd96466083d1e71acf00594ec9c7b2bdb5e02816f77f303 \ + --hash=sha256:c9de622fd397495695d0598ddc789222bfcfec9d7c9ec3a1e385c855e3bc5e01 \ + --hash=sha256:cb734b2668c1f40d07ce093bbeb4407e9527c67901627b0e1679825be3f09975 \ + --hash=sha256:d01c5d26a352acc6d5cf3128225477e1e8465e8d3029d4c345807fbf7f3cf093 \ + --hash=sha256:e05ff2925d8b37ad26c80c1b9dc43ae5d455da2df1e23c24c095a6425917c57e \ + --hash=sha256:e1fde197ae65be856a034a91b70ed747a16562ca69577785f06c661548424bf1 \ + --hash=sha256:e3ebe2f72a2a1156065a9851570ffbf50c0a93cdccadef9c6e05c508a4fd10b1 \ + --hash=sha256:e76172710bd4e5eeae061abfd68347e5264632e02778be61784671ae3a2132f5 \ + --hash=sha256:f83353e47ab520bf6fd4df8f5897d9fe081211f2fbc4b7d37736a3e3c166cbcf \ + --hash=sha256:f9b028548b3af480e2b1ed8df14ccaac86f99c9f600d1580770ab7ba3dcd40f0 \ + --hash=sha256:fb443e81057896132d3642d6be219e6efd158691ac7883e3ba8fcb469865f05d \ + --hash=sha256:ffda5244fd1ad71a1e54405e35f50d09b80c3978efa120f58bd1252ae32c62d2 + +ruff==0.17.2 \ + --hash=sha256:122258f6ecde0e0dd1bb60b24263c3edde4338caf723fca711cc729f9bfd414a \ + --hash=sha256:195ad918027f817c119310ca89a42d347887dfc28934aaa869325e9c5dc9d962 \ + --hash=sha256:1b59d447f197bef9ef48a22ba1a6016137753c6099e9bac9c36eb566e422424c \ + --hash=sha256:2d086be452eed686aa51271ecf2cbe5037956517cc6ee39a17b4489ac12f8bbe \ + --hash=sha256:39c5b55c960515cdb32be16e69efb5734c589c06a0eb38a22694524a336b889d \ + --hash=sha256:583bd67bb9e7061dd09bc46a0a884bae2bbd214dbcd48e509813612619683869 \ + --hash=sha256:5875a16b0de70b43cb81eabea5ae5e8f5d1b2fd1b5e6ed07f80cfe5019d0bd5a \ + --hash=sha256:6bce7d1f5165701ae18f5df85c65afe78cddfa6c5fb4b9bdb0014d60388f3edd \ + --hash=sha256:7cfd783590edf2b77317882631900f6a13ea2e8eacefe6ebf2370d2715299160 \ + --hash=sha256:7f8fba429d6ac4128ab29cfb8fa6240d2fa671d04adf7f84d3b22fc7b6b6bea4 \ + --hash=sha256:84d632402d14c23b488f941db23712d6736a013225162062d34558c3f3c780eb \ + --hash=sha256:8c21b4ab24485358f17291a4f8dc5399b834e963b8efedd920fe4fdb02081bb4 \ + --hash=sha256:92a80d2f44584dabb0189b3a6c393e37e525e04942838482010575cb81945be7 \ + --hash=sha256:96a65e755d8dba094707fb23ab602111d9a31ec2835b3a6992a1914a8fcb211e \ + --hash=sha256:b7d0906677d89bbf3cf73cde64b5bbe2ab35df3ae43e49ee2627301e91265bac \ + --hash=sha256:ba405cbd5ec7bda1f555d5e52be1fd68d02608d4d95d24591bdac92cfacb579f \ + --hash=sha256:c555a877f8f41447e318101cfefea9bf12dfa9f82eedf90c376e57e069f917d0 \ + --hash=sha256:c60f93fec7142dba5b8f487f94bf2f6231dae24dc11e4c48648347c8b0720004 + # --- transitive (pytest) --- colorama==0.4.6 ; sys_platform == "win32" \ @@ -37,3 +111,282 @@ pluggy==1.6.0 \ pygments==2.21.0 \ --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + +# --- transitive (mypy) --- + +ast-serialize==0.13.1 \ + --hash=sha256:02686b281ec9b8e9614f6f0db27e12e4cac515f336c5d05b678bc9eb73ff2af5 \ + --hash=sha256:032b09d86c785c3984e975378b7861f2a68ac44e22ba7b07b93a536b2524eeb8 \ + --hash=sha256:0d764913374ba21df8cd618045b371ed104ce55c6276de7c59e554a4f5ba26fd \ + --hash=sha256:121e1d08d58b75c15491cc3003609cbad71ac2b0016043c36d30a6b2032b6605 \ + --hash=sha256:1b158f1c9e3a0d386913924c57995d6df78224868a4d0f215bdd41587c97b34a \ + --hash=sha256:2015fd0165e463aac3b1aadc6a1fa1e5820c14ca950821f4fe02f8a3f6a388d6 \ + --hash=sha256:23d768369b89d551077dc173da6892fad00443338417523b6732f4e8ba4a7ed5 \ + --hash=sha256:2876444a5f87a7c76f371af3c07c2fda563a08bc8af52568df812e8aba4bf568 \ + --hash=sha256:29422d7bcd9e7f39411571c36cec45ce8a7a9b3ad4b2e7aa64497c15a9e397f6 \ + --hash=sha256:2bad302ae2427454ef9dab71bd5c3ad67188860c39d36daee38f72031edb39c4 \ + --hash=sha256:2f9a9ce8b0994f757b941f7f7fe384a6666b4162b941ebc5c248081baea18e47 \ + --hash=sha256:30cf89b6a6844542d116c758725d13de5b97b9410a51ee577715f761270d0c1c \ + --hash=sha256:3e129d44b0cffbea24a7f40117ce49e30993c3d119e584d4817ebd6f87246456 \ + --hash=sha256:401cc84e8888d4d76f2c8d296fa006ed1a5ff993b6aff90dd777c03973efd025 \ + --hash=sha256:43e8ddd63e80fc280c2f790157386f27f9b9f6ccb9c4f8e803b863cbfb085769 \ + --hash=sha256:48f11d96c9ec71119612f8e79a8c55c12f2b136716d68347433c561d27a25f43 \ + --hash=sha256:4bcf1d5f4eefecaa325a34ecf079b2741189e7d0821523f92cb343a389a67489 \ + --hash=sha256:4e385b17d5bde892a6184003a9aff3518387908b73f8f5ea539313c07c8c0ede \ + --hash=sha256:4f98da562e5901d69c7dd30885594c5438f859edf354e9ac7517d188fb026c3a \ + --hash=sha256:523d45bab16ddaf1399c04c4fe81bbf180f81e33c2f7bcbe76be56b0f7311ab4 \ + --hash=sha256:5d99910e6b9a9b9360bcca2ae99aee36cc912ce058df2a57d1c33099b6cb2593 \ + --hash=sha256:5db12d843d931b9d1f1519ce66b9cbfe72d8563b75c1204d255322254d1964dd \ + --hash=sha256:638333fb8ae72c62753f9e794fda20290883ddc65b0d85e26e08f8bb227567ad \ + --hash=sha256:68f8829d114dab6c9db4a8bbb8d0e342958faee1b5e15a29022678df639960f8 \ + --hash=sha256:6eb0be64c9b8f1ed26637eef345abe75ea90351c92c85a8aec7838b3af971011 \ + --hash=sha256:6ffb0f548a7dc126067fdfb7b2325ab5dfcdeecc29faf5e0f0461c96d6ebfa3d \ + --hash=sha256:732aaae9e498c973fc63fd7f308009b57c4afa6ff59d0b674d894e57d0bca01a \ + --hash=sha256:75d7750fe98f6f35a96c43aba8d98a2e623e22c778a0c17ab9ede92eb10f5d82 \ + --hash=sha256:77a6201b69ce42e919f2fd9a559138d5410b5d8afff7c3a6b1f500b0481ff7d8 \ + --hash=sha256:7a5c38bdb93669c596f94fb7204dd6ebe4fadb19058ac9a2faf2948c2a175f96 \ + --hash=sha256:7f2427bddfca208e84de700044fadf9aef2769a70bae740a1bdabb89ee9e1117 \ + --hash=sha256:7f3e8a3eff69af640db985360fef35073fb46956dfc3e5c614cf00920d2746dd \ + --hash=sha256:847cad4c22953823f5e76d0b126c6f9da670cfa45b1df41b9324e22189d77611 \ + --hash=sha256:87427913c325adc811e81e7c9a757b061f47ef1e29114ac479aa0953a2687a98 \ + --hash=sha256:885a62ec6481ce58ab24c6357360d7a82023b62dd7e5e7ceccf59e9662a3272b \ + --hash=sha256:8899c038b1c15ae8b98e379057fb3ef398ceb0b21d1737a8156dd5a0923b7e51 \ + --hash=sha256:88df960e67184a71874e5b2ee014f87d1254a0dbc1f903c7bd7b1ab7733ecaef \ + --hash=sha256:8c8ebd419811c435812d242fb2a173e57dfede198c9ac9a0a584666ebc9b9331 \ + --hash=sha256:9221e09496fb942901993c1812ee87950cc0ecbb514c7c0f1f5fa6d78f701ed8 \ + --hash=sha256:929eda0c419e622ab49df3a4c45f81c57dcc10874bbad55d0470a4a03324dac3 \ + --hash=sha256:9f99786e0d62082a5b121b3437f3167562133711df22128de1e9886f57f0b226 \ + --hash=sha256:a01b515b25a03184eff4acef25b981ae007916379682e908fc8785b342a0bf3e \ + --hash=sha256:a789cbe14d0a0e9ec9b32a456e5abc81ad8799479c3cfd88a29e3442262aebe6 \ + --hash=sha256:a8ab2c2585b2e22e48cd4aff2136fce4eab4ced569b85f150ef9a5a5d6f34d49 \ + --hash=sha256:aca07de3ca8e3d9f2b716c3c31f4f384fe4bd2d03e09912417cff7a3e08036f7 \ + --hash=sha256:acf4e9f055d75058ab01536da330648746f2f82b7697a85230547f2d5ebe895c \ + --hash=sha256:ad88921f95b92bc144d9ad059a0e16f8c1ae69f5239f2d3c6536fd2774f73dfe \ + --hash=sha256:bf53a3ba1cbe45399fc1e4ccd684b8f2a11cca450f89706b79122f478534c6f5 \ + --hash=sha256:c652d9f5d52d8b81eff3ff2c95c9dd0f281de45f6ad5d3fa95527550c396454a \ + --hash=sha256:ca9acc67f796aa2554fa3037dccc022a4b08b38b2ad153edfac9c9bb7cb1f3c6 \ + --hash=sha256:cc34c670747d85d4232834665e247009413eaebe47c69eaf60a5c439aaecaed9 \ + --hash=sha256:d9c1a920b2aa0c1e016bed7721689717034ca0dba64ac3a3ff243b003d4e8dd8 \ + --hash=sha256:ddf7cbc9c8abf70102c4a088efe154bb86bfb825b0a10e556d2b15972cf99bd7 \ + --hash=sha256:e14c4de98f965647a74e6eaf37e20207124e45451ccd36e84a6f6ebab45e0372 \ + --hash=sha256:eb41e1906502ac80d24ff343f45008552a3f5b5158607959fe6a07dbfc6eb037 \ + --hash=sha256:f8b17e2921e56621bcb46792bd3e0b82ca3b85de78d5936e7b8174ba1758b0aa \ + --hash=sha256:fb8021f4dc3296de18737e3b8b34994411da725374d74a0b9eb32a738f2a23d0 + +librt==0.16.0 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:001bfd59a7d45b17e3e75f2a8c6405280b35e7b84471792778e718c4f368950e \ + --hash=sha256:0058f9d68721094105917254c72ac0569117bb7b13b9769cf45d26d89f9d21cd \ + --hash=sha256:02118f56a9c36ddd07dfd9b919d9ecc117ba20a90987d56aa4c429fa34509188 \ + --hash=sha256:0253721561787b8df8443eb347b7a6461015354e5bdd37ee38a41fef220d2bb0 \ + --hash=sha256:02d89c813d5ff74b17df72d3a34819d132cd168e56b81bf755b809bd9e46b8c4 \ + --hash=sha256:0314058469f4d2fd279ce7c62ac274ac82c3918ef7db62ef0697c4c359370155 \ + --hash=sha256:0dbe4096a7ecc00fa835d24510ad8545a4efef738dac96e0e63516783ccde905 \ + --hash=sha256:0ead24d2562a49473dddd9efef8581f020007eb0054389c3ee3ffad38b1ca4c9 \ + --hash=sha256:13b4e8aba90b0b1c82474e9844aa9ffe7ad3faa484350e1da64cb8188d903134 \ + --hash=sha256:14ed6ebe3e4f85f326d7920011ad30ff49ed9334e62cf88caef9ba973d9e3a92 \ + --hash=sha256:17bac7f7a16b328fff77e440287693eb017abde913595b5827ebccbc21ecd8a6 \ + --hash=sha256:1b384b90ab79a7bc30b566895809a636e0666f21f3cf12b54823d025b7e83839 \ + --hash=sha256:1bc17e54e5305f8d40b7ca203671ff5a9e59c1d0f8ea0f625dcca53a3984de11 \ + --hash=sha256:1d28ae980ae2218f9c5b95d191e947296f918c9bf0b400d467a9430275bbe678 \ + --hash=sha256:1e511762a074005bb0aa569166779834e75e438370226930d0ce1866d4b6a33b \ + --hash=sha256:20fe0bf9053885e21c62b3e091fb5e73e1c54d1daf2e70eb388d70763bcd4220 \ + --hash=sha256:242e00b3d4fa37c3d3c1ca5f5c9adb7d909ddb1eac9c41f2787320d00caa0af2 \ + --hash=sha256:25a58a19ea8d83b68209f04912df765e9260635ef77646542ed4b4abe6bc7940 \ + --hash=sha256:273d00be33792a15189331df10f1f1331621b043881e66c6c4377f7f776e1291 \ + --hash=sha256:28e038895b998d7a0c7798922ce8a1dc157675df5cf1c9ef0aca809ed804b7a1 \ + --hash=sha256:2bec3818c7da7c96ceae0ef5915a3d16c52dd08f3ea913bf1fe8568c447c7978 \ + --hash=sha256:2c4aa329c17bd1aaea4f6e89335d8ccd494b3a5830b6654462273e50e11023f0 \ + --hash=sha256:300c3ffdc459f4a779a8411ecb188e3ac0b1ff3a3a7b099642555dedae06c69b \ + --hash=sha256:30b7beaf3f4487b7d8adef1f158b49067cb4d5a19fa7a3bf31a4e7a820e435c5 \ + --hash=sha256:314e703f0c19320dc8094e7a784b9cf29e1b67402515abf580069a6363c0b4f1 \ + --hash=sha256:33f41443a1f4e1f099331b3d8120e409fbff84b9760bc1cc9ea496f37ddaa5cc \ + --hash=sha256:349c0bcb87ebd07481b6ff781e25cdc699723dbe2212e57dabb27f7a13b7b87d \ + --hash=sha256:36e53948e99bbe3ffea257124cfcae1cfb01831555c9a9c903c9f9a72db7fd07 \ + --hash=sha256:375bfe6b572a8f6cfc398709356046173bf27e64c4c5edaf5f7062f051fb4bf9 \ + --hash=sha256:378dfaffb38e59c24a87cde5713cd865d51ff7383fa12947f3907f306ea1ca55 \ + --hash=sha256:3931f7a3db322e7f44e02a280e3949326ce9579ad388ee8d691dc7c76da9fb70 \ + --hash=sha256:39ca4f2f2fe05de8e63493da592d84311adabe5bef52b193851981da9816b302 \ + --hash=sha256:39ec1d5a14e37baf1450a6cabf03fe552340808bf1ad9d71824ab90117716459 \ + --hash=sha256:3ddeb3c9dedb461bb457c6c7d9aa7fbf35329da313d1a7543d00c8d0f3473c96 \ + --hash=sha256:3e0c39bdc85370422e8b637be76eb1fd07d30967551b03e62267dd156f553152 \ + --hash=sha256:3e483a8d69ede8067db70c0e83007423b6925de6fd53afed01d66160f2e9398c \ + --hash=sha256:3f0b8114c44b2ac06ff5dacd08e07e8e807ff4f46083f2a1602685122559be41 \ + --hash=sha256:3ff4b2367926b69c6215635902cccb04048e73094e9862900d27cb2c6bbff143 \ + --hash=sha256:4323193ac0cd025f85af531df8ba91bf24d1973b401697347a6282e8fd3fcf5e \ + --hash=sha256:468df902df016a06eb0e40b0747dc8d14e47d7a38b18b63b1fb167d85cb94d63 \ + --hash=sha256:473eebc7866bb0a0c8849a292b5e7157c1aba5d14d0f0f610c52158d6d964262 \ + --hash=sha256:47ada6ea32636492c61aa8ad27ae3b9404bfe7a97e3ba946d1984236cc741da0 \ + --hash=sha256:4b6183e2e2e0ee00aac2ec07c7f7d151c97e85666b304f574b71cff0f9fccc4e \ + --hash=sha256:4e29522c62e28595ff7e324c6834ade51127707f0e255b18d1c1cf03d39c1048 \ + --hash=sha256:4eb1313a19847089ee81e88742abedf285c60538816640b99742d8534b81d26a \ + --hash=sha256:52327da75a94012e7f932f913d20d3876bed3c102be00e6c3e8600ff7bdd58a7 \ + --hash=sha256:54d11f726aae9df5a6ffbbf0a03a52449bbac84a53ef03669cb41cdfd4ae41bf \ + --hash=sha256:5696d7f52e7b37217cb3a8f92c744fe835942602fdd4c1a8bc4741d3bfdce15e \ + --hash=sha256:5750a105b42a416f930edc59054927a406effb2550cd5bab92ad7a5842ed5d05 \ + --hash=sha256:5810ba811297fdf37a1531a57667cb8ace0842013ca8606bf9eb7c24cf4be154 \ + --hash=sha256:5981c011b306781ce561e18e14230a14524a3d8109b97553666c942c18f31a96 \ + --hash=sha256:5a269c46ae327d8e6f8c1f85f7516cb52c0fa48127565a1105a4f4a05ff2a0b4 \ + --hash=sha256:5b976054553670829985ed767feb78fb6bcede0175327c4844dd5c281c1be659 \ + --hash=sha256:5bcc2c4726ced915b00de0c9856a4eeabfb3fddb93e10e0b8f735b7709358b6d \ + --hash=sha256:5cd5b092441053364af968ea12084692cb9d4a22f3ce9524e377880bf028761e \ + --hash=sha256:5f49cff01bd608ef7d97104cb035c75455e79c2d70bf4a506cf773338ac1860d \ + --hash=sha256:6072e92dd876ff6ceeb6cf371e35e51f479349837391341f479b08df4564242b \ + --hash=sha256:64c79520414a3fdfc6aabd7593e6169afa14d5f8d9908d4b498db068868b08dd \ + --hash=sha256:67e718c7a43f8db325abbbf1404e2d535f12f8f7a1a82259568385cc5274b82a \ + --hash=sha256:69ba927445cfaaffb4081003ef5224c55a5c2ab67ef956f416ef744916e44121 \ + --hash=sha256:6a63610fa76524edfa605b5b259a603915c7a6e10e54f003e5503030506e81de \ + --hash=sha256:6c5da27e8056439f927ea896735da60e616c477a8293feaa3233d4e7781a6726 \ + --hash=sha256:6c8893eae2fd13c5488d94056f3e6e5cf3142bfb1c4acaf136cb33d760c5964b \ + --hash=sha256:6d4a64283ee61824b5790de882bc68e2d9d7a5143537cb7a966f7354f71646d4 \ + --hash=sha256:6fe436af2eaf630474f491af5d032cbe45f93fcff5c3b9fe4ab194a7255b20ff \ + --hash=sha256:71b93b42784e25b975079573c642a8fedb049a7bb31d70a51721b1666b3b2ced \ + --hash=sha256:7393c9a48dcce4817dbd4b0d8ff6237efe9b0a0609f5b0adaef315f8541726b5 \ + --hash=sha256:77c7a2b4fe2c1369e0d5aa1cade26740a7b14be32fbc9a5535d617d20065c39d \ + --hash=sha256:7a1d272724b581bb6bc769dfdafed6da2ecc9886ba2450311de55a4ac2e1e9cd \ + --hash=sha256:7cc365f006891afb006b52d5ee5ee74c09306ffa20e2f8705a32a4450af2f3ba \ + --hash=sha256:7e510b7770bee609617a3374a96548eb114cae048023e3f049ee449e7ff2db32 \ + --hash=sha256:80039ba9b6a7d5f1a0175a4cca6bbefead87bd854c80abad1cb30afe47a830db \ + --hash=sha256:83d4041a3d9b2fd053a8a4e1f22878b3e5833e2712956382d5c048d791454e91 \ + --hash=sha256:845a511b60ca43b9880dcc84a9784c891d6a2098c829130b320846c69c9c0c68 \ + --hash=sha256:877698bf6bca5721d8be345f2fe09778e40ecadea8b58c73075f2b1a53666bf2 \ + --hash=sha256:8caf96a4ef8fb27d0ac0d1ad8337d26a240acd4a02fe4345d0a8f264753e8f99 \ + --hash=sha256:8ceafb70f2a4f0826f11031942e59c0728fd98da112dc346d4352bde1e486866 \ + --hash=sha256:8f36c58e33b304b525c6c9c5076399c6ebf1109e17b9051a05a407b091b9215b \ + --hash=sha256:8ff5d26c529336be9bd7ae04483235d77778ee7d6444a95353102b542601ce81 \ + --hash=sha256:909d8e3c1faee44cb762b1c519ff8613dcc5ceae5c99987a00917b5a31fd1d6a \ + --hash=sha256:92caf82ebef5e12d21c72242b70d1e92536f1711cf2a727a4c276de4b4469087 \ + --hash=sha256:931a0bb0fcac88f263e269e46eb30ba8e21402cd3c62ca40cb97034c0693fab1 \ + --hash=sha256:943c6bbecbdf7fa575a4f2952fcfd848c88ef95507c3fca411e89d4ac3ff8143 \ + --hash=sha256:94aed6a8308818b91677957d1bd03188869cd7aeb23c5dba7912a6c0402f7602 \ + --hash=sha256:94be5cb7bca4df6201f4183e9e4fa2086c655283d20b38cd84500a69057575a7 \ + --hash=sha256:953107e2f68d0f3512c48f898b0dbf0ce5cc52bba0f318d847c985dc555ee4cc \ + --hash=sha256:96f576f2711f8519152ec76d0e599243555c1f07679fa73606ca8c8c868c0be6 \ + --hash=sha256:a33e0dae1f8592146a4764d54ce842b278732d21a84e17c3bbe6b1bc158a2248 \ + --hash=sha256:a4aaefb4ba6c07e1aeebb2795c8958148f1d6f9af3b555b53d23d766edb6d67a \ + --hash=sha256:a8afb6557920860b7a3a596eb804cf37e09e7cf8a803db2478c202acc72d8c2e \ + --hash=sha256:aa9357a1b4d4fc787bb718a59cb1112c28c8a976d6bfa268b71cc0a4ab8f3a94 \ + --hash=sha256:ac38d6d8d66bf3d744148dbbc0b8e193e195a51e364ed55e224631f5721891fc \ + --hash=sha256:ad37d5b9abd49c9a655dcda7ea52a8a752884062ef1ee71ae17c2f2a0f81fe6a \ + --hash=sha256:aea7b1f2b125dad5de85f049136651bff256c883c65e6b9209b2da0a1ac3cdef \ + --hash=sha256:afced3dfc17cd805ecf7a3d77996a71cf5f2c75aa66eb0c21a9930f4fc992f86 \ + --hash=sha256:b0e3e721c75d2e79a76d4422c79d7ba705fe1bbafec907037fe7a657a480a0e3 \ + --hash=sha256:b6d085d70bce51d43c5c7c36d63490770180d8779e71c49305c87b4213918de7 \ + --hash=sha256:b95d5d92ab83d39e760a52091bb1baba664f3a2351e39b1e16801e5747c2f0e9 \ + --hash=sha256:b9d6d4b14e92d876f8026b54c20c445f36425214c1081dc76f74e40db386b82b \ + --hash=sha256:bc02954b1295de798bbdb0b4e2d8a28c2117de8b5c73dcbeb27dc32572dfb971 \ + --hash=sha256:bd3150023d3dc2bc70f3784e59ffa1140d56ddba3d8125b3d6f9f85221279bfc \ + --hash=sha256:be56ba9c884143495b517f23fe794ae367d58cd89ea0fdd6d437e3c024a87f9f \ + --hash=sha256:c17194318e4c0c0348b36f36c2ec7534436fe0a4c15582403162a4f08c80797a \ + --hash=sha256:c3d1bb7841a816ace6449bb26d3f9560dbfa20e71c568d23f0f62bf1e68f50b1 \ + --hash=sha256:c43bd6e642d8a248c114327f98dd25ac5a7cb5aa168ef02f0559b91874df16b8 \ + --hash=sha256:c5db585d43449a5f54303d4b2774e45e1babd975cfe1630a3d708c0b80c3e560 \ + --hash=sha256:c5e6144e68b577f157519f2ba88ca20e3ed61c29b00e5cdfa76cd2d45acf059a \ + --hash=sha256:c6f1b27bf1632a7e016af9f145f82be95e1edd7721a646505c21059257cb5a04 \ + --hash=sha256:c71d1b76210a36729fedfc5115069b50a3d8619054745f8758fe5d6f19e86671 \ + --hash=sha256:c72c5295a84bd249526da9bdca38f2e176d15c31c13bb0063c5053f4ca023421 \ + --hash=sha256:ca8052401c55d7511dda6760719fda7618067e83535d7d0010096d216c34b667 \ + --hash=sha256:d1aabe3925cbb4a08d15b7b20ba4011b53019da0c4173a25155139b7b1baed65 \ + --hash=sha256:d3c94211ee0c4f8d649ec06b7c115c0ec4eadb873a0e3154ca15cef3f814b071 \ + --hash=sha256:d46ca272b251d033dd4527b0dec5f261a28a52bd5fa0f99c117b0a1f8588cc2d \ + --hash=sha256:d608f0bf3b8cbddd0067fe02cb8cab7d13e8eb9386b23a1843d4363044fd9e22 \ + --hash=sha256:d6a365f2ab45a984d0e00eee0dd17f599ceab8cadab6ea07b6111c8132fc0e42 \ + --hash=sha256:d92db7a0f6aee44f1baee94750457e8d2d1c6ccea41842de6268d34e8dc7eddd \ + --hash=sha256:df183721229ae51eef90108c115b43e98cb169b6155d34480338e5fc6616df00 \ + --hash=sha256:e05108e0849966f53a8d2d3112a7af881d0efaa479bc735bba91108f9f2350a7 \ + --hash=sha256:e1967e36ac4cae0c7e9615ad32e1a513cdacff79f9e8afb28bedc91caf48b4f3 \ + --hash=sha256:e42f8e098b9c5396fefa05fb1cc7e33b0e08fc51da106b5de4a45fd22aac6743 \ + --hash=sha256:e56aaf8c167548dc8e5d6f3bd0f48dcdd299a23c73be3f744aab79d99e9c7f5d \ + --hash=sha256:e9ce0bc440e7fd09b5f51f372f0f6640658f854b8fb05260f819cc669c93c42d \ + --hash=sha256:ebefd60b42e2a82b32d136bb5f7c94eadfcd29f772b547df6f3291d1ed855a1c \ + --hash=sha256:ef46c1a29ffb8c72e882e22618ec618778eacd0578fb22c6e7cf9c11d15f357b \ + --hash=sha256:efc49c462d4516b8a58b00b490078fa64689fd1fe66970cc190131d7afb8027e \ + --hash=sha256:f01f3805f2dae4781c0c34b440e31740d082950bdaf89a6f601ad589a28af57a \ + --hash=sha256:f06c689cb14afd9b612727553a5ec5a40febf113ca41c4413a2b0b334285884b \ + --hash=sha256:f1e8591bd8a5a628cd7f07954c6a1592359a878bf032957a8e9057a41d644311 \ + --hash=sha256:f4462528b6000afe8f16907b5c7c2553abf1df005ba5140e6eb394541c3624c3 \ + --hash=sha256:f7be7cf555bc30ec12622e9447299cc4a9b8ff307548b634794353db0c2065dc \ + --hash=sha256:f81b5b19ce748ef68d4746656b7929762eb2fe99269b266e4be07e2ee4de7144 \ + --hash=sha256:f9807485a908f00355820f18e91e045ffdcdc5adb68aaec40a1e2b88c5f7bba1 \ + --hash=sha256:fbe4fb8c5445f7496d7f7f6bb0807875d09d47e6771ffa175fb2df2895fb86ba \ + --hash=sha256:fe4372c52d4849096c6cc1cda2817d293ec51440c890474ed59ef38d46556f18 \ + --hash=sha256:fe52bf4641069e7978a14253b036cb9002def1926317e710f2e249f8a8c47742 \ + --hash=sha256:ff7baa55f8e7c69851419e50a666015d02a74198716fd45c0125a2112e0a389f + +mypy-extensions==1.1.0 \ + --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ + --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 + +pathspec==1.1.1 \ + --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 + +tomli==2.5.0 ; python_full_version < '3.11' \ + --hash=sha256:069435bd5480429b98c5e5afb02ab21c219b6f0064680671c6dc0d46817346ea \ + --hash=sha256:0dc598040da8d42cf20f0be588ed7004f46db12a0ac6c32e03a59dccedaaadcd \ + --hash=sha256:1245a6638fc4bb0a60af38a7d45413db34a13842027c77597c712c998c62fdf0 \ + --hash=sha256:19b0dd8749f4ea2f112c5fcfb3c5248390c899d7e2e173f1d91abee1fa0ff391 \ + --hash=sha256:1f4a40d03fb9f63424f0979855bdeaf44dd7696b8d59501822c10ed30ba532df \ + --hash=sha256:20aa36de8f2cf87237143bc1fa1aae8d6612c09118f4da21c6a684db5dd1f6f9 \ + --hash=sha256:21e4cae4114aba25aa0d4f85cdf486d290fb35c0954d7bba536248da64d43066 \ + --hash=sha256:22185fad8a1e622f064e78008018a0dd3323550dcb479cb7a1d296888d74024f \ + --hash=sha256:2419c2a189551987b59d80e63ec355671283336f41c6b9b89462df679c7d0c57 \ + --hash=sha256:264507556cd8b8c8e7c6ee037cdf443a463f03f4c958e57195e3d369711b8ff6 \ + --hash=sha256:32a7b79ac57a2e83670ce329ccf675798bc5a2094783a63676866b70503f2e2b \ + --hash=sha256:3f89d10c1ff6a38d992c27fc8a4816af71a909e08a40ec66934240b1e74347c3 \ + --hash=sha256:463b16086865b97facd8d0b3fb4cb7c544e3f58d2a69dc3113d6db9653fdb043 \ + --hash=sha256:49096930c8d886c9bbdab62d2d0d17ce823ddeea522309a190b36245d5b49e01 \ + --hash=sha256:521345fd1f19d45b8df87657aaa38b6f2ca3800059fadf428e7ebf479a383646 \ + --hash=sha256:57b1c3b01fab802e2899bc3d168dca320e14165e2fd9fd584760fb4ca5826859 \ + --hash=sha256:5d8bac3d603c97e6854424e5b2b5b741bdbde387e09f162fb0446812b4a8362b \ + --hash=sha256:610b27d99f28ec5f191c7064a48f3ddb179a1fe6ca73d571483ae859f57b605e \ + --hash=sha256:61ea1ebe1e55a34ea8199cc8dbff398d35027b82271c8ac4802fd3a1fd5b1bcc \ + --hash=sha256:62fc1bc8eb03e3a9cadfca713d65614ed8e09d974a283295ffe3a831976b4dc5 \ + --hash=sha256:6664b7ae7af7294256c53960a6103077f4914cec8ff98479c352f622c6f6b2f0 \ + --hash=sha256:667e521b37a6c5ccaa044202c235b530f90177ffe2cd4a64ecc213c7dd535feb \ + --hash=sha256:69491c143d2fe063046e0301e62a810bed338fa4d1ce0fd870c27dc1e09b0d84 \ + --hash=sha256:6cf74416bdc94ae458b14e37286c1073081850ac8459a00d0c5efef5d44294c6 \ + --hash=sha256:6e95c7614e705bfe2b04b27aa124adec59752d15813df37e2156747cab3a006b \ + --hash=sha256:6f041843c4d3a37245c0c056fd955b186bf8b1fb85690cbe40b81230891dc34b \ + --hash=sha256:752e8b1aa6a4367ef8bf6a1a1e005540f7ed055ba36d7193796812ca5404eb52 \ + --hash=sha256:75dbcde8751b0a960aa3de173aa5e894d590755c6d7758b7e774c06f1dc3cbdd \ + --hash=sha256:7ac2027d37c3afbdf4bdd377f2676f6f1d2122a5be1f1137b49dced590b37e75 \ + --hash=sha256:7ad1ea345759240d6463efa0ed1c704402752e49aa21476620738d74d72d8aa1 \ + --hash=sha256:86665cee9c4835b7a7f1e8ec2c719b5258d4dc782887aded5a8ae7352a96843b \ + --hash=sha256:8ff3a2ca028c7eee0c777f9a092038d0a594a9fa04e215f929a22c329e2cb142 \ + --hash=sha256:91294a9fb94a75542f6e46e4a2ae709bd8d9b51134098cae5cf3bea5478b6d03 \ + --hash=sha256:943276cf269e0071948d9ff697159c1735e623c1151d88abb09b74659ef0cbea \ + --hash=sha256:96243987194634bd411066ce40c952e108f86af04db533ecd8ac3ff2a85b1885 \ + --hash=sha256:984012f71908165449a951de2050d52f276bfe3aa5d5f570f63ddad814370374 \ + --hash=sha256:9b03d7dc168353b4132965bde20feceabaa470e570c6f59660dfae59b1f9eeb3 \ + --hash=sha256:9dbb18c1cfb2f6517942fc9314437f66aa06d94436ffb1f06102ef3572f35276 \ + --hash=sha256:9ebf8d19b17bd0daeb7b7dec81a946a439b753942fd0210d6e96c532249eea6b \ + --hash=sha256:a525685c2f97da40762b8695eb7aa0af4c8344ca1905c73e4e29cb04d34607dc \ + --hash=sha256:abdbf6313b8d9efe157edeb7ab6eae4de064b1300ad31abf73755154b30abe68 \ + --hash=sha256:b69564772b5c8f22ea5f498dff08cfa825045b4d4c4400529000bdf818aa3b2a \ + --hash=sha256:b8ade5023067f99fe72b88accd30d0ea05a158e9e32a11f124e731ea9695313f \ + --hash=sha256:bbaefc84548d754be821bba7c4141c4787dda182f9e77f2f87b71213529efa7b \ + --hash=sha256:bd05de8c1698f8413dd7d869492693a0bf2211543b787ac78cd5e7536af1a6d7 \ + --hash=sha256:bf0b5e8e0f68ebb494356e577c06c139161efd8d3b9050f93b39b7c26cc54ff0 \ + --hash=sha256:c414be4ed9d3cac80c42e348fa5a956117d1a48227f48026e31f59cb4a7671eb \ + --hash=sha256:c47300f9bf791808f77d82747691c4bb09cb14bdf3060cca99b42cdc4361d5a7 \ + --hash=sha256:c4dc1c1781f2f716de763d1e9a7b34c6a894e167e291c7c5d16c72f7a9538545 \ + --hash=sha256:c804ae44fe7b4bab5da295e4f980a1ff04670bca9d23fe0a4e887e08ebd741a8 \ + --hash=sha256:cfac177ebd6236003846ea339981f71457cb6eb748f23381eb257e45092e3980 \ + --hash=sha256:d2ba24db8a9376921b5e87b4762b9adb0f3f1deaea68f2b8b0bb2c11efb9c3e7 \ + --hash=sha256:d3182ee2d887e507bd67319a0a61105d1dd33facc111329559a233b772c1a105 \ + --hash=sha256:d747252933c8a65ef6bd8da0fbb7ce28a90eb6119d8cd00772cd528aa07b68d5 \ + --hash=sha256:d7e369fd63331746182360977b1892bfc215476a30d61612d732425311639f56 \ + --hash=sha256:e12bbcd32897272fb05929110362ae9ff4c1b9bb26bd9e971e71dcd3275b4c3d \ + --hash=sha256:e7ad033e27a516a233bea839cdb77b80146facb3b4f40bf02cd0cac165cdd5c2 \ + --hash=sha256:e9e15b4a6c7dd6b85b5fbab29488a73f1f70de516942308daa266bf0e0aeb0d4 \ + --hash=sha256:ed53f7e89bb04f6d9e8e7799112360b0c4d5cbff067de0814c98c37c39b920f7 \ + --hash=sha256:eff8babca5a7999bc137acbc7482a8b7e17ffca5075ab41f5d770ab408c7bfef \ + --hash=sha256:f15e3e0b835a6d68b10c86bf80a3149780498d6911c93c3ffd1861d19f9200f1 \ + --hash=sha256:f3fcbc57b1791fa6cbe5d8434179d51de12be1a4811469529f47f6e7487a2571 \ + --hash=sha256:f4b653094e18f9031102d3a1da5c729c8f222d85225b18037dac621695e46e1a \ + --hash=sha256:f79203b3965b4000e91808aaa7c040206093f2b8bf86f455982f2274c9ccf442 \ + --hash=sha256:fd4dc129784e0c5335bd4e61dfcc4487499a013419e655cf2da1d091b7e0efdc + +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt index 046b05790..a95e7d5fc 100644 --- a/.github/scripts/requirements.txt +++ b/.github/scripts/requirements.txt @@ -1,16 +1,5 @@ -# Dependencies for the CI scripts in this directory. -# -# Installed with --require-hashes, so every distribution pip may choose has to be -# listed here by hash. That is why the transitive dependencies appear too, and why -# each entry carries several hashes: the pure-python wheel, the sdist, and for -# charset-normalizer the compiled wheels for the interpreters the runners ship. -# An unpinned install would be arbitrary third-party code running in the same job -# as the repository-server publish credential. -# -# Refreshing after a version bump: -# pip install pip-tools -# pip-compile --generate-hashes --output-file requirements.txt requirements.in -# or re-query the PyPI JSON API for the new release's digests. +# Hashed for --require-hashes (this job holds the publish credential). Direct deps: requests, semver; the rest is transitive. +# Refresh with: pip-compile --generate-hashes --output-file requirements.txt <(printf 'requests\nsemver\n') # --- direct --- diff --git a/.github/scripts/tests/conftest.py b/.github/scripts/tests/conftest.py deleted file mode 100644 index dbfacb9c7..000000000 --- a/.github/scripts/tests/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -import sys -from pathlib import Path - -TESTS = Path(__file__).resolve().parent - -# The scripts import each other as top-level modules, the way CI runs them: the CI scripts from -# .github/scripts, the build helpers from scripts/. The tests directory itself goes on the path too, -# so test modules share support.py whatever pytest's import mode. -for path in (TESTS.parent, TESTS.parents[2] / 'scripts', TESTS): - sys.path.insert(0, str(path)) diff --git a/.github/scripts/tests/support.py b/.github/scripts/tests/support.py index 964af7260..a3059548c 100644 --- a/.github/scripts/tests/support.py +++ b/.github/scripts/tests/support.py @@ -9,6 +9,11 @@ ROOT = Path(__file__).resolve().parents[3] +def blank(chip: str = 'ESP32-S3') -> bytes: + """A packed board descriptor with no board stamped in.""" + return espimage.Descriptor('', chip, -1, -1, -1, -1, 0, 0).pack() + + def make_image( descriptor: bytes | None, chip_id: int = 0x0009, hash_appended: bool = True, extra_segments: int = 1 ) -> bytes: diff --git a/.github/scripts/tests/test_build_cli.py b/.github/scripts/tests/test_build_cli.py index 396e43f13..053aaaa37 100644 --- a/.github/scripts/tests/test_build_cli.py +++ b/.github/scripts/tests/test_build_cli.py @@ -2,10 +2,10 @@ import importlib.util -import pytest - import gen_env_header +import pytest from support import ROOT + from utils import boards, github # Loaded from its path: a plain `import build` could resolve to PyPA's `build` package instead. diff --git a/.github/scripts/tests/test_build_scripts.py b/.github/scripts/tests/test_build_scripts.py index 27a5e6ce0..4ce9b5215 100644 --- a/.github/scripts/tests/test_build_scripts.py +++ b/.github/scripts/tests/test_build_scripts.py @@ -3,10 +3,10 @@ import os from pathlib import Path -import pytest - import gen_staticfs +import pytest from support import ROOT + from utils import partitions @@ -47,7 +47,7 @@ def test_staged_assets_are_byte_identical_across_runs(tmp_path): def stage(name): staging = tmp_path / name - gen_staticfs.stage_assets(str(build), str(staging)) + gen_staticfs.stage_assets(build, staging) return {p.relative_to(staging): p.read_bytes() for p in staging.rglob('*') if p.is_file()} first = stage('a') @@ -73,7 +73,7 @@ def test_frontend_build_is_current_tracks_source_changes(tmp_path): os.utime(source, (1000, 1000)) os.utime(index, (2000, 2000)) - assert gen_staticfs.frontend_build_is_current(str(frontend), str(frontend / 'build')) + assert gen_staticfs.frontend_build_is_current(frontend, frontend / 'build') os.utime(source, (3000, 3000)) - assert not gen_staticfs.frontend_build_is_current(str(frontend), str(frontend / 'build')) + assert not gen_staticfs.frontend_build_is_current(frontend, frontend / 'build') diff --git a/.github/scripts/tests/test_build_variants.py b/.github/scripts/tests/test_build_variants.py index 063307e9c..0263a31c7 100644 --- a/.github/scripts/tests/test_build_variants.py +++ b/.github/scripts/tests/test_build_variants.py @@ -1,8 +1,8 @@ """Tests for the build-variant grouping in scripts/utils/boards.py: which boards one compile can serve.""" import pytest - from support import write_fragment + from utils import boards BASE = 'CONFIG_IDF_TARGET="esp32s3"\nCONFIG_PARTITION_TABLE_CUSTOM=y\n' diff --git a/.github/scripts/tests/test_ci_build_gate.py b/.github/scripts/tests/test_ci_build_gate.py index f4be2c546..bf4e98d5d 100644 --- a/.github/scripts/tests/test_ci_build_gate.py +++ b/.github/scripts/tests/test_ci_build_gate.py @@ -1,8 +1,7 @@ import json -import pytest - import ci_build_gate +import pytest from ci_build_gate import CANARY_BOARD, FULL_BUILD_LABEL, decide SHA = 'c' * 40 @@ -109,7 +108,7 @@ def test_nightly_with_source_changes_builds(branch, publish): api = FakeApi(runs={'workflow_dispatch': run(BASE)}, files=['README.md', 'components/rf/src/Foo.cpp']) d = nightly(branch, api) assert outcome(d) == (True, publish, 'prod', '') - assert api.run_queries == [('schedule', branch), ('workflow_dispatch', branch)] + assert api.run_queries == [('workflow_dispatch', branch)] assert api.compares == [(BASE, SHA)] @@ -142,7 +141,7 @@ def test_nightly_with_unchanged_sha_skips(branch): @pytest.mark.parametrize('branch', ['develop', 'beta', 'master']) def test_nightly_with_only_non_source_changes_skips(branch): - api = FakeApi(runs={'schedule': run(BASE)}, files=NON_SOURCE_CHANGES) + api = FakeApi(runs={'workflow_dispatch': run(BASE)}, files=NON_SOURCE_CHANGES) d = nightly(branch, api) assert outcome(d) == (False, False, 'prod', '') @@ -157,7 +156,7 @@ def test_non_source_paths(name): [ '.changes/feat-rf.md', '.github/workflows/ci-build.yml', - '.github/scripts/get-vars.py', + '.github/scripts/get_vars.py', '.github/ISSUE_TEMPLATE', 'components/rf/src/Foo.cpp', 'docs', @@ -195,24 +194,6 @@ def test_first_ever_nightly_builds_without_comparing(): assert api.compares == [] -def test_nightly_compares_against_the_newer_of_schedule_and_dispatch(): - older, newer = 'a' * 40, 'd' * 40 - api = FakeApi( - runs={ - 'schedule': run(older, '2026-09-01T03:00:00Z'), - 'workflow_dispatch': run(newer, '2026-10-01T03:00:00Z'), - }, - files=['README.md'], - ) - nightly('develop', api) - assert api.compares == [(newer, SHA)] - - api.runs = {'schedule': run(newer, '2026-10-02T03:00:00Z'), 'workflow_dispatch': run(older, '2026-10-01T03:00:00Z')} - api.compares = [] - nightly('develop', api) - assert api.compares == [(newer, SHA)] - - def test_nightly_run_listing_failure_builds(capsys): api = FakeApi(runs_error=RuntimeError('HTTP 502'), files=['README.md']) d = nightly('develop', api) @@ -317,7 +298,9 @@ def test_main_writes_every_output(tmp_path, monkeypatch): monkeypatch.setenv('GITHUB_REPOSITORY', 'OpenShock/Firmware') monkeypatch.setenv('GITHUB_EVENT_PATH', str(event)) monkeypatch.setenv('GITHUB_OUTPUT', str(out)) + monkeypatch.setenv('GITHUB_STEP_SUMMARY', str(tmp_path / 'summary')) assert ci_build_gate.main() == 0 + assert f'boards={CANARY_BOARD}' in (tmp_path / 'summary').read_text(encoding='utf-8') assert out.read_text(encoding='utf-8').splitlines() == [ 'build=true', 'publish=false', diff --git a/.github/scripts/tests/test_get_vars.py b/.github/scripts/tests/test_get_vars.py index 7fd1379b8..1506685fd 100644 --- a/.github/scripts/tests/test_get_vars.py +++ b/.github/scripts/tests/test_get_vars.py @@ -1,14 +1,8 @@ -import importlib.util import json -from pathlib import Path +import get_vars import pytest -# get-vars.py has a hyphen in its name, so it can't be imported with a plain import statement. -_spec = importlib.util.spec_from_file_location('get_vars', Path(__file__).resolve().parent.parent / 'get-vars.py') -get_vars = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(get_vars) - @pytest.mark.parametrize( 'tag, expected', @@ -28,21 +22,19 @@ def test_parse_version_rejects(tag): @pytest.mark.parametrize( - 'tag, stable, beta, dev', + 'tag, channel', [ - ('1.5.0', True, False, False), - ('1.5.0-rc.1', False, True, False), - ('1.5.0-beta.3', False, True, False), - ('1.5.0-develop.4', False, False, True), - ('1.5.0-dev.4', False, False, True), - ('1.5.0-feature.1', False, False, False), + ('1.5.0', 'stable'), + ('1.5.0-rc.1', 'beta'), + ('1.5.0-beta.3', 'beta'), + ('1.5.0-develop.4', 'develop'), + ('1.5.0-dev.4', 'develop'), + ('1.5.0-feature.1', None), ], ) -def test_channel_of_version(tag, stable, beta, dev): +def test_channel_of_version(tag, channel): version = get_vars.parse_version(tag) - assert get_vars.is_stable(version) is stable - assert get_vars.is_beta(version) is beta - assert get_vars.is_dev(version) is dev + assert get_vars.CHANNEL_BY_PRERELEASE.get(get_vars.prerelease_id(version)) == channel @pytest.mark.parametrize( @@ -61,7 +53,7 @@ def test_sanitize(name, expected): def run_main(monkeypatch): """Runs get-vars main() for a ref and board selection, git's tag list stubbed out; returns its outputs.""" monkeypatch.setattr(get_vars.subprocess, 'check_output', lambda *a, **kw: '1.4.0\n1.3.0\n') - for name in ('GITHUB_HEAD_REF', 'RELEASE_NEXT_VERSION', 'RELEASE_SKIP', 'CHANNEL_INPUT'): + for name in ('GITHUB_HEAD_REF', 'RELEASE_NEXT_VERSION', 'RELEASE_SKIP', 'CHANNEL_INPUT', 'NIGHTLY_INPUT'): monkeypatch.delenv(name, raising=False) def run(ref, boards_input=''): @@ -103,3 +95,14 @@ def test_main_on_a_tag_releases_the_tag(run_main, tag, channel): def test_main_on_a_feature_branch_publishes_to_develop(run_main): outputs, _ = run_main('refs/heads/feature/thing') assert outputs['release-channel'] == 'develop' + + +@pytest.mark.parametrize( + 'channel, nightly, expected', + [('beta', 'false', 'beta'), ('derive', 'false', 'develop'), ('', '', 'develop'), ('beta', 'true', 'develop')], +) +def test_main_channel_input(run_main, monkeypatch, channel, nightly, expected): + monkeypatch.setenv('CHANNEL_INPUT', channel) + monkeypatch.setenv('NIGHTLY_INPUT', nightly) + outputs, _ = run_main('refs/heads/develop') + assert outputs['release-channel'] == expected diff --git a/.github/scripts/tests/test_merge_image.py b/.github/scripts/tests/test_merge_image.py index 4651a997b..021351eb2 100644 --- a/.github/scripts/tests/test_merge_image.py +++ b/.github/scripts/tests/test_merge_image.py @@ -1,8 +1,7 @@ """Tests for the command line of scripts/merge_image.py; esptool itself is never run.""" -import pytest - import merge_image +import pytest ARGS = ['--board', 'OpenShock-Core-V2', '--staticfs', 'staticfs.bin'] diff --git a/.github/scripts/tests/test_package_boards.py b/.github/scripts/tests/test_package_boards.py index f711209e5..1deb50d0f 100644 --- a/.github/scripts/tests/test_package_boards.py +++ b/.github/scripts/tests/test_package_boards.py @@ -3,24 +3,30 @@ import shutil import package_boards -from support import make_image +import pytest +from support import blank, make_image + from utils import boards, espimage, stamping +@pytest.fixture +def dirs(tmp_path): + return tmp_path / 'variants', tmp_path / 'out' + + def write_variant(variants, board): """The build directory of this board's variant, holding an unstamped app.""" vid = boards.variant_id(boards.board_fragment(board)) src = variants / vid src.mkdir(parents=True) - descriptor = espimage.Descriptor('', 'ESP32-S3', -1, -1, -1, -1, 0, 0).pack() - (src / 'app.bin').write_bytes(make_image(descriptor, chip_id=espimage.CHIP_IDS['esp32s3'])) + (src / 'app.bin').write_bytes(make_image(blank(), chip_id=espimage.CHIP_IDS['esp32s3'])) (src / 'bootloader.bin').write_bytes(b'boot') (src / 'partition-table.bin').write_bytes(b'table') return vid -def test_packages_every_board_of_a_variant(tmp_path): - variants, out = tmp_path / 'variants', tmp_path / 'out' +def test_packages_every_board_of_a_variant(dirs): + variants, out = dirs vid = write_variant(variants, 'OpenShock-Core-V2') assert package_boards.package(['OpenShock-Core-V2', 'Wemos-Lolin-S3-Mini'], variants, out) == [] @@ -34,8 +40,8 @@ def test_packages_every_board_of_a_variant(tmp_path): assert stamping.read_descriptor(espimage.load(variants / vid / 'app.bin'), 'app').board == '' -def test_a_missing_variant_fails_only_its_boards(tmp_path, capsys): - variants, out = tmp_path / 'variants', tmp_path / 'out' +def test_a_missing_variant_fails_only_its_boards(dirs, capsys): + variants, out = dirs write_variant(variants, 'OpenShock-Core-V2') # Core-V1 is on the UART console, a different variant from Core-V2's. failed = package_boards.package(['OpenShock-Core-V1', 'OpenShock-Core-V2'], variants, out) @@ -44,8 +50,8 @@ def test_a_missing_variant_fails_only_its_boards(tmp_path, capsys): assert (out / 'OpenShock-Core-V2' / 'app.bin').is_file() -def test_a_variant_missing_an_image_fails_only_its_boards(tmp_path, capsys): - variants, out = tmp_path / 'variants', tmp_path / 'out' +def test_a_variant_missing_an_image_fails_only_its_boards(dirs, capsys): + variants, out = dirs (variants / write_variant(variants, 'OpenShock-Core-V2') / 'bootloader.bin').unlink() write_variant(variants, 'OpenShock-Core-V1') failed = package_boards.package(['OpenShock-Core-V2', 'OpenShock-Core-V1'], variants, out) @@ -54,9 +60,9 @@ def test_a_variant_missing_an_image_fails_only_its_boards(tmp_path, capsys): assert (out / 'OpenShock-Core-V1' / 'app.bin').is_file() -def test_a_corrupt_stamp_is_caught_before_it_ships(tmp_path, capsys, monkeypatch): +def test_a_corrupt_stamp_is_caught_before_it_ships(dirs, capsys, monkeypatch): """The written image is verified, so a patch that broke the image fails its board instead of shipping.""" - variants, out = tmp_path / 'variants', tmp_path / 'out' + variants, out = dirs write_variant(variants, 'OpenShock-Core-V2') real = stamping.stamp_image @@ -75,9 +81,9 @@ def corrupt_core_v2(image, spec, path): assert (out / 'Wemos-Lolin-S3-Mini' / 'app.bin').is_file() -def test_a_wrong_stamp_is_caught_before_it_ships(tmp_path, capsys, monkeypatch): +def test_a_wrong_stamp_is_caught_before_it_ships(dirs, capsys, monkeypatch): """An image that carries another board's descriptor after stamping fails its board.""" - variants, out = tmp_path / 'variants', tmp_path / 'out' + variants, out = dirs write_variant(variants, 'OpenShock-Core-V2') other = boards.read_board('Wemos-Lolin-S3-Mini') real = stamping.stamp_image @@ -88,13 +94,13 @@ def test_a_wrong_stamp_is_caught_before_it_ships(tmp_path, capsys, monkeypatch): assert (out / 'Wemos-Lolin-S3-Mini' / 'app.bin').is_file() -def test_a_board_name_too_long_for_the_descriptor_fails_only_that_board(tmp_path, capsys, monkeypatch): +def test_a_board_name_too_long_for_the_descriptor_fails_only_that_board(tmp_path, dirs, capsys, monkeypatch): root = tmp_path / 'repo' shutil.copytree(boards.ROOT / 'boards', root / 'boards') long_name = 'L' * (boards.NAME_MAX_LEN + 1) shutil.copyfile(root / 'boards' / 'OpenShock-Core-V2.defaults', root / 'boards' / f'{long_name}.defaults') monkeypatch.setattr(boards, 'ROOT', root) - variants, out = tmp_path / 'variants', tmp_path / 'out' + variants, out = dirs write_variant(variants, 'OpenShock-Core-V2') failed = package_boards.package([long_name, 'OpenShock-Core-V2'], variants, out) diff --git a/.github/scripts/tests/test_repo_server_policy.py b/.github/scripts/tests/test_repo_server_policy.py index f93487319..a87fec2dc 100644 --- a/.github/scripts/tests/test_repo_server_policy.py +++ b/.github/scripts/tests/test_repo_server_policy.py @@ -1,5 +1,4 @@ import pytest - import repo_server_policy ENV = { diff --git a/.github/scripts/tests/test_repo_server_publish.py b/.github/scripts/tests/test_repo_server_publish.py index d2c915334..55a933f8e 100644 --- a/.github/scripts/tests/test_repo_server_publish.py +++ b/.github/scripts/tests/test_repo_server_publish.py @@ -1,7 +1,6 @@ import pytest -import requests - import repo_server_publish +import requests class FakeResponse: @@ -40,29 +39,26 @@ def upload_env(monkeypatch): def test_successful_publish_does_not_discard(): session = FakeSession(publish=201) - assert repo_server_publish.do_upload(session, 'https://repo', 'publish') == 0 + assert repo_server_publish.do_upload(session, 'https://repo') == 0 assert session.deleted == [] def test_rejected_publish_discards_the_release(): session = FakeSession(publish=500) with pytest.raises(SystemExit): - repo_server_publish.do_upload(session, 'https://repo', 'publish') + repo_server_publish.do_upload(session, 'https://repo') assert len(session.deleted) == 1 -def test_publish_without_a_response_is_not_discarded(): +def test_publish_without_a_response_is_not_discarded(tmp_path, monkeypatch): # The release may already be live, so deleting it would be worse than leaving it for a human to check. + output = tmp_path / 'output' + monkeypatch.setenv('GITHUB_OUTPUT', str(output)) session = FakeSession(publish=requests.ConnectionError('timed out')) with pytest.raises(SystemExit): - repo_server_publish.do_upload(session, 'https://repo', 'publish') + repo_server_publish.do_upload(session, 'https://repo') assert session.deleted == [] - - -def test_dry_run_always_discards(): - session = FakeSession(publish=201) - assert repo_server_publish.do_upload(session, 'https://repo', 'dry-run') == 0 - assert len(session.deleted) == 1 + assert 'outcome-unknown=true' in output.read_text() # --- artifact layout --- @@ -81,7 +77,7 @@ def write_board_artifacts(artifacts, board, version='1.5.0'): def test_resolve_artifacts_reads_the_package_layout(tmp_path): write_board_artifacts(tmp_path, 'BoardA') - write_board_artifacts(tmp_path, 'BoardA-Mini') + write_board_artifacts(tmp_path, 'BoardA_Mini') found = repo_server_publish.resolve_artifacts(tmp_path, 'BoardA') @@ -97,3 +93,19 @@ def test_resolve_artifacts_fails_on_a_missing_image(tmp_path): (tmp_path / 'firmware_boards' / 'BoardA' / 'bootloader.bin').unlink() with pytest.raises(SystemExit): repo_server_publish.resolve_artifacts(tmp_path, 'BoardA') + + +@pytest.mark.parametrize('status, expected', [(204, 'discarded'), (404, 'already_closed'), (409, 'already_closed'), (500, 'failed')]) +def test_discard_reports_what_happened(status, expected): + session = FakeSession(publish=201) + session.delete = lambda url, **kwargs: FakeResponse(status) + assert repo_server_publish.discard(session, 'https://repo', 'rel-1') == expected + + +def test_discard_without_a_response_fails_quietly(): + def unreachable(url, **kwargs): + raise requests.ConnectionError('down') + + session = FakeSession(publish=201) + session.delete = unreachable + assert repo_server_publish.discard(session, 'https://repo', 'rel-1') == 'failed' diff --git a/.github/scripts/tests/test_repo_server_token.py b/.github/scripts/tests/test_repo_server_token.py new file mode 100644 index 000000000..6cc1365d9 --- /dev/null +++ b/.github/scripts/tests/test_repo_server_token.py @@ -0,0 +1,20 @@ +import pytest +import repo_server_token + + +def test_audience_is_passed_as_a_query_parameter(monkeypatch): + calls = [] + + class Response: + status_code = 500 + text = '' + + monkeypatch.setenv('AUDIENCE', 'a&b c') + monkeypatch.setenv('ACTIONS_ID_TOKEN_REQUEST_URL', 'https://oidc/token?api=1') + monkeypatch.setattr(repo_server_token.requests, 'get', lambda url, **kwargs: calls.append((url, kwargs)) or Response()) + + with pytest.raises(SystemExit): + repo_server_token.main() + + assert calls[0][0] == 'https://oidc/token?api=1' + assert calls[0][1]['params'] == {'audience': 'a&b c'} diff --git a/.github/scripts/tests/test_stamp_board.py b/.github/scripts/tests/test_stamp_board.py index fbbdf6be8..1eec6358a 100644 --- a/.github/scripts/tests/test_stamp_board.py +++ b/.github/scripts/tests/test_stamp_board.py @@ -1,11 +1,12 @@ """Tests for scripts/stamp_board.py and the board/image helpers under scripts/utils.""" -import pytest +import struct +import pytest import stamp_board -from support import make_image, write_fragment -from utils import boards, espimage +from support import blank, make_image, write_fragment +from utils import boards, espimage S3_FRAGMENT = ( 'CONFIG_IDF_TARGET="esp32s3"\n' @@ -143,10 +144,6 @@ def cli(*argv): return stamp_board.main([str(a) for a in argv]) -def blank(chip='ESP32-S3'): - return espimage.Descriptor('', chip, -1, -1, -1, -1, 0, 0).pack() - - def test_stamp_verify_and_read(tmp_path, capsys, test_board): image = tmp_path / 'firmware.bin' image.write_bytes(make_image(blank())) @@ -181,3 +178,7 @@ def test_stamp_refuses_the_wrong_chip(tmp_path, test_board): image.write_bytes(make_image(blank('ESP32'), chip_id=0x0000)) with pytest.raises(SystemExit, match='chip id'): cli('stamp', '--image', image, '--board', test_board) + + +def test_descriptor_format_fills_the_descriptor(): + assert struct.calcsize(espimage.DESC_FORMAT) == espimage.DESC_LEN diff --git a/.github/workflows/check-changes.yml b/.github/workflows/check-changes.yml index bdf432d00..78762e5cb 100644 --- a/.github/workflows/check-changes.yml +++ b/.github/workflows/check-changes.yml @@ -1,10 +1,10 @@ +name: check-changes + on: pull_request: branches: [develop, beta, master] types: [opened, reopened, synchronize, ready_for_review, labeled, unlabeled] -name: check-changes - permissions: contents: read pull-requests: write diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index fef7234b4..b47f428e1 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -4,6 +4,8 @@ # - pull requests: one canary board, or every board with the ci:full-build label; # - dispatch: every board, on demand. # Per-commit checks are host-tests.yml, script-tests.yml, frontend-checks.yml and cpp-linter.yml. +name: ci-build + on: push: tags: @@ -36,8 +38,6 @@ on: type: boolean default: false -name: ci-build - # Per ref. A label other than ci:full-build gets its own group, so its no-op run does not cancel the PR's build. concurrency: group: >- @@ -49,11 +49,14 @@ concurrency: permissions: contents: read -# Python jobs use ubuntu-latest, which ships the pinned interpreter; ubuntu-slim downloads it every time. +defaults: + run: + shell: bash + jobs: # Decides build, publish, the server and the boards (.github/scripts/ci_build_gate.py). gate: - runs-on: ubuntu-latest + runs-on: ubuntu-latest # ships the pinned Python; ubuntu-slim would download it every run timeout-minutes: 3 permissions: contents: read @@ -68,15 +71,7 @@ jobs: with: sparse-checkout: .github - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version-file: .github/scripts/.python-version - cache: 'pip' - cache-dependency-path: .github/scripts/requirements.txt - - - name: Install script dependencies - shell: bash - run: pip install --require-hashes -r .github/scripts/requirements.txt + - uses: ./.github/actions/setup-ci-scripts - name: Decide build and publish id: decide @@ -84,7 +79,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} run: python .github/scripts/ci_build_gate.py - # Version, channel, boards, and the build variants they need (.github/scripts/get-vars.py). + # Version, channel, boards, and the build variants they need (.github/scripts/get_vars.py). plan: needs: [gate] if: ${{ needs.gate.outputs.build == 'true' }} @@ -106,15 +101,7 @@ jobs: boards scripts/utils - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version-file: .github/scripts/.python-version - cache: 'pip' - cache-dependency-path: .github/scripts/requirements.txt - - - name: Install script dependencies - shell: bash - run: pip install --require-hashes -r .github/scripts/requirements.txt + - uses: ./.github/actions/setup-ci-scripts # The next version from .changes/; status mode works on every branch and creates no tag. - name: Compute next version @@ -125,14 +112,13 @@ jobs: - name: Get variables id: get-vars - shell: bash env: RELEASE_NEXT_VERSION: ${{ steps.relmeta.outputs.next-version }} RELEASE_SKIP: ${{ steps.relmeta.outputs.skip }} - # Manual dispatch only; otherwise get-vars derives the channel from the ref. - CHANNEL_INPUT: ${{ !inputs.nightly && inputs.channel != 'derive' && inputs.channel || '' }} + CHANNEL_INPUT: ${{ inputs.channel }} + NIGHTLY_INPUT: ${{ inputs.nightly }} BOARDS_INPUT: ${{ needs.gate.outputs.boards }} - run: python .github/scripts/get-vars.py + run: python .github/scripts/get_vars.py # Opens and reserves the release before anything is built, so the server's checks fail in a minute, not after the matrix. stage: @@ -173,16 +159,7 @@ jobs: .github CHANGELOG.md - # The policy runs before repo-server-token, which would otherwise install these. - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version-file: .github/scripts/.python-version - cache: 'pip' - cache-dependency-path: .github/scripts/requirements.txt - - - name: Install script dependencies - shell: bash - run: pip install --require-hashes -r .github/scripts/requirements.txt + - uses: ./.github/actions/setup-ci-scripts # This repository's prod rules, which the server cannot know (stable descends from master, a draft is staged). - name: Apply repository publishing policy @@ -209,13 +186,14 @@ jobs: mode: init # The frontend packed into the static0 image, once for every board (its checks are frontend-checks.yml's). - # Waits on stage so a prod release is approved before compute is spent; a skipped stage still lets it run. + # The jobs below wait on stage so a prod release is approved before compute is spent: a skipped stage + # (nothing to publish) lets them run, a failed one stops them. staticfs: needs: [gate, stage] if: >- ${{ !cancelled() - && needs.gate.outputs.build == 'true' - && needs.stage.result != 'failure' }} + && needs.stage.result != 'failure' + && needs.gate.outputs.build == 'true' }} runs-on: ubuntu-latest timeout-minutes: 10 @@ -240,11 +218,10 @@ jobs: name: 'build (${{ matrix.chip }}: ${{ matrix.boards }})' runs-on: ubuntu-latest needs: [plan, stage] - # !cancelled() so a skipped stage does not skip the build; plan named so a failed get-vars does. if: >- ${{ !cancelled() - && needs.plan.result == 'success' - && needs.stage.result != 'failure' }} + && needs.stage.result != 'failure' + && needs.plan.result == 'success' }} timeout-minutes: 30 strategy: fail-fast: false @@ -265,13 +242,18 @@ jobs: # Stamp every board from its variant and merge it with static0. One job: it is seconds of work per board. package: runs-on: ubuntu-latest - needs: [plan, staticfs, build] - # !cancelled() so a skipped stage upstream does not skip this; a successful build implies plan. + needs: [plan, stage, staticfs, build] if: >- ${{ !cancelled() + && needs.stage.result != 'failure' && needs.staticfs.result == 'success' && needs.build.result == 'success' }} timeout-minutes: 10 + permissions: + contents: read + id-token: write + attestations: write + artifact-metadata: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -287,6 +269,12 @@ jobs: version: ${{ needs.plan.outputs.version }} boards: ${{ needs.plan.outputs.board-list }} + - name: Attest build provenance + if: ${{ startsWith(github.ref, 'refs/tags/') }} + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: merged/OpenShock_*.bin + # Upload every board into the staged release and publish it; fail on the first non-success. # Publishing consumes the version (only a maintainer can undo it). No environment: stage was the approval. # A plain if is safe: on a publish run nothing upstream is skipped, and a failure upstream must skip this. @@ -299,6 +287,8 @@ jobs: permissions: contents: read id-token: write + outputs: + outcome-unknown: ${{ steps.publish.outputs.outcome-unknown }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -309,6 +299,7 @@ jobs: id: token - uses: ./.github/actions/repo-server-publish + id: publish with: server-url: ${{ needs.stage.outputs.server-url }} token: ${{ steps.token.outputs.token }} @@ -322,7 +313,7 @@ jobs: runs-on: ubuntu-latest needs: [stage, repo-server] timeout-minutes: 10 - if: ${{ always() && needs.stage.outputs.release-id != '' && needs.repo-server.result != 'success' }} + if: ${{ always() && needs.stage.outputs.release-id != '' && needs.repo-server.result != 'success' && needs.repo-server.outputs.outcome-unknown != 'true' }} permissions: contents: read id-token: write @@ -354,6 +345,10 @@ jobs: && startsWith(github.ref, 'refs/tags/') permissions: contents: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + TAG: ${{ needs.plan.outputs.version }} steps: - name: Download release artifacts @@ -361,14 +356,7 @@ jobs: with: name: firmware_merged - - name: Display artifacts - run: ls -R - - name: Upload binaries to staged release - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - TAG: ${{ needs.plan.outputs.version }} run: | shopt -s globstar nullglob files=(**/OpenShock_*.bin) @@ -379,10 +367,6 @@ jobs: gh release upload "$TAG" "${files[@]}" --clobber - name: Publish the release - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - TAG: ${{ needs.plan.outputs.version }} run: gh release edit "$TAG" --draft=false # Announce to the OpenShock API last, so it never points a device at a version the server is not yet serving. @@ -396,9 +380,19 @@ jobs: id-token: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: .github + + - uses: ./.github/actions/repo-server-token + id: token + with: + audience: openshock-release-ingest + # release.json is the release asset release.yml staged. - name: Announce release to OpenShock API env: + OIDC_TOKEN: ${{ steps.token.outputs.token }} GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} API_URL: ${{ vars.OPENSHOCK_API_URL }} @@ -406,14 +400,6 @@ jobs: run: | set -euo pipefail gh release download "$TAG" --pattern release.json --dir . - OIDC_TOKEN=$(curl -sLS \ - -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ - "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=openshock-release-ingest" \ - | jq -r .value) - if [ -z "$OIDC_TOKEN" ] || [ "$OIDC_TOKEN" = "null" ]; then - echo "::error::Failed to fetch GitHub OIDC token" - exit 1 - fi curl -fsSL --retry 3 --retry-delay 5 -X POST "$API_URL/1/public/releases" \ -H "Authorization: Bearer $OIDC_TOKEN" \ -H "Content-Type: application/json" \ @@ -427,13 +413,8 @@ jobs: if: ${{ always() }} timeout-minutes: 1 steps: - - name: Check every job's result - env: - FAILED: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} - RESULTS: ${{ toJSON(needs.*.result) }} + - name: Fail when a job failed or was cancelled + if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} run: | - echo "job results: $RESULTS" - if [ "$FAILED" = "true" ]; then - echo "::error::A ci-build job failed or was cancelled." - exit 1 - fi + echo "::error::A ci-build job failed or was cancelled." + exit 1 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0c34a472e..4006c3aa1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,9 +13,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +defaults: + run: + shell: bash + jobs: - analyze-js-py: - name: Analyze JS/PY + analyze: + name: Analyze ${{ matrix.language }} runs-on: 'ubuntu-latest' timeout-minutes: 10 permissions: @@ -26,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - language: ['javascript-typescript', 'python'] + language: ['javascript-typescript', 'python', 'actions'] steps: - name: Checkout repository @@ -36,15 +40,15 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@416ff0dea110f80f0f56f0046500dbf7420e4bb0 # codeql-bundle-v2.27.1 + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: languages: ${{ matrix.language }} - # No build step: JavaScript/TypeScript and Python are extracted from source. + # No build step: JavaScript/TypeScript, Python and workflow YAML are extracted from source. - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@416ff0dea110f80f0f56f0046500dbf7420e4bb0 # codeql-bundle-v2.27.1 + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: - category: '/language:${{matrix.language}}' + category: '/language:${{ matrix.language }}' # Weekly or on demand only: the C/C++ analysis has to compile the firmware. analyze-cpp: @@ -57,9 +61,6 @@ jobs: contents: read security-events: write - env: - language: 'c-cpp' - steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -69,17 +70,17 @@ jobs: run: git submodule update --init --depth 1 components/flatbuffers/flatbuffers - name: Initialize CodeQL - uses: github/codeql-action/init@416ff0dea110f80f0f56f0046500dbf7420e4bb0 # codeql-bundle-v2.27.1 + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: - languages: ${{ env.language }} + languages: c-cpp # build, not configure: the tracer must see the compiler run. The version is a placeholder. - uses: ./.github/actions/build-compilationdb with: version: 0.0.0-codeql+build - mode: build + target: app - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@416ff0dea110f80f0f56f0046500dbf7420e4bb0 # codeql-bundle-v2.27.1 + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: - category: '/language:${{ env.language }}' + category: '/language:c-cpp' diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index e9b183b10..016a31bb8 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -15,6 +15,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +defaults: + run: + shell: bash + jobs: cpp-linter: name: C/C++ Linter @@ -24,13 +28,12 @@ jobs: permissions: contents: read pull-requests: write - + steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # No ESP-IDF and no compile database: with clang-tidy off (below) the linter only - # runs clang-format, which needs neither. Vendored submodules are ignored anyway. + # clang-format only, so no ESP-IDF or compile database is needed. - uses: cpp-linter/cpp-linter-action@8120e05c7fbd25fa09fa93424ec49f55f7b304ab # v2.23.1 id: linter env: @@ -38,28 +41,15 @@ jobs: with: style: file version: 21 - # clang-tidy is disabled: the native ESP-IDF compile DB is generated by - # xtensa-esp32s3-elf-gcc, whose flags (-mlongcalls, -fstrict-volatile-bitfields, - # ...) LLVM clang-tidy rejects, and standalone headers have no compile command - # so their ESP-IDF includes don't resolve - both surface as spurious - # clang-diagnostic-errors. clang-format still gates style, and CodeQL provides - # deep static analysis. Re-enabling it means restoring the - # build-compilationdb step and `database:` once the DB is clang-toolchain compatible. + # clang-tidy is off: it rejects the gcc-only flags of the ESP-IDF compile DB. CodeQL covers static analysis. tidy-checks: '-*' - # Walk the source tree instead of the PR diff. GitHub returns 406 for very - # large diffs (the Arduino migration PR is ~27k lines), which drops cpp-linter - # into a paginated fallback that crashes on files GitHub omits a patch for. - # Scanning the tree sidesteps the diff-size limit entirely. + # Scan the tree, not the PR diff: GitHub returns 406 for very large diffs. files-changed-only: false ignore: '.github|build|managed_components|components/littlefs|components/flatbuffers|components/serialization/include/serialization/_fbs' - # Commit comments (push events) need contents:write, which this job - # deliberately does not grant; only post thread comments on PRs, where - # pull-requests:write applies. Otherwise the action 403s and fails the - # job even when the lint is clean. + # Commit comments on push need contents:write, which this job does not grant. thread-comments: ${{ github.event_name == 'pull_request' && 'update' || 'false' }} - no-lgtm: false - - name: Fail fast?! + - name: Fail on lint errors if: steps.linter.outputs.checks-failed > 0 run: | echo "::error::Some files failed the linting checks!" diff --git a/.github/workflows/frontend-checks.yml b/.github/workflows/frontend-checks.yml index 46dcb36b1..727add016 100644 --- a/.github/workflows/frontend-checks.yml +++ b/.github/workflows/frontend-checks.yml @@ -1,4 +1,6 @@ # Per-commit captive-portal check: svelte-check, unit tests and a production build. +name: frontend-checks + on: push: branches: ['**'] @@ -15,8 +17,6 @@ on: - '.github/workflows/frontend-checks.yml' workflow_dispatch: -name: frontend-checks - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/host-tests.yml b/.github/workflows/host-tests.yml index dbf028fe5..72bbd62d8 100644 --- a/.github/workflows/host-tests.yml +++ b/.github/workflows/host-tests.yml @@ -1,4 +1,6 @@ # Per-commit firmware check: build and run the host (linux target) Unity suites. +name: host-tests + on: push: branches: ['**'] @@ -28,8 +30,6 @@ on: - '.github/workflows/host-tests.yml' workflow_dispatch: -name: host-tests - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -37,6 +37,10 @@ concurrency: permissions: contents: read +defaults: + run: + shell: bash + jobs: host-tests: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork }} @@ -56,15 +60,17 @@ jobs: # On develop this also refills the ESP-IDF cache after a key change (it runs on setup-esp-idf edits). - uses: ./.github/actions/setup-esp-idf + # The toolchains' inner /bin dirs hold an unprefixed `as` that would shadow the host's one. + - name: Drop toolchain assemblers from PATH + run: | + set -euo pipefail + path="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vE -- '-elf/[^/]*-elf/bin/?$' | paste -sd: -)" + echo "PATH=$path" >> "$GITHUB_ENV" + # Each components//host_test is its own linux-target project; the ELF exits with the Unity failure count. - name: Build and run host test suites - shell: bash run: | set -euo pipefail - # Drop the toolchains' inner /bin dirs: their unprefixed `as` would shadow the host's - # for the linux target ("as: unrecognized option '--64'"). - PATH="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vE -- '-elf/[^/]*-elf/bin/?$' | paste -sd: -)" - export PATH echo "as: $(command -v as)" failed=() for dir in components/*/host_test; do diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3484c933b..e00c60734 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,13 +3,13 @@ # on that branch, so this dispatches ci-build once per branch instead of building here. ci-build's # gate skips a branch whose firmware sources have not changed since its last build, and decides # what each nightly publishes (develop to prod; beta and master build only). +name: nightly + on: schedule: - cron: '0 3 * * *' workflow_dispatch: -name: nightly - permissions: contents: read diff --git a/.github/workflows/pinned-cert-audit.yml b/.github/workflows/pinned-cert-audit.yml index a16e6707f..3746bd920 100644 --- a/.github/workflows/pinned-cert-audit.yml +++ b/.github/workflows/pinned-cert-audit.yml @@ -20,6 +20,10 @@ permissions: contents: write # push the retire-pin branch pull-requests: write # open the retire-pin PR +defaults: + run: + shell: bash + jobs: audit: name: Audit pinned certs @@ -30,17 +34,34 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Install cryptography - # Same pin as requirements.txt: this job holds write permissions, so don't pull whatever is latest - run: python3 -m pip install --user cryptography==50.0.2 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version-file: .python-version + + - name: Install Python dependencies + # Hashed: this job holds write permissions, so don't pull whatever is latest + run: pip install --require-hashes -r requirements.txt # Probes the live endpoints + checks expiry. Never fails the step itself (so the # follow-up steps always see its outputs); the expiry alert is raised at the end. - name: Audit id: audit continue-on-error: true - working-directory: certificates - run: python3 cert_bundle.py audit + run: python certificates/cert_bundle.py audit + + - name: Summarize audit + if: always() + env: + OUTCOME: ${{ steps.audit.outcome }} + ALERT: ${{ steps.audit.outputs.alert }} + REMOVABLE: ${{ steps.audit.outputs.removable }} + run: | + { + echo '### Pinned certificate audit' + echo "- Outcome: \`$OUTCOME\`" + echo "- Expiry alert: \`${ALERT:-false}\`" + echo "- Removable pins: \`${REMOVABLE:-none}\`" + } >> "$GITHUB_STEP_SUMMARY" # A pinned root that no monitored endpoint chains through anymore: delete it, # regenerate cacert-merged.pem, and open a PR. Runs only when the audit positively found @@ -49,28 +70,29 @@ jobs: if: steps.audit.outputs.removable != '' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REMOVABLE: ${{ steps.audit.outputs.removable }} + REF_NAME: ${{ github.ref_name }} + RUN_ID: ${{ github.run_id }} run: | set -euo pipefail - branch="chore/retire-pinned-certs-${{ github.run_id }}" + branch="chore/retire-pinned-certs-$RUN_ID" git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git checkout -b "$branch" - for f in ${{ steps.audit.outputs.removable }}; do + for f in $REMOVABLE; do echo "Retiring certificates/$f" git rm "certificates/$f" done # Regenerate the base + packed bundle (verified curl base + remaining pinned certs). - ( cd certificates && python3 cert_bundle.py generate ) + python certificates/cert_bundle.py generate git add certificates/cacert-curl.pem certificates/cacert-merged.pem git commit -m "chore(certs): retire pinned root(s) no longer needed by live endpoints" git push origin "$branch" gh pr create \ - --base "${{ github.ref_name }}" \ + --base "$REF_NAME" \ --head "$branch" \ --title "chore(certs): retire unneeded pinned root(s)" \ - --body "Automated by \`pinned-cert-audit\`: \`cert_bundle.py audit\` found pinned root(s) that no monitored endpoint (api.openshock.app / api.openshock.dev) chains through anymore. Deleted them from \`certificates/pinned_certs/\` and regenerated \`certificates/cacert-merged.pem\`. - - Merge only after confirming the endpoints still verify against the base trust store (i.e. the server chain was intentionally trimmed)." + --body "Automated by \`pinned-cert-audit\`: \`cert_bundle.py audit\` found pinned root(s) that no monitored endpoint (api.openshock.app / api.openshock.dev) chains through anymore. Deleted them from \`certificates/pinned_certs/\` and regenerated \`certificates/cacert-merged.pem\`. Merge only after confirming the endpoints still verify against the base trust store (i.e. the server chain was intentionally trimmed)." # Fail last so the retire-PR step above still runs. A still-needed pinned root that # is expired / nearing expiry means we must rotate it. @@ -79,3 +101,10 @@ jobs: run: | echo "::error::A still-needed pinned certificate is expired or nearing expiry; rotate it in certificates/pinned_certs/ and regenerate cacert-merged.pem." exit 1 + + # Exit 1 with an alert is handled above; anything else (exit 2, a crash) means the audit never ran. + - name: Fail on audit error + if: steps.audit.outcome == 'failure' && steps.audit.outputs.alert != 'true' + run: | + echo "::error::The certificate audit crashed; see the Audit step." + exit 1 diff --git a/.github/workflows/pr-check-comment.yml b/.github/workflows/pr-check-comment.yml index 81ebd9f20..bc5233862 100644 --- a/.github/workflows/pr-check-comment.yml +++ b/.github/workflows/pr-check-comment.yml @@ -1,10 +1,10 @@ +name: pr-check-comment + on: workflow_run: workflows: [check-changes] types: [completed] -name: pr-check-comment - permissions: pull-requests: write @@ -12,6 +12,7 @@ jobs: comment: runs-on: ubuntu-slim # lightweight: jq + gh api sticky comment if: github.event.workflow_run.event == 'pull_request' + timeout-minutes: 5 steps: - name: Check PR comment uses: OpenShock/actions/pr-check-comment@21ca2d511bca92cf24fb502fa302f82700822584 # v1.1.1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e4bfbac3..84291092f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,11 +1,11 @@ +name: release + on: # Releases are cut manually, never on push. Run this workflow from the beta # branch to cut an rc, or from master for a stable release. A plain push to # beta/master (e.g. merging CI changes) must not create a release. workflow_dispatch: -name: release - concurrency: group: release-${{ github.ref }} cancel-in-progress: false @@ -13,20 +13,19 @@ concurrency: permissions: contents: write +defaults: + run: + shell: bash + jobs: auto-release: runs-on: ubuntu-slim # lightweight: release-tool + git/gh, no build timeout-minutes: 10 - # Only beta (rc) and master (stable) produce releases; refuse a dispatch - # from any other branch. + # Only beta (rc) and master (stable) produce releases. if: github.ref_name == 'beta' || github.ref_name == 'master' steps: - # Mint a short-lived, repo-scoped installation token from the release - # GitHub App. A non-GITHUB_TOKEN actor is required so the tag push below - # triggers ci-build; the default GITHUB_TOKEN does not trigger downstream - # workflows. Using an App (instead of a personal PAT) keeps the token - # short-lived, least-privilege, and owned by the org rather than a person. + # An App token, not GITHUB_TOKEN, so the tag push below triggers ci-build. - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 id: app-token with: @@ -41,7 +40,7 @@ jobs: - name: Configure git run: | git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - name: Rebase on top of latest remote run: git pull --rebase --autostash origin "${GITHUB_REF_NAME}" @@ -73,15 +72,7 @@ jobs: with: tag: ${{ steps.meta.outputs.tag }} - # The OpenShock API is announced from ci-build's release job, only after - # the firmware is actually built and uploaded - not here, before it exists. - # release.json is attached to the draft below so ci-build can read it. - - # Stage the release as a draft. The tag push above triggers ci-build, - # which builds the firmware and then attaches the binaries to this draft - # and publishes it (see the `release` job in ci-build.yml). Staging as a - # draft keeps the release hidden until its artifacts are in place, and - # avoids ci-build failing on an "already exists" conflict. + # A draft: the tag push triggers ci-build, whose `release` job attaches the binaries and publishes it. - name: Stage GitHub Release (draft) if: steps.meta.outputs.tag != '' env: diff --git a/.github/workflows/script-tests.yml b/.github/workflows/script-tests.yml index 7e287d244..60c8f1a1f 100644 --- a/.github/workflows/script-tests.yml +++ b/.github/workflows/script-tests.yml @@ -1,4 +1,6 @@ # Unit tests for the CI/release scripts (.github/scripts) and the build helpers in scripts/. +name: script-tests + on: push: branches: ['**'] @@ -8,6 +10,10 @@ on: - 'partitions/**' - 'boards/**' - 'sdkconfig.defaults' + - 'certificates/cert_bundle.py' + - 'pyproject.toml' + - 'requirements.txt' + - '.python-version' - '.github/workflows/script-tests.yml' # Same-repo PR branches are covered by their push run; pull_request is for forks. pull_request: @@ -17,11 +23,13 @@ on: - 'partitions/**' - 'boards/**' - 'sdkconfig.defaults' + - 'certificates/cert_bundle.py' + - 'pyproject.toml' + - 'requirements.txt' + - '.python-version' - '.github/workflows/script-tests.yml' workflow_dispatch: -name: script-tests - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -29,6 +37,10 @@ concurrency: permissions: contents: read +defaults: + run: + shell: bash + jobs: pytest: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork }} @@ -39,12 +51,20 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version-file: .github/scripts/.python-version + python-version-file: .python-version cache: 'pip' - cache-dependency-path: .github/scripts/requirements-dev.txt + cache-dependency-path: | + requirements.txt + .github/scripts/requirements-dev.txt + + - name: Install dependencies + run: pip install --require-hashes -r requirements.txt -r .github/scripts/requirements-dev.txt + + - name: Lint + run: ruff check . - - name: Install test dependencies - run: pip install --require-hashes -r .github/scripts/requirements-dev.txt + - name: Type check + run: mypy - name: Run tests - run: python -m pytest .github/scripts/tests + run: python -m pytest diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml new file mode 100644 index 000000000..6b3088ac2 --- /dev/null +++ b/.github/workflows/workflow-lint.yml @@ -0,0 +1,32 @@ +name: workflow-lint + +on: + push: + branches: ['**'] + paths: ['.github/**'] + pull_request: + paths: ['.github/**'] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + actionlint: + # Same-repo PR branches are covered by their push run; pull_request is for forks. + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: .github + persist-credentials: false + + - uses: docker://rhysd/actionlint@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 # 1.7.12 + with: + args: -color diff --git a/CMakeLists.txt b/CMakeLists.txt index abe2dff78..2a4646c0f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,12 +1,9 @@ cmake_minimum_required(VERSION 3.16.0) # --- OpenShock generated build headers --------------------------------------- -# gen_env_header.py writes openshock_board.h (chip, build mode, log level) and openshock_version.h -# (version, commit) into ${CMAKE_BINARY_DIR}/config. Neither is force-included: a header carrying the -# commit in every translation unit would defeat ccache on every commit. -# -# OPENSHOCK_BOARD (-D, from scripts/build.py) names the board to stamp into the image after linking. -# Empty for a variant build, whose image stays unstamped until scripts/package_boards.py stamps each board. +# gen_env_header.py writes openshock_board.h and openshock_version.h into ${CMAKE_BINARY_DIR}/config. +# Neither is force-included: the commit in every translation unit would defeat ccache. +# OPENSHOCK_BOARD (-D, from scripts/build.py) is empty for a variant build, which package_boards.py stamps later. set(OPENSHOCK_BOARD "" CACHE STRING "OpenShock board to stamp (boards/.defaults); empty for a variant build") # idf.py passes the interpreter of the IDF Python environment as PYTHON; fall back to the one on PATH. @@ -55,9 +52,7 @@ idf_component_get_property(_os_common_lib common COMPONENT_LIB) add_dependencies(${_os_common_lib} openshock_version_header) # --- Board descriptor stamp ------------------------------------------------- -# Stamp the board into the .bin as soon as gen_project_binary writes it, so `idf.py flash` carries the -# board. It runs on every build, which is fine because stamping is idempotent. With secure boot this -# would have to move before signing, and scripts/utils/espimage.py learn the signature block. +# stamp_board.py stamps the board into the .bin as soon as it is written, so `idf.py flash` carries it. if(OPENSHOCK_BOARD) idf_build_get_property(_os_build_dir BUILD_DIR) add_custom_command(TARGET gen_project_binary POST_BUILD @@ -69,14 +64,7 @@ if(OPENSHOCK_BOARD) endif() # --- Static filesystem image (captive-portal web assets) --------------------- -# Build the SvelteKit frontend, gzip it, and pack it into a littlefs image whose -# geometry matches the runtime mount in components/fs/src/LfsPartition.cpp (4 KiB -# blocks, 128-byte read/prog). `idf.py flash` writes it to the `static0` -# partition. -# -# The partition only exists in the OpenShock custom partition CSV; if the active -# partition table has no `static0` (e.g. a bare default-table build), skip the -# image instead of failing the whole build. +# gen_staticfs.py packs the frontend into the `static0` partition image; skipped when the partition table has none. partition_table_get_partition_info(_staticfs_offset "--partition-name static0" "offset") partition_table_get_partition_info(_staticfs_size "--partition-name static0" "size") @@ -86,12 +74,7 @@ if(_staticfs_offset STREQUAL "" OR _staticfs_size STREQUAL "") else() set(_staticfs_image ${CMAKE_BINARY_DIR}/staticfs.bin) - # ALL target so a normal `idf.py build` regenerates the image. The generator only - # runs pnpm when frontend/build is older than the frontend sources, so this stays - # cheap on incremental builds. CI never builds this target: its staticfs job builds - # the frontend itself and runs gen_staticfs.py --skip-frontend-build, and the image - # is packaged beside every variant. Staging lives in the build directory so - # concurrent board builds don't share (and wipe) one directory. + # CI never builds this target: its staticfs job runs gen_staticfs.py --skip-frontend-build once for every variant. add_custom_target(staticfs ALL BYPRODUCTS ${_staticfs_image} COMMAND ${_os_python} ${CMAKE_CURRENT_LIST_DIR}/scripts/gen_staticfs.py diff --git a/README.md b/README.md index bfbe64061..636c412c1 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ You will need: ```bash # Install dependencies -pip install -r requirements.txt +pip install --require-hashes -r requirements.txt ``` ## Support diff --git a/certificates/cert_bundle.py b/certificates/cert_bundle.py index 6609004d5..023eceff5 100644 --- a/certificates/cert_bundle.py +++ b/certificates/cert_bundle.py @@ -31,19 +31,17 @@ # See the License for the specific language governing permissions and # limitations under the License. - -from __future__ import annotations - +import argparse import base64 import datetime import hashlib import os import re +import ssl import subprocess import sys import tempfile import urllib.request -import ssl import warnings from pathlib import Path @@ -53,11 +51,13 @@ from cryptography.utils import CryptographyDeprecationWarning except ImportError: print( - 'cert_bundle.py: cryptography package not installed ' '(pip install cryptography)', + 'cert_bundle.py: cryptography package not installed (pip install cryptography)', file=sys.stderr, ) raise +HERE = Path(__file__).resolve().parent + URL_PEM = 'https://curl.se/ca/cacert.pem' URL_SHA256 = 'https://curl.se/ca/cacert.pem.sha256' MANUAL_HASH_PAGE = 'https://curl.se/docs/caextract.html' @@ -88,17 +88,18 @@ # Mozilla/curl base has retired but our infrastructure still chains through -- e.g. # GlobalSign Root CA R1, which cross-signs GTS Root R4 in the api.openshock.app chain. PINNED_CERTS_DIR = 'pinned_certs' -PINNED_CERTS_GLOB = '*.pem' # Custom certs (self-hosted/internal PKI). Local-only, git-ignored. CUSTOM_CERTS_DIR = 'custom_certs' # optional; if missing -> ignored -CUSTOM_CERTS_GLOB = '*.pem' # required extension +CERT_GLOB = '*.pem' # A pinned cert this close to (or past) expiry is a problem: generation hard-fails on an # already-expired pin; the `audit` subcommand / the CI workflow raise an alert once a # still-needed pin is within this window. Override with PINNED_CERT_EXPIRY_WARN_DAYS. EXPIRY_WARN_DAYS_DEFAULT = 90 +PEM_BEGIN = b'-----BEGIN CERTIFICATE-----' +PEM_END = b'-----END CERTIFICATE-----' PEM_BLOCK_RE = re.compile( br'-----BEGIN CERTIFICATE-----\s*[\s\S]+?\s*-----END CERTIFICATE-----\s*', re.MULTILINE, @@ -109,7 +110,7 @@ class InputError(RuntimeError): pass -def critical(msg: str) -> None: +def log(msg: str) -> None: sys.stderr.write('cert_bundle.py: ' + msg + '\n') @@ -118,21 +119,14 @@ def download_bytes(url: str, timeout: int = 30) -> bytes: req = urllib.request.Request( url, headers={ - 'User-Agent': 'esp32-gen-crt-bundle/strict', + 'User-Agent': 'openshock-cert-bundle', 'Accept': 'text/plain,*/*', }, method='GET', ) with urllib.request.urlopen(req, context=ctx, timeout=timeout) as resp: - # urllib responses vary; handle both .status and .getcode() - status = getattr(resp, 'status', None) - if status is None: - try: - status = resp.getcode() - except Exception: - status = 200 - if status != 200: - raise InputError(f'Download failed: HTTP {status} for {url}') + if resp.status != 200: + raise InputError(f'Download failed: HTTP {resp.status} for {url}') data = resp.read() if not data: raise InputError(f'Download failed: empty response from {url}') @@ -161,10 +155,9 @@ def parse_sha256_file(content: bytes, *, expect_filename: str | None = None) -> h = m.group(1).lower() fn = (m.group(2) or '').strip() - if expect_filename and fn: - # Some sha256sum formats include full paths; compare basename. - if os.path.basename(fn) != expect_filename: - raise InputError(f'SHA-256 file refers to unexpected filename {fn!r} (expected {expect_filename!r})') + # Some sha256sum formats include full paths; compare basename. + if expect_filename and fn and os.path.basename(fn) != expect_filename: + raise InputError(f'SHA-256 file refers to unexpected filename {fn!r} (expected {expect_filename!r})') return h @@ -179,15 +172,14 @@ def extract_pem_cert_blocks(pem_bytes: bytes, *, source: str) -> list[bytes]: def _load_single_pem_cert_block_from_bytes_strict(data: bytes, *, source: str) -> bytes: # Hard fail if file contains any private key material. if b'PRIVATE KEY' in data: - raise InputError(f'Custom cert file contains private key material (PRIVATE KEY): {source}') + raise InputError(f'Cert file contains private key material (PRIVATE KEY): {source}') - begin = b'-----BEGIN CERTIFICATE-----' - end = b'-----END CERTIFICATE-----' + begin, end = PEM_BEGIN, PEM_END begin_idx = data.find(begin) end_idx = data.find(end) if begin_idx == -1 or end_idx == -1: - raise InputError(f'Custom cert file must contain exactly 1 certificate: {source}') + raise InputError(f'Cert file must contain exactly 1 certificate: {source}') end_idx += len(end) @@ -199,10 +191,10 @@ def _load_single_pem_cert_block_from_bytes_strict(data: bytes, *, source: str) - # Exactly one cert block (no extra PEM blocks) if data.count(begin) != 1 or data.count(end) != 1: - raise InputError(f'Custom cert file must contain exactly 1 certificate: {source}') + raise InputError(f'Cert file must contain exactly 1 certificate: {source}') tmp = data.replace(begin, b'') if b'-----BEGIN ' in tmp: - raise InputError(f'Custom cert file contains additional PEM blocks besides the certificate: {source}') + raise InputError(f'Cert file contains additional PEM blocks besides the certificate: {source}') # Validate base64 payload decodes cleanly pem = data[begin_idx:end_idx] @@ -225,16 +217,16 @@ def _load_single_pem_cert_block_from_bytes_strict(data: bytes, *, source: str) - # Canonicalize output PEM (stable formatting) b64_canon = base64.b64encode(der) out_lines = [b64_canon[i : i + 64] for i in range(0, len(b64_canon), 64)] - return b'-----BEGIN CERTIFICATE-----\n' + b'\n'.join(out_lines) + b'\n-----END CERTIFICATE-----\n' + return PEM_BEGIN + b'\n' + b'\n'.join(out_lines) + b'\n' + PEM_END + b'\n' def _load_single_pem_cert_block_from_file(path: Path) -> bytes: try: - data = path.read_bytes() + data = (HERE / path).read_bytes() except Exception as e: - raise InputError(f'Failed to read custom cert file: {path}') from e + raise InputError(f'Failed to read cert file: {path}') from e - return _load_single_pem_cert_block_from_bytes_strict(data, source=f'custom cert file: {path}') + return _load_single_pem_cert_block_from_bytes_strict(data, source=f'cert file: {path}') def load_dir_cert_blocks(dirname: str, glob: str) -> tuple[list[bytes], list[Path]]: @@ -249,47 +241,26 @@ def load_dir_cert_blocks(dirname: str, glob: str) -> tuple[list[bytes], list[Pat - If the directory exists but contains no matches -> return ([], []) - Each file must contain exactly one certificate """ - d = Path(dirname) + d = HERE / dirname if not d.exists(): return ([], []) if not d.is_dir(): raise InputError(f'{dirname} exists but is not a directory') - paths = sorted(d.glob(glob)) + paths = sorted(p.relative_to(HERE) for p in d.glob(glob)) if not paths: return ([], []) - critical(f'Loading certificates from: {dirname}/ ({len(paths)} file(s))') + log(f'Loading certificates from: {dirname}/ ({len(paths)} file(s))') blocks: list[bytes] = [] for p in paths: - if p.suffix.lower() != '.pem': - # Should not happen due to glob, but keep strict. - raise InputError(f'Cert file must have .pem extension: {p}') blocks.append(_load_single_pem_cert_block_from_file(p)) return (blocks, paths) -def load_pinned_cert_blocks() -> tuple[list[bytes], list[Path]]: - """Load tracked, shipped pinned CA certs from pinned_certs/*.pem.""" - return load_dir_cert_blocks(PINNED_CERTS_DIR, PINNED_CERTS_GLOB) - - -def load_custom_cert_blocks() -> tuple[list[bytes], list[Path]]: - """Load local-only custom CA certs from custom_certs/*.pem.""" - return load_dir_cert_blocks(CUSTOM_CERTS_DIR, CUSTOM_CERTS_GLOB) - - -def _cert_not_after_utc(cert: x509.Certificate) -> datetime.datetime: - # cryptography >= 42 exposes tz-aware not_valid_after_utc; older exposes naive UTC. - try: - return cert.not_valid_after_utc - except AttributeError: - return cert.not_valid_after.replace(tzinfo=datetime.timezone.utc) - - def check_certs_expiry( certs: list[x509.Certificate], origins: list[str], @@ -308,16 +279,16 @@ def check_certs_expiry( expiring: list[str] = [] for idx, cert in enumerate(certs): - origin = origins[idx] if idx < len(origins) else '' - not_after = _cert_not_after_utc(cert) + origin = origins[idx] + not_after = cert.not_valid_after_utc days_left = (not_after - now).days desc = f'{cert.subject.rfc4514_string()} (origin: {origin}; notAfter: {not_after:%Y-%m-%d}; {days_left}d left)' if days_left < 0: - critical(f'EXPIRED certificate: {desc}') + log(f'EXPIRED certificate: {desc}') expired.append(desc) elif days_left <= warn_days: - critical(f'WARNING: certificate expires within {warn_days}d: {desc}') + log(f'WARNING: certificate expires within {warn_days}d: {desc}') expiring.append(desc) return (expired, expiring) @@ -330,32 +301,26 @@ def _expiry_warn_days() -> int: try: val = int(raw) except ValueError: - raise InputError(f'PINNED_CERT_EXPIRY_WARN_DAYS must be an integer, got {raw!r}') + raise InputError(f'PINNED_CERT_EXPIRY_WARN_DAYS must be an integer, got {raw!r}') from None if val < 0: raise InputError('PINNED_CERT_EXPIRY_WARN_DAYS must be >= 0') return val def _reject_non_ca(certs: list[x509.Certificate], origins: list[str], *, kind: str) -> None: - non_ca = [(c, o) for (c, o) in zip(certs, origins) if not _is_ca_certificate(c)] + non_ca = [(c, o) for (c, o) in zip(certs, origins, strict=True) if not _is_ca_certificate(c)] if not non_ca: return - critical(f'FATAL: One or more {kind} certificates are not CA certificates (BasicConstraints CA=TRUE missing)') + log(f'FATAL: One or more {kind} certificates are not CA certificates (BasicConstraints CA=TRUE missing)') for c, origin in non_ca: - critical(f' Origin : {origin}') - critical(f' Subject: {c.subject.rfc4514_string()}') - critical(f' Issuer : {c.issuer.rfc4514_string()}') + log(f' Origin : {origin}') + log(f' Subject: {c.subject.rfc4514_string()}') + log(f' Issuer : {c.issuer.rfc4514_string()}') raise InputError(f'Refusing to include non-CA {kind} certificates') -# -------------------------------------------------------------------------------------- -# audit: expiry of every shipped cert + live-endpoint relevance for pinned roots -# -------------------------------------------------------------------------------------- - -# Endpoints whose live chains decide whether a pinned root still earns its place. esp_crt_bundle -# does not do path building: it trusts the chain as the server sends it and looks up the issuer -# of the topmost presented cert among the trusted roots. So the anchor an endpoint "requires" is -# precisely the issuer of the last cert in its presented chain. +# Endpoints whose live chains decide whether a pinned root is still needed: esp_crt_bundle does no +# path building, so the required anchor is the issuer of the last cert in the presented chain. MONITORED_DOMAINS = ['api.openshock.app', 'api.openshock.dev'] PROBE_TIMEOUT_S = 20 @@ -390,24 +355,23 @@ def required_anchor_subject(domain: str) -> str | None: timeout=PROBE_TIMEOUT_S, ) except (subprocess.TimeoutExpired, FileNotFoundError) as e: - critical(f' probe error for {domain}: {e}') + log(f' probe error for {domain}: {e}') return None chain = re.findall(rb'-----BEGIN CERTIFICATE-----.+?-----END CERTIFICATE-----', proc.stdout, re.S) if not chain: - critical(f' {domain}: no certificates presented (unreachable / handshake failed)') + log(f' {domain}: no certificates presented (unreachable / handshake failed)') return None return _openssl_name(chain[-1], 'issuer') -def _emit_gh_outputs(*, removable: list[str], alert: bool, probe_ok: bool) -> None: +def _emit_gh_outputs(*, removable: list[str], alert: bool) -> None: gh_out = os.environ.get('GITHUB_OUTPUT') if not gh_out: return with open(gh_out, 'a', encoding='utf-8') as f: f.write(f'removable={" ".join(removable)}\n') f.write(f'alert={"true" if alert else "false"}\n') - f.write(f'probe_ok={"true" if probe_ok else "false"}\n') def audit() -> int: @@ -416,7 +380,7 @@ def audit() -> int: - expiry of all of them; - for pinned roots, probe the live endpoints to decide which are still needed. - Emits GitHub outputs (removable / alert / probe_ok). Returns exit code 1 on a genuine + Emits GitHub outputs (removable / alert). Returns exit code 1 on a genuine rotate-now alert (a still-needed cert expired/expiring); removal of unneeded pins is left to the workflow's PR step keyed on the `removable` output. Fail-safe: if any monitored endpoint is unreachable, nothing is proposed for removal. @@ -425,7 +389,7 @@ def audit() -> int: # The bundle as actually shipped. try: - bundle_bytes = Path(MERGED_PEM_FILE).read_bytes() + bundle_bytes = (HERE / MERGED_PEM_FILE).read_bytes() except OSError as e: raise InputError(f'Cannot read {MERGED_PEM_FILE}: {e}') from e bundle_blocks = extract_pem_cert_blocks(bundle_bytes, source=MERGED_PEM_FILE) @@ -433,12 +397,12 @@ def audit() -> int: # Pinned roots, keyed by canonical subject, so we can tell them apart from the curl # base within the merged bundle and know which file to retire. - pinned_blocks, pinned_paths = load_pinned_cert_blocks() + pinned_blocks, pinned_paths = load_dir_cert_blocks(PINNED_CERTS_DIR, CERT_GLOB) pinned_by_subject: dict[str, str] = {} - for blk, path in zip(pinned_blocks, pinned_paths): + for blk, path in zip(pinned_blocks, pinned_paths, strict=True): pinned_by_subject[_openssl_name(blk, 'subject')] = path.as_posix() - critical(f'Auditing {len(bundle_certs)} certificate(s) in {MERGED_PEM_FILE} ' + log(f'Auditing {len(bundle_certs)} certificate(s) in {MERGED_PEM_FILE} ' f'({len(pinned_by_subject)} pinned); warn window {warn_days}d') # Which anchors do the live endpoints actually require? @@ -448,10 +412,10 @@ def audit() -> int: subj = required_anchor_subject(domain) if subj is None: probe_ok = False - critical(f' {domain}: UNREACHABLE (will not retire any pin this run)') + log(f' {domain}: UNREACHABLE (will not retire any pin this run)') continue required.add(subj) - critical(f' {domain}: requires anchor -> {subj}') + log(f' {domain}: requires anchor -> {subj}') now = datetime.datetime.now(datetime.timezone.utc) removable: list[str] = [] @@ -459,37 +423,36 @@ def audit() -> int: for cert in bundle_certs: subject = _rfc2253_subject(cert) - days_left = (_cert_not_after_utc(cert) - now).days + days_left = (cert.not_valid_after_utc - now).days pinned_path = pinned_by_subject.get(subject) - if pinned_path is not None: - needed = subject in required - if not needed: - if probe_ok: - critical(f'REMOVABLE pin (no endpoint chains through it): {pinned_path} [{subject}]') - removable.append(pinned_path) - else: - critical(f'unused pin, but a probe failed -> keeping (fail-safe): {pinned_path}') - continue - # Still needed -> expiry is an operational risk. + if pinned_path is None: + # Curl/base (or committed custom) root: not retired individually, but an expired one still fails. if days_left < 0: - critical(f'ALERT: pinned root EXPIRED and still required: {pinned_path} [{subject}]') + log(f'ALERT: shipped root EXPIRED: {subject}') alert = True elif days_left <= warn_days: - critical(f'ALERT: pinned root expires in {days_left}d and still required: {pinned_path} [{subject}]') - alert = True - else: - # Curl/base (or committed custom) root. We can't retire these individually, but a - # shipped root that is already expired is still worth failing on. - if days_left < 0: - critical(f'ALERT: shipped root EXPIRED: {subject}') - alert = True - elif days_left <= warn_days: - critical(f'note: base root expires in {days_left}d (curl refresh will handle it): {subject}') + log(f'note: base root expires in {days_left}d (curl refresh will handle it): {subject}') + continue + + if subject not in required: + if probe_ok: + log(f'REMOVABLE pin (no endpoint chains through it): {pinned_path} [{subject}]') + removable.append(pinned_path) + else: + log(f'unused pin, but a probe failed -> keeping (fail-safe): {pinned_path}') + continue + # Still needed, so expiry is an operational risk. + if days_left < 0: + log(f'ALERT: pinned root EXPIRED and still required: {pinned_path} [{subject}]') + alert = True + elif days_left <= warn_days: + log(f'ALERT: pinned root expires in {days_left}d and still required: {pinned_path} [{subject}]') + alert = True - critical('') - critical(f'Summary: {len(removable)} removable pin(s), alert={alert}, probe_ok={probe_ok}') - _emit_gh_outputs(removable=removable, alert=alert, probe_ok=probe_ok) + log('') + log(f'Summary: {len(removable)} removable pin(s), alert={alert}, probe_ok={probe_ok}') + _emit_gh_outputs(removable=removable, alert=alert) return 1 if alert else 0 @@ -500,10 +463,9 @@ def _is_ca_certificate(cert: x509.Certificate) -> bool: (We keep this strict to avoid adding leaf/server certs by mistake.) """ try: - bc = cert.extensions.get_extension_for_class(x509.BasicConstraints).value - except Exception: + return cert.extensions.get_extension_for_class(x509.BasicConstraints).value.ca + except x509.ExtensionNotFound: return False - return bool(getattr(bc, 'ca', False)) def load_all_certs( @@ -515,9 +477,7 @@ def load_all_certs( certs: list[x509.Certificate] = [] for idx, b in enumerate(blocks, start=1): - origin = '' - if origins and 0 <= (idx - 1) < len(origins): - origin = origins[idx - 1] + origin = origins[idx - 1] if origins else '' with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always', CryptographyDeprecationWarning) @@ -525,12 +485,12 @@ def load_all_certs( try: cert = x509.load_pem_x509_certificate(b) except Exception as e: - critical(f'FATAL: Invalid certificate at index {idx} ({label})') + log(f'FATAL: Invalid certificate at index {idx} ({label})') if origin: - critical(f' Origin : {origin}') - critical(f' Reason: {e}') - critical(' Offending PEM:') - critical(b.decode('ascii', errors='replace').rstrip()) + log(f' Origin : {origin}') + log(f' Reason: {e}') + log(' Offending PEM:') + log(b.decode('ascii', errors='replace').rstrip()) raise InputError('Aborting due to invalid CA certificate') from e for warn in w: @@ -538,16 +498,16 @@ def load_all_certs( subject = cert.subject.rfc4514_string() issuer = cert.issuer.rfc4514_string() - critical('WARNING: CA certificate triggers CryptographyDeprecationWarning') - critical(' This may become fatal in future cryptography releases.') - critical(f' Index : {idx} ({label})') + log('WARNING: CA certificate triggers CryptographyDeprecationWarning') + log(' This may become fatal in future cryptography releases.') + log(f' Index : {idx} ({label})') if origin: - critical(f' Origin : {origin}') - critical(f' Subject : {subject}') - critical(f' Issuer : {issuer}') - critical(f' Reason : {warn.message}') - critical(' Offending PEM:') - critical(b.decode('ascii', errors='replace').rstrip()) + log(f' Origin : {origin}') + log(f' Subject : {subject}') + log(f' Issuer : {issuer}') + log(f' Reason : {warn.message}') + log(' Offending PEM:') + log(b.decode('ascii', errors='replace').rstrip()) certs.append(cert) @@ -571,28 +531,26 @@ def validate_unique_subjects(certs: list[x509.Certificate]) -> None: seen[subject_der] = crt -def atomic_write(path: str, data: bytes) -> None: - out_dir = os.path.dirname(os.path.abspath(path)) or '.' - os.makedirs(out_dir, exist_ok=True) +def atomic_write(path: Path, data: bytes) -> None: + out_dir = path.parent + out_dir.mkdir(parents=True, exist_ok=True) fd = None tmp_path = None try: - fd, tmp_path = tempfile.mkstemp(prefix='.tmp_crt_bundle_', dir=out_dir) + fd, tmp_name = tempfile.mkstemp(prefix='.tmp_crt_bundle_', dir=out_dir) + tmp_path = Path(tmp_name) with os.fdopen(fd, 'wb') as f: f.write(data) f.flush() os.fsync(f.fileno()) fd = None - os.replace(tmp_path, path) + tmp_path.replace(path) tmp_path = None finally: - try: - if tmp_path and os.path.exists(tmp_path): - os.remove(tmp_path) - except Exception: - pass + if tmp_path: + tmp_path.unlink(missing_ok=True) def _append_pem_blocks(base: bytes, blocks: list[bytes]) -> bytes: @@ -606,66 +564,60 @@ def _append_pem_blocks(base: bytes, blocks: list[bytes]) -> bytes: return bytes(out) +def load_group(dirname: str, kind: str) -> tuple[list[bytes], list[x509.Certificate], list[str]]: + """Load and CA-check one directory of certs; empty when the directory holds none.""" + blocks, paths = load_dir_cert_blocks(dirname, CERT_GLOB) + if not blocks: + log(f'No {kind} certificates found ({dirname}/{CERT_GLOB})') + return [], [], [] + origins = [p.as_posix() for p in paths] + certs = load_all_certs(blocks, label=kind, origins=origins) + _reject_non_ca(certs, origins, kind=kind) + log(f'Found {len(blocks)} {kind} certificate(s)') + return blocks, certs, origins + + def generate() -> int: warn_days = _expiry_warn_days() - critical(f'Downloading CA bundle from: {URL_PEM}') + log(f'Downloading CA bundle from: {URL_PEM}') pem = download_bytes(URL_PEM) got_hash = hashlib.sha256(pem).hexdigest().lower() - critical(f'Downloaded {len(pem)} bytes') - critical(f'SHA-256(curl download) = {got_hash}') + log(f'Downloaded {len(pem)} bytes') + log(f'SHA-256(curl download) = {got_hash}') - critical(f'Downloading published hash from: {URL_SHA256}') + log(f'Downloading published hash from: {URL_SHA256}') sha_file = download_bytes(URL_SHA256) expected_hash = parse_sha256_file(sha_file, expect_filename='cacert.pem') - critical(f'Published SHA-256 = {expected_hash}') + log(f'Published SHA-256 = {expected_hash}') - critical('') - critical('MANUAL VERIFICATION REQUIRED') - critical(' Compare the "SHA-256(curl download)" hash above (the upstream bytes, computed') - critical(' BEFORE pinned/custom certs are appended) against:') - critical(f' {MANUAL_HASH_PAGE}') - critical('') + log('') + log('MANUAL VERIFICATION REQUIRED') + log(' Compare the "SHA-256(curl download)" hash above (the upstream bytes, computed') + log(' BEFORE pinned/custom certs are appended) against:') + log(f' {MANUAL_HASH_PAGE}') + log('') if got_hash != expected_hash: raise InputError('SHA-256 mismatch against cacert.pem.sha256; refusing to proceed') - critical('Automatic SHA-256 validation passed') + log('Automatic SHA-256 validation passed') # Curl/Mozilla base. blocks = extract_pem_cert_blocks(pem, source='downloaded curl CA bundle') curl_certs = load_all_certs(blocks, label='CA', origins=['curl bundle'] * len(blocks)) - critical(f'Found {len(blocks)} certificate blocks in curl bundle') - - # Pinned roots (tracked, shipped, appended into cacert-merged.pem, expiry-monitored). - pinned_blocks, pinned_paths = load_pinned_cert_blocks() - pinned_certs: list[x509.Certificate] = [] - if pinned_blocks: - pinned_origins = [f'{PINNED_CERTS_DIR}/{p.name}' for p in pinned_paths] - pinned_certs = load_all_certs(pinned_blocks, label='pinned', origins=pinned_origins) - _reject_non_ca(pinned_certs, pinned_origins, kind='pinned') - - # Never ship an already-dead anchor. - expired, expiring = check_certs_expiry(pinned_certs, pinned_origins, warn_days) - if expired: - raise InputError(f'Refusing to ship {len(expired)} expired pinned certificate(s)') - if expiring: - critical(f'NOTE: {len(expiring)} pinned cert(s) expire within {warn_days}d; rotate soon') - critical(f'Found {len(pinned_blocks)} pinned certificate(s)') - else: - critical(f'No pinned certificates found ({PINNED_CERTS_DIR}/*.pem)') - - # Custom roots (local-only, git-ignored self-hosted PKI). - custom_blocks, custom_paths = load_custom_cert_blocks() - custom_certs: list[x509.Certificate] = [] - if custom_blocks: - custom_origins = [f'custom cert file: {p}' for p in custom_paths] - custom_certs = load_all_certs(custom_blocks, label='custom', origins=custom_origins) - _reject_non_ca(custom_certs, custom_origins, kind='custom') - critical(f'Found {len(custom_blocks)} custom certificate(s)') - else: - critical('No custom certificates found (custom_certs/*.pem)') + log(f'Found {len(blocks)} certificate blocks in curl bundle') + + # Pinned roots (tracked, shipped, expiry-monitored), then custom roots (local-only, git-ignored self-hosted PKI). + pinned_blocks, pinned_certs, pinned_origins = load_group(PINNED_CERTS_DIR, 'pinned') + # Never ship an already-dead anchor. + expired, expiring = check_certs_expiry(pinned_certs, pinned_origins, warn_days) + if expired: + raise InputError(f'Refusing to ship {len(expired)} expired pinned certificate(s)') + if expiring: + log(f'NOTE: {len(expiring)} pinned cert(s) expire within {warn_days}d; rotate soon') + custom_blocks, custom_certs, custom_origins = load_group(CUSTOM_CERTS_DIR, 'custom') # Deduplicate: drop any pinned/custom cert whose DER is already present (in the base # or an earlier pin/custom) so the merged output never contains the same cert twice. @@ -673,13 +625,14 @@ def generate() -> int: extra_blocks: list[bytes] = [] extra_certs: list[x509.Certificate] = [] for blk, cert, origin in zip( - list(pinned_blocks) + list(custom_blocks), + pinned_blocks + custom_blocks, pinned_certs + custom_certs, - [f'{PINNED_CERTS_DIR}/{p.name}' for p in pinned_paths] + [f'custom_certs/{p.name}' for p in custom_paths], + pinned_origins + custom_origins, + strict=True, ): der = cert.public_bytes(serialization.Encoding.DER) if der in seen_der: - critical(f'Skipping duplicate certificate (already in bundle): {cert.subject.rfc4514_string()} [{origin}]') + log(f'Skipping duplicate certificate (already in bundle): {cert.subject.rfc4514_string()} [{origin}]') continue seen_der.add(der) extra_blocks.append(blk) @@ -688,47 +641,43 @@ def generate() -> int: # Validate the deduplicated set: unique subject names so esp_crt_bundle's subject-name # lookup stays unambiguous (this also catches two *different* certs sharing a subject). certs = curl_certs + extra_certs - critical(f'Bundle: {len(curl_certs)} base + {len(extra_certs)} pinned/custom = {len(certs)} certs') + log(f'Bundle: {len(curl_certs)} base + {len(extra_certs)} pinned/custom = {len(certs)} certs') validate_unique_subjects(certs) # Save the verbatim curl base (so `sha256sum {BASE_PEM_FILE}` matches curl's published # hash) and the merged store the firmware actually packs. The firmware build converts # and packs cacert-merged.pem via ESP-IDF's certificate-bundle step; this script does not emit # the packed x509 bundle itself. - critical(f'Saving {BASE_PEM_FILE} (verbatim curl base, {len(blocks)} certs, atomic)') - atomic_write(BASE_PEM_FILE, pem) + log(f'Saving {BASE_PEM_FILE} (verbatim curl base, {len(blocks)} certs, atomic)') + atomic_write(HERE / BASE_PEM_FILE, pem) # Merged file gets our own banner in place of curl's leading header, then all certs. - base_no_header = pem[pem.find(b'-----BEGIN CERTIFICATE-----') :] + base_no_header = pem[pem.find(PEM_BEGIN) :] combined = MERGED_BANNER + _append_pem_blocks(base_no_header, extra_blocks) - critical(f'Saving {MERGED_PEM_FILE} ({len(certs)} certs, atomic)') - atomic_write(MERGED_PEM_FILE, combined) + log(f'Saving {MERGED_PEM_FILE} ({len(certs)} certs, atomic)') + atomic_write(HERE / MERGED_PEM_FILE, combined) - critical('Done') + log('Done') return 0 -USAGE = 'usage: cert_bundle.py [generate|audit]' - - def main() -> int: - args = sys.argv[1:] - if not args or args == ['generate']: - return generate() - if args == ['audit']: - return audit() - if args in (['-h'], ['--help']): - critical(USAGE) - critical(' generate (default) download + verify; write cacert-curl.pem (base) + cacert-merged.pem (base+pinned+custom)') - critical(' audit audit expiry of every shipped cert + probe endpoints for pinned') - critical(' relevance; exits 1 on a still-needed expired/expiring cert') - return 0 - raise InputError(USAGE) + ap = argparse.ArgumentParser(description='OpenShock CA trust store manager.') + ap.add_argument( + 'command', + nargs='?', + choices=['generate', 'audit'], + default='generate', + help='generate (default): download + verify curl\'s bundle, write cacert-curl.pem (base) and ' + 'cacert-merged.pem (base+pinned+custom); audit: check expiry of every shipped cert and probe endpoints ' + 'for pinned relevance, exit 1 on a still-needed expired/expiring cert', + ) + return audit() if ap.parse_args().command == 'audit' else generate() if __name__ == '__main__': try: raise SystemExit(main()) except InputError as e: - critical(str(e)) - raise SystemExit(2) + log(str(e)) + raise SystemExit(2) from None diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..115da66c0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = 'openshock-firmware-tools' +version = '0.0.0' +requires-python = '>=3.10' + +[tool.pytest.ini_options] +pythonpath = ['.github/scripts', 'scripts', '.github/scripts/tests'] +testpaths = ['.github/scripts/tests'] + +[tool.ruff] +target-version = 'py310' +line-length = 120 +extend-exclude = ['components', 'managed_components', 'frontend', 'build', 'tmp'] + +[tool.ruff.format] +quote-style = 'single' + +[tool.ruff.lint] +select = ['E', 'F', 'I', 'UP', 'B', 'SIM'] +ignore = ['E501', 'SIM108'] + +[tool.ruff.lint.isort] +known-first-party = ['utils'] + +[tool.mypy] +python_version = '3.10' +files = ['scripts', '.github/scripts'] +exclude = ['/tests/'] + +[[tool.mypy.overrides]] +module = ['esptool', 'esptool.*', 'littlefs', 'requests'] +ignore_missing_imports = true diff --git a/requirements.in b/requirements.in new file mode 100644 index 000000000..4e2b452d3 --- /dev/null +++ b/requirements.in @@ -0,0 +1,5 @@ +cryptography==50.0.2 +esptool==5.6.0 +# scripts/gen_staticfs.py packs the captive-portal assets into a littlefs image +# matching the runtime mount geometry (components/fs/src/LfsPartition.cpp). +littlefs-python==0.19.0 diff --git a/requirements.txt b/requirements.txt index 4e2b452d3..536d0f927 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,650 @@ -cryptography==50.0.2 -esptool==5.6.0 -# scripts/gen_staticfs.py packs the captive-portal assets into a littlefs image -# matching the runtime mount geometry (components/fs/src/LfsPartition.cpp). -littlefs-python==0.19.0 +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in --universal --python-version 3.10 --generate-hashes -o requirements.txt +cffi==2.1.1 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via + # esp-pylib + # esptool + # rich-click +colorama==0.4.6 ; sys_platform == 'win32' \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 + # via rich-click +cryptography==50.0.2 \ + --hash=sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602 \ + --hash=sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2 \ + --hash=sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047 \ + --hash=sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c \ + --hash=sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42 \ + --hash=sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18 \ + --hash=sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51 \ + --hash=sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81 \ + --hash=sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856 \ + --hash=sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2 \ + --hash=sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de \ + --hash=sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7 \ + --hash=sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd \ + --hash=sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2 \ + --hash=sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be \ + --hash=sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45 \ + --hash=sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0 \ + --hash=sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e \ + --hash=sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c \ + --hash=sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5 \ + --hash=sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452 \ + --hash=sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48 \ + --hash=sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05 \ + --hash=sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1 \ + --hash=sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93 \ + --hash=sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04 \ + --hash=sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e \ + --hash=sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67 \ + --hash=sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7 \ + --hash=sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107 \ + --hash=sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079 \ + --hash=sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134 \ + --hash=sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227 \ + --hash=sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1 \ + --hash=sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539 \ + --hash=sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e \ + --hash=sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d \ + --hash=sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c \ + --hash=sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd \ + --hash=sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020 \ + --hash=sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd \ + --hash=sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94 \ + --hash=sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a \ + --hash=sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408 \ + --hash=sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37 \ + --hash=sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e \ + --hash=sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454 \ + --hash=sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c \ + --hash=sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc \ + --hash=sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37 \ + --hash=sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767 \ + --hash=sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a \ + --hash=sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5 \ + --hash=sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc \ + --hash=sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67 \ + --hash=sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8 \ + --hash=sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480 \ + --hash=sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb \ + --hash=sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b + # via + # -r requirements.in + # esptool +esp-pylib==1.2.0 \ + --hash=sha256:51f0c44a0f7a92fdded9cab54bde92441b22b464f7d684ed179afb57000bed70 \ + --hash=sha256:581fd31eb20db4b80519fe883eaeacf5b3c6a2c0b7cce353a7d326ae7cbd4f5f + # via esptool +esptool==5.6.0 \ + --hash=sha256:86c7249e9cc3a824e032ede9ba20abad057a2c49e231418cc4f02b177bb36f42 + # via -r requirements.in +intelhex==2.3.0 \ + --hash=sha256:87cc5225657524ec6361354be928adfd56bcf2a3dcc646c40f8f094c39c07db4 \ + --hash=sha256:892b7361a719f4945237da8ccf754e9513db32f5628852785aea108dcd250093 + # via esptool +littlefs-python==0.19.0 \ + --hash=sha256:012a4117d9e07376100907ae737ee35b48d58e394a17b00f3cbcd83d24d353bb \ + --hash=sha256:020e2cc8b08e4246b41d8413146e8193c84e779e8e10e5ff08d6e60250022331 \ + --hash=sha256:0864448636264eee6e2673118c711ad22e0663b6a6c63d2a95c05ee212d9c1fa \ + --hash=sha256:0965a06332177694681c1485a564d0ede4958258a649f45a4ed409c98ad2ab2c \ + --hash=sha256:0aaae1347d5a05da5e582b0d4c241d70b2736d01e5dbc58f9ae39ac659fc8393 \ + --hash=sha256:0e5ca23e6f1938d92ea75f7ce2409b77e09480e7b523cac35b2bdbf6c957305e \ + --hash=sha256:0fcf1e8c35e63d399a89f55fd2dad370ed312ea8584aba2f5fea691dabe2db72 \ + --hash=sha256:109dbfdd1bd381567d7f831aabcb279a09ffd85c80e9c696434395fa83fb7606 \ + --hash=sha256:140bcd3e34896e90dae524cc4762d97378cbd0e3b473a06829d6144e6ca9dd5b \ + --hash=sha256:15007c78d1bb6b1a06ec0e7c662dd9e7469a06a5134fadd67bcd9812223905a2 \ + --hash=sha256:1586da7b9c6a4508479395322378fb36ca021e586cef2d2fe4b4b3f26c4e3c04 \ + --hash=sha256:19d436537ace9f01d0f43b0453242d3aad3132abf97c5265302f03d84cc56482 \ + --hash=sha256:1bd7dc438f27d6b3d6ec9a9210740653eefa1239745d0c9da230554d8bed9470 \ + --hash=sha256:2c6df3da27680b08350f898c3a66ebddce15f1892f6aa007f9ca5f36d262aa76 \ + --hash=sha256:2c6fc43c47003b8a394d9639b89c82bbe234cf9fdc1f835be4f30b5e4ba05d49 \ + --hash=sha256:30b3c6c8f7e85526747eb8188c2b8ccf572ac1701c7ecf4949b1231de0ced510 \ + --hash=sha256:4058e125fa7b44f91327ecf57db18f3a0d67d9015a67d4579e2e683870d33627 \ + --hash=sha256:4a7be934562a5e3053646ce5bb0a98dea8361b97b8aa06bc830c18d49f79a8aa \ + --hash=sha256:4ce8c6eeeb2703abaf8cb60f2dab624b17af42b13df3dbc6c42d4142bad79a59 \ + --hash=sha256:51425e64a4e37bcbaaa2c99e12ffbbb1d81f96618bb824ac68c938757ed853d3 \ + --hash=sha256:53edd72cbce541939beec404a5509f4afc08ef694759201ffe91aff8adc5f254 \ + --hash=sha256:53f19918ad27c7fec890eb08d2f0d5649cf55c2c179e2777c4ea2ed958ddbe67 \ + --hash=sha256:5421c47b5bdd43d4cb855910263ee8884c2b6b0a779dfab6f31e1e27604a93d0 \ + --hash=sha256:54945e95faaf1edaf7bd554ea13889ad2ab786e213d0fca114385ee16de2c54d \ + --hash=sha256:55343a36ccbf389d927f2e07ed7097b3b65f3e87b0416a670d5cae0f6131b5a8 \ + --hash=sha256:5675b86b67a12a9764924525ddc15ef82b688714d4093ce696e082253ae2e790 \ + --hash=sha256:58c37bc1d4d845af40a2456f1850359dd35325001f52208b04fcc07068e2e5fb \ + --hash=sha256:58e0db33a7481cd85e4e7074959a4df39bc4f76426596fcab7462913ad13f288 \ + --hash=sha256:58e8eb151741747447c953286c35743d82a51527bc500d61678823abdf14b6b5 \ + --hash=sha256:5a9a84ac0a4da2e37f9bfed0ecf70d2a8c57063000a39e93d1e4ab247643e7bd \ + --hash=sha256:5eef1580bc4a1678b5777ace3a646cf064bcb282114eea8c4a251c3704d2affd \ + --hash=sha256:6051ff3ece097e21eb301693ca93e7bceac55cbe378ca14dfdd948a08dfe9866 \ + --hash=sha256:62b7b9ff5ee463bf0359e2bfd04231cc06550481a16a50c79bb3d7af4bd77a4c \ + --hash=sha256:643ffb360cedfde6a310da09ae806d030dd1769ef78bd79afe09eb345c4acccc \ + --hash=sha256:65a415e6efa597091bc292f8e6b7ffb7b96d4e7209cbbf9c3b856df18fb5934a \ + --hash=sha256:6a5a799297570241487bdf42de82fa8d238966f1af502f4f4f5b32bfc4d74676 \ + --hash=sha256:6d59cb8a72912cbf7f10a3f88f881e8cef625945b3259083f69b71816087c617 \ + --hash=sha256:711fa5c5748ba24cf3178ccff632357e4f93fb28901ceab6927e9c03a6b37e43 \ + --hash=sha256:71252f91d871357d2581473367840806c6dc05427e965e100f4b1433ae402c62 \ + --hash=sha256:7eb56074fbef683a923908091cae60fb3554f9bdaf82db92811e5bae8675b2cd \ + --hash=sha256:870f9b7c6d309bbd6d4ad95757c1af39376a5ba3c43800cab8c0933f5ad6da34 \ + --hash=sha256:89905592c4f509375d7d3d1a0187ceda57db74d8371093ccf42d5db6f190b10f \ + --hash=sha256:8abdaa1c78cb665c6cad6c8c5f0893d702af86587aadd7f1bc0442f61c742a19 \ + --hash=sha256:903bea7431e625f05cf76babcdf9bd39fcee7e12ebae43284711af9424c5f3c7 \ + --hash=sha256:947b3590e995d022051509c58c310550478f47074dca58479217b5938d5b7128 \ + --hash=sha256:986360d0fe75a123f156b5d6009cbc100ed2d2b966bada9b180ce7ff0faa43ee \ + --hash=sha256:9a409b071260aa0006b773c17528f8e2c91cf115799bddf6951e0e07b99b359e \ + --hash=sha256:9cefa021649f760979657a414874f88292317effdd38dfbdd7edac7800ea514e \ + --hash=sha256:9e18b5359f24e6189d2869ae40297311d073ebad3be68cc83df6643b3a1089ba \ + --hash=sha256:a27c00ad37a5ad79ad1b7dd72dd18d82b01b1deebe6aba50527424f13f16cc43 \ + --hash=sha256:b64cc280bb53a0d93a8ccc4ab6ec3e90f53bbd06c6d7f318567cdaac26968046 \ + --hash=sha256:c170794e550c18853f2ea1767df526aa879c27c963064cd88920994430cc6ffb \ + --hash=sha256:c197db6623142b33f484709445d04902c4c6af91c686d065f703d6ae94309fa2 \ + --hash=sha256:c8d017ff77bd7435bdacb4d94e7f7c53aceab0bed5ada73940be9ed5ba4e205e \ + --hash=sha256:c99b0ec49e9b840b2b4b143bc60272a9bed24eaf5187d166b2d54b09aac19d39 \ + --hash=sha256:ccfa7e423d55b2645095a8abf56d4f46ae932c04e19ecd9e08d9e1191e5603cf \ + --hash=sha256:d054ac280541afdae5722fb8d7ebc0e121d822c33d95d57889fd8fab2a211486 \ + --hash=sha256:d15020333abda47819d9d11d6fabe13eb64b83abeb9ae2d93c07a142c53f6aa4 \ + --hash=sha256:d63e077eecd9982f0d3816ee0c95dd3b6c3e514e54272a4e80154df7f320e342 \ + --hash=sha256:d643f7d5c4adf601e500b8958deb99d6f8bae914cf6c6be14e3cad047888506f \ + --hash=sha256:e006ad20d0af696967b55ff6d74a5ce1a0ecce5ba5a6a0f8030d56b0a6f6e92f \ + --hash=sha256:ec8d3de4f05d9420d0e1d013d9134cc68b2b6c15656115a7add2f1f9f5a655be \ + --hash=sha256:f155325d95e95690b39ec716e20b87b1c5a1a09a91cde158208aaa6277bea896 \ + --hash=sha256:f5125a8d763129808d52e874501e45100b7fdd619584e558674507202a1f1b38 \ + --hash=sha256:f526d9216cc9dfccb81b9d7d990d9a2e9cee62261ec608bea8465e951006bd56 \ + --hash=sha256:f889f3d1839df23fda7c69a3b4d9a3fa1083330ab03b121448b71f51442c6d63 \ + --hash=sha256:f8ecde73f89f4bd8edf764f5c422a40aef68bc401ce3465ced10cd0525bd23f3 \ + --hash=sha256:fa1280eeb6d32b026dea65e5078d5af0fadccb26dbf709c073d0e5729270a4b7 \ + --hash=sha256:fb9c65c235d33e10b9caa06d3cc37e396cc10f85d0ed7a4de7bfdb35f1c94814 + # via -r requirements.in +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +pycparser==3.11 ; python_full_version < '3.11' and implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' \ + --hash=sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80 \ + --hash=sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc + # via cffi +pycparser==3.12 ; python_full_version >= '3.11' and implementation_name != 'PyPy' and platform_python_implementation != 'PyPy' \ + --hash=sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e \ + --hash=sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6 + # via cffi +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via rich +pyserial==3.5 \ + --hash=sha256:3c77e014170dfffbd816e6ffc205e9842efb10be9f58ec16d3e8675b4925cddb \ + --hash=sha256:c4451db6ba391ca6ca299fb3ec7bae67a5c55dde170964c7a14ceefec02f2cf0 + # via + # esp-pylib + # esptool +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via esptool +reedsolo==1.7.0 \ + --hash=sha256:2b6a3e402a1ee3e1eea3f932f81e6c0b7bbc615588074dca1dbbcdeb055002bd \ + --hash=sha256:c1359f02742751afe0f1c0de9f0772cc113835aa2855d2db420ea24393c87732 + # via esptool +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via + # esp-pylib + # rich-click +rich-click==1.9.9 \ + --hash=sha256:324cba7513cd4187ee92b2eef21f071714e45be062458c8b157bd7e0c81103e3 \ + --hash=sha256:365e7a9d0adb42e41ea832a0a12e02c44c079a26536dee688125eb9814f97274 + # via + # esp-pylib + # esptool +typing-extensions==4.16.0 ; python_full_version < '3.11' \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # cryptography + # rich-click +websockets==16.1.1 ; python_full_version < '3.11' \ + --hash=sha256:01fbdcbac298efe19360b94bc0039c8f746f0220ba570f327577bfee81059175 \ + --hash=sha256:024193f8551a2b0eafbdd160911012c4e6c228c28430c84433253299a9e42d6a \ + --hash=sha256:04fd29a0e2fe9414a95b00e92c67ae51bf900c50c0f8a4b2dafdad621f49ea1d \ + --hash=sha256:056ae37939ed7e9974f364f5864e76e49182622d8f9751ac1903c0d09b013985 \ + --hash=sha256:0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1 \ + --hash=sha256:0ffd3031ea8bda8d61762e84220186105ba3b748b3c8da2ae4f7816fac03e573 \ + --hash=sha256:1214e673c404684b9bf7154f5cf43b45025b1a6160fac3a9e438e9c1a97e22cb \ + --hash=sha256:125f22dbefaf1554fea66fc83851490edb284ce4f501d37ffed2752f418332d9 \ + --hash=sha256:130937b167a52af203c8d58e78d67705874e82759862e3b9671a452fec4abc87 \ + --hash=sha256:1427fb4cf0d72f66333e2cacc3ff5f575bf2d7008166ce991a4a470b21d51a22 \ + --hash=sha256:195c978b065fa40910582464f99d6b15c8b314c68e0546549a55ed83f4735328 \ + --hash=sha256:1d27fa8462ad6a1cb36206a3d0640b2333340def181fae11ed7f9adeaa5c0747 \ + --hash=sha256:1db4de4a0e95673f7545d393c49eeb0c2f18ac1ef93073218c79d5cdb2ee75ab \ + --hash=sha256:1f79c89b5eb034d1722938a891916582f8f7f503f58ca22518a63c3f2cd18499 \ + --hash=sha256:23253dd5bcae3f9aaee0a1d30967a8dbd52e5d3cff93a2e5b84df57b77d4750d \ + --hash=sha256:249116b4a76063d930a46391ad56e135c286e4562a18309029fc2c73f4ed4c62 \ + --hash=sha256:29dfa8114c4a620c69591c5973860f768eac29d3fd6904f37f34266cb219c512 \ + --hash=sha256:2a606d9c24035242a3e256e9d5b77ed9cd6bccfcb7cf993e5ca3c0f6f68fb6a7 \ + --hash=sha256:2a636ff1e7a5c4edf71ef0e79adae7f25dba93b4fcbe3dc958733477ffeb0eaf \ + --hash=sha256:2bb5d041a8307d2e18782e7ce777f6fdb1e8c2f5d09291484b18c294b789d9aa \ + --hash=sha256:2e28e602bb13da44fbe518c1781a88e3b9d4c3d48d02c9bad83e546164336f57 \ + --hash=sha256:30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e \ + --hash=sha256:34420aaa64440ebd51ac72ca8a45ef4626429438c9b02e633ae412ed43f925d3 \ + --hash=sha256:38565aca3e01ea8734e578fb2118dade0ecb0250533f29e22b8d1a7a196cf4d0 \ + --hash=sha256:387e8e4aa5df2f90b198fa3cad3478822a89cf905b6a6d6c97dc3664689640cc \ + --hash=sha256:39f2a024af5c345ffe8fcf1ee18c049c024c94df393bb09b044a6917c77bde43 \ + --hash=sha256:3df13f73af9b3b38ab1195eb299ecb67a4330c911c97ae04043ff74085728abe \ + --hash=sha256:414e596c75f74e0994084694189d7dc9229fb278e33064d6784b73ffbba3ca31 \ + --hash=sha256:41c8e77f17294c0ac18008a7309b99b34ee72247ef10b6dff4c3f8b5ac29896b \ + --hash=sha256:42290eb6db4ccaca7012656738214f8514082fb6fa40cdeb61bb9a471b52e383 \ + --hash=sha256:42f599f4d48c7e1a3338fdaac3acd075be3b3cf02d4b274f3bf2767aedd3d217 \ + --hash=sha256:43e3a9fdd7cbf7ba6040c31fae0faf84ca1474fef777c4e37912f1540f854499 \ + --hash=sha256:443aefe96b7fdb132e2a70806cca1f2af49bb3f28e47abcd7c2e9dcf4d8fa1b8 \ + --hash=sha256:46dcaa042cd1de6c59e7d9269fa63ff7572b6df40510600b678f0826b3c7af51 \ + --hash=sha256:496af849a472b531f758dbd4d61338f5000538cb1a7b3d20d9d32a264517f509 \ + --hash=sha256:49ae99bdfcae803a885c926bf14f886196e84925395bb3f568fef5c0f0979d7d \ + --hash=sha256:4b57693728576d84ede0a77987ab16881b783d2cd9f1dc180a8fbbc3f79c4428 \ + --hash=sha256:4e3b680b1e0a27457e727a0d572fd81dffa87b6dbf8b228ab57da64f7d85aead \ + --hash=sha256:4e8d01cc3bcae7bbf8167f944aeafefed590fae5693552bba9794a9df68371cc \ + --hash=sha256:5283810d2646741a0d8da2aa733d6aefa0545809afccb2a5d105a26bc45125f1 \ + --hash=sha256:53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3 \ + --hash=sha256:536676848fc5961aca9d20389951f59169508f765637a172403dc5434d722fa0 \ + --hash=sha256:54509b8e92fee4453e152b7558ddef37ce9705a044922f2095a6105e3f80c96f \ + --hash=sha256:56cd5fc4f10a9ea8aa0804bddb7b42506cf9e136046f3b4c27de8fec9e2ecba5 \ + --hash=sha256:5bfd1ac19b1b9986a9c95a82d5e23a391ebb09e12c34d7be6094b86efcc35731 \ + --hash=sha256:5c31aa7e39ee3e8a358573257f1c0bb5c52430d1b637030dd9c8cc2c282926be \ + --hash=sha256:5e3b7d601f6f84156b08cc4a5e541c2b50ad7b36cfc302b657a12477c904a5df \ + --hash=sha256:61922544a0587a13fd3f53e4c0e5e606510c7b0d9d22c8444e5fae22a06b38cb \ + --hash=sha256:6456ff333092d509127d75a638cb411afae8ff17f092635015d1902efec8a293 \ + --hash=sha256:69159730a823dde3ea8d08783e8d47ef135a6d7e8d44eb127e32b321c9db8e3e \ + --hash=sha256:69e52d175a0a7d1e13b4b67ad41c560b7d98e8c6f6126eb0bda496c784faf8c7 \ + --hash=sha256:6aaface73b9c71974c6497366d8b9628357f6c9749e09c4ea3610176c63f2ae3 \ + --hash=sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3 \ + --hash=sha256:6ff9417c0ada4d0f7d212f928303e5579bdf3ace4c802fa4afabb30995da58c3 \ + --hash=sha256:7421fad442de870a8cbf2287d1cad7e706ece0dbfeba5e911df132cbdc1cb56a \ + --hash=sha256:7883388947767080f094950b342b30d35a2a06b849cd967c422fa0db72b40ea9 \ + --hash=sha256:79eace538c6a97e96d0d03d4f9d314f9677f5ed85a8a984992ffd90b13cb8a56 \ + --hash=sha256:7b1b19636af86a3c7995d4d028dbe376f39b4bf31541146f9c123582a6c94562 \ + --hash=sha256:7dfcad78ea1492ee3a9ec765cb7f51bbc17d477107aaf6b22abf7b2558d1c5a0 \ + --hash=sha256:8087e82f842609734c9b5a1330464f8e94e346ba0e18c832c08bafa4b0d63c15 \ + --hash=sha256:820fb8450edddae3812fd58cbc08e2bf22812cb248ecb5f06dbb82119a56e869 \ + --hash=sha256:8483c2096363120eea8b07c06ae7304d520f686665fffd4811fad423930a65d7 \ + --hash=sha256:84a2cef8deffbd9ab8ee0ea546a2a6a7030c28f44e6cdd4547dbfeb489eb8999 \ + --hash=sha256:86d7f0f8bdb25d2c632b72527325e4776430fd5bc61b9118de4e2b8ddb5f5b01 \ + --hash=sha256:8fe0b50da2d84535fb4f7b4bfa951280f97ce3d558a0443b541166d609e67b57 \ + --hash=sha256:90001d893bc368e302ef168d82130b4e4fdd27b85fa094682df9b667c2d48838 \ + --hash=sha256:9246a0d063cfcbcc85f2359dd6876d681213f4790832272aa16641b4ed5d64d4 \ + --hash=sha256:92b820d345f7a3fc7b8163949ee92df910f290c3fc517b3d5301c78065adafe1 \ + --hash=sha256:952303a7318d4cbe1011400839bb2051c9f84fa0a35923267f5daba34b15d458 \ + --hash=sha256:97fd3a0e8b53efa41970ac1dff3d8cf0d2884cadeb4caaf95db7ad1526926ee3 \ + --hash=sha256:9c1c5705e314449e3308872fe084b8571ce078ee4fc55a98a769bdefe5917392 \ + --hash=sha256:9c9f23004a3d40e89c01a7955d186a6cc83418d93b749701944ce2de3e95a1f3 \ + --hash=sha256:9f63bcef7f4b02b06b35fc01c93b96c43b5e88e1e8868676caacf493d5a31f3a \ + --hash=sha256:a0eadbbf2c30f01efa58e1f110eb6fa293261f6b0b1aa38f7f48707107690af9 \ + --hash=sha256:a28fcbc9b6baf54a2e23f8655f308e4ccc6afdd7266f8fe7954f320dcda0f785 \ + --hash=sha256:a6a61aff018180c9c50b7b0da33bfd29d378af3497429c95006c589a23a11648 \ + --hash=sha256:aabe464bfd13bd25f4821faf111da6fefdc389f870265a53105580e45b0a2e49 \ + --hash=sha256:ab59169ace05dcb49a1d4118f0bde139557adf45091bd85747e36bf5de984dd1 \ + --hash=sha256:b436f6ec4fc3a6b4237c84d3f83170ed2b40bb584222f0ac47a0c8a5921980c7 \ + --hash=sha256:b6b9dadbef0cccd9f4c4ee96b08898afa73e26803bbe0f6aeb5bb12b0074206d \ + --hash=sha256:b852788aa51764e2d8e4cf5493d559326bcae5e38d16ba25ffa322b034df272a \ + --hash=sha256:bae954c382e013d5ea5b190d2830526bfa45ad121c326da0049b8c769f185db6 \ + --hash=sha256:bcce07e23e5769375158f5efdcdafa8d5cd014b93c6683865b840ed65b96f231 \ + --hash=sha256:cc97814dfb786a83b6e2dc2e79351e1b83e6d715647d6887fcabd83026417a00 \ + --hash=sha256:cd2ca96a082a36964aca83e992f72abeb61b7306c1a6cba4c7d06a7b93750cac \ + --hash=sha256:cfb70b4eb56cac4da0a83588f3ad50d46beb0690391082f3d4e2d488c70b68ea \ + --hash=sha256:d0fcf657e9f13ff4b177960ab2200237b12994232dfb6df16f1cfe1d4339f93c \ + --hash=sha256:d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81 \ + --hash=sha256:d57685547e0060cc6fd90ee6a28405d6bd395e525545f13c8d7cd99c78afd79f \ + --hash=sha256:d6bec75c290fe484a8ba4cacdf838501e17c06ecfbbf31eede81a9e431bd7751 \ + --hash=sha256:d9531d9cbeac99af6f038fb1bc351403531f7d634a2c2e10e2f7c854c6ed5b68 \ + --hash=sha256:da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2 \ + --hash=sha256:dab9eb87869da2d6ed3af3f3adf28414baae6ec9d4df355ffc18889132f3436c \ + --hash=sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57 \ + --hash=sha256:dc0fad4933f427acd5b1cec210f3ea6dce7089e1724e4b9ec6ef47c6c04d1b3b \ + --hash=sha256:dc385593a42e31cd6fb60c19f0ecb015b386603818fc2c6c274fb42bd2bb4165 \ + --hash=sha256:dcc04fedf83effaeb9cce98abc9469bb1b42ef85f03e01c8c1f4438ef7555737 \ + --hash=sha256:e047dc87ef7ca50f4d309bf775ad4a71711c58556d75d7bd0604b2317f43e94b \ + --hash=sha256:e09f753a169951eb4f28c2c774f71069304f66e7277e0f5a2892423599cfa854 \ + --hash=sha256:ed5bb271084b46530ee2ddc0410537a9961152c5ccba2fc98c5276d992ccba87 \ + --hash=sha256:f0aa4aad3b1b69ad3fd85a0fd0952ec64331c762bd77ec51cc814170873890b2 \ + --hash=sha256:f17dbe07eb3ea7f99e4df9b7e0efefe80fbf30d37a8cc4d561a0aed310bc8847 \ + --hash=sha256:f2769a0344a09e9ccf5b3cce538bc75a51b53eff3275d3896310c8552049195d \ + --hash=sha256:f55f0b01956a094c8587146d9558c91937e78789c333860ffaf35931a6e5dbc4 \ + --hash=sha256:f5d497865f05bb222cab7016c6034542e84e5f29f49c6fd3f4939cda7197b5b8 \ + --hash=sha256:f70541f3104339f59f830522d94ebadb1bf47426287381623443d8bb1cdbf33d \ + --hash=sha256:fb9a0a6dc3d1b3986cb88091b6899f0396651e0f74e2c9766ab8d6ffc3842e29 \ + --hash=sha256:fce6c48559c86d1ac3632ccb1bebc7d5442fbe79bd9bb0e40379ee54be2a4051 \ + --hash=sha256:fd46fff7eb62c24804d234f0051c7a8ea81285ad63e0337d3dcf33ca82aee58a + # via esp-pylib +websockets==17.3 ; python_full_version >= '3.11' \ + --hash=sha256:018fab26140ce221c3853454a608e4e7004f4f10641cb97819dcd5b917bb2e99 \ + --hash=sha256:07246d4f0293ef0b87eff3d7dd88c1f03dd9f678fa39f6de87b2af8504e7e0de \ + --hash=sha256:078d1d41f45ee34eed5a4f3bbe7c588caecbdd6e12dfca73e8c1b0e803773793 \ + --hash=sha256:0824e6ecee838cc28e6481062574c7fde49f9a2cedfacd36f0e91cb4b37fd6fe \ + --hash=sha256:09f007b2150eff015b2109f006e2ac2012d376f41486defa88ae78a776b00cff \ + --hash=sha256:0a0bcf66289b9d017c175562c35c5304b5cc3ed56aded994ae30b0a9654ba405 \ + --hash=sha256:0a657b51e4a5f4ac577c11e706ff9a884ea5aaf3e2666bae283a251c63e66acb \ + --hash=sha256:0c63ad657695e8f1a48624754cb6b464118ac50d4c1d477c26461c98390f0a62 \ + --hash=sha256:0d20d6ea5aabbeba2e054d33e405192ef75dc12947bbd00ec97d23b14969d325 \ + --hash=sha256:0e65260a1b1d7b81651f47343d836de70b88b02358ae3658efa6382104ec37b1 \ + --hash=sha256:0f78bb2e82fe88912c9d7b25c98e57a9b08655579af89f164fd4c0657a075d62 \ + --hash=sha256:10b24e836ee0233ca078e1244f6ba40343a22127a9a736f46a7e7b18de2cc711 \ + --hash=sha256:13cffaedde6e06183860fc92d88fe1d190abda73837fe4c14f5d4803c472f5dd \ + --hash=sha256:13eace1b315345ebac5b27821f5dcce24e5cf9976ab18f79e00a59bdd67e5d19 \ + --hash=sha256:15decc0fe293d9e3f924bd483f3d76f3d0ff26845112a470e3f8842fec2c82a1 \ + --hash=sha256:1c7dd93a1c9f89297471493674b1fc1d9467cafd45374f2172213046bdb3a329 \ + --hash=sha256:1dae5cd6c92d4440b15f20aa2987baa73c47ea14baa0fec8536a542e82e0ab33 \ + --hash=sha256:1db2604e6aedb7a2cb2f6e0575d6c9a852ccac8239703a930ac83b63309177a5 \ + --hash=sha256:2129c9a5fc05a230ce088745a790af073e582b584badc7b05db067ee326f56d7 \ + --hash=sha256:23046098ebf844753cfd211033624a8247520e5bf2194c24cb1a8c222917acff \ + --hash=sha256:23cb6f6d64e3a389bb8ba150f947f9a6c8c0aca383e814ca3c5cffab738d5646 \ + --hash=sha256:26838e2d95d7004c52e8c7b6b51676c872093a3b55a497824f899116a15a8a55 \ + --hash=sha256:2789d203babb6ceb2770aadc2394c1f99263ec37fba57a93064f9a8ad899e8ff \ + --hash=sha256:28d1935c16782a1e3650c9baa79069b016fff739e292e4cb14bbb830d6930491 \ + --hash=sha256:29b01520c39ef2f23c88b002f11e6ef4165675f6c92f5f3578fd587d69b5a5fc \ + --hash=sha256:2a377c7132b1e69be7a50d7434d7f26f69e93c70166ebb6d8f8cabf802ac4506 \ + --hash=sha256:2b06213c8e1ce93b18d0476d7a5e647436c8f23d099888c9065e930e80f4bed5 \ + --hash=sha256:2c460aeb12c2bd1c204944d9e9199595bf9e505eb542018d304bf9e63cb60cad \ + --hash=sha256:2d08ca4bf5a9cdb6fd81b86b51bd5cfb10527d3a1199888f2b02cc6c6a3d5f80 \ + --hash=sha256:2d875224bac1a05a20acade268a7a0e3498e7f8cf890acb7c2b23bacd2bd3659 \ + --hash=sha256:2e99440ef446c7833690c1754e23b0188d440f3cb89812b0764c697a7880cb33 \ + --hash=sha256:2fa79b9a48ae59744316c71a34cb9f76f30c93e117338522c5135cb7eac10351 \ + --hash=sha256:32029441af953e980b9940f042b337fe4ad9a97fa8b17a4ed878ec67b5b89ef4 \ + --hash=sha256:33221e49be79d7bfe6d12bc52ef9458b256ab74fa8ac5660acfa3065de1c7870 \ + --hash=sha256:34c78052277053bcb09810ac8e9ba4029f9e3e277833fc18ec1c928a6246ac91 \ + --hash=sha256:35381b575f70a08d4241e73c0cde4672bb9b2a9aae8767ca6b7fb61305a052c8 \ + --hash=sha256:35fe35d20baa661cef4a12ee7bc42eaf2376d43ef9816ab29dd47ca957f7a73a \ + --hash=sha256:39f532fa0d9026de4b9315a175998646112776baad54d55348435c11b05d034e \ + --hash=sha256:3c4f631dc8d00e470cde3ec0a35d4b58a817bea592411956f1cf4099dbb306b3 \ + --hash=sha256:3d78d93846eddc990551c1c31914d90d71957ca3af468969826f7ed0d7d1d51c \ + --hash=sha256:411aa0fb234f42a1444dea1e177890ddfc577b36ffdfafec25cb0d2c818e1af8 \ + --hash=sha256:41665c7ea01d3ae497a0c61fd22d3424230d18782a5fceb79f04db9ccb2042c9 \ + --hash=sha256:4170556762c6fd2913d99dfdd8dbb4c927584899df7facb1034627e355263c45 \ + --hash=sha256:43f270a5f21e12db08c73d83849a877c02ab7b7d0fb6e2718b5101d9c810212f \ + --hash=sha256:443fa28652682d2a3ec3eca49af1b87cb2523561257683e62e03a98967b22138 \ + --hash=sha256:45cc1ccb12ced6d242335159e8221733c8696da5ad0b6ceef6debb0f5ce89110 \ + --hash=sha256:46ae05aa1aecd47d4d7de0f05ce2d1d2eb418f8de614964d754cc3b9a409c400 \ + --hash=sha256:49fb5b4960ebe1bc557b435da8687deef1302a08f030a33c5606fa6fb11b773b \ + --hash=sha256:4a9133dbaedc6f1af358127275676171d9a1bb3e61f47ee62b2ff9acb78af3d0 \ + --hash=sha256:4aa85b34c21cf97b3470793d0bf0763d32bfd1e67b83067a0a3ba8dc3474dacf \ + --hash=sha256:4d5f2492cb2192aab2d42e57f2caa2159b4fd20fde06f859ec13354f524b176c \ + --hash=sha256:4e7b5d23ef144b32645d97bd99f382c4394c443395f45fe1576786838d26e308 \ + --hash=sha256:4f7431e71072962d3ed1768038200aa4f1e76d9c2d7bedd554d57a55faaf6644 \ + --hash=sha256:508775efa11d26aec186d10154359f52276e2baf9ca201b52356330883276dfd \ + --hash=sha256:5142798871a248b422f353e0684d51dfa119de3403a6fa0904b7cb806d9790ce \ + --hash=sha256:52a2ca6f401fb49a8d5d9e0e48ade5f58f1bdda33132907b0e4e3b702628fa4e \ + --hash=sha256:53209dffd91b0aefed9ea630fa6ad5c2944d6210ba3a8a3a8664dc1575289b14 \ + --hash=sha256:534ed583fd4323de45d196535a444cebd1af2105e6e4e0cdcbf8cea266a8747a \ + --hash=sha256:54fcd24944e6d127d198c28dc363967b7131ff0c97833962c2ef2048018de02d \ + --hash=sha256:55494475206113dbaf731dc8c5ffd5e1094175b753b7bbaad09fe3d7df61165c \ + --hash=sha256:5670a762c7ae388bf8aa04199fd436180a4d28318dca9c1eae7b38604676dbfd \ + --hash=sha256:56a76556ab04ab54a675acce2cac8b4e74ea2cec558a7c9ed37e0367b04b5d50 \ + --hash=sha256:56d43c311f3519fb1057b6114dc24e9060bc3fb6aedf4aa11160484154b66b63 \ + --hash=sha256:5868a02c4f6d698ddd19b4c9742748c03dec045289915806845189d96bc3d314 \ + --hash=sha256:58e5d1e935a90878a935c7d2737a25c71cf4f1c13b6e6bef454ab722c4f338bb \ + --hash=sha256:5ad409b7ea102339041281117cf2dadfe71edbb653dc48037cc55d516fb7f448 \ + --hash=sha256:5af7070d80d278832cbca9c3c3dd44b30a26dcfa3ae1e0f6f19ff34edc27b689 \ + --hash=sha256:5d0c871efc696ef5eb8514df64b86ab28736db753ebb42a2101f3b562f4849c0 \ + --hash=sha256:5e0c0ed37472c4729f3001066510a656486f08f149dda1ebfd3dfe4450c9756e \ + --hash=sha256:5e1c174b366565e101f903f9d1cc96d5508f805759dd37d012743f1c73ec0d7b \ + --hash=sha256:62059d739af7fc035ed6e63c5390118a60a5e9aa5a5541cd0486a83e741842c7 \ + --hash=sha256:623d4b44f64efdf235b1e8dce266057073d93cde6197e7d7684f20c297f67f30 \ + --hash=sha256:628efe13630ac3397e7d1278b6be6f22f37a86d638a324334b4cd43c1daf9928 \ + --hash=sha256:6342f79d6f50e680c8aa59e0615d745e867e51ffd59c77093be60b7c433067af \ + --hash=sha256:63b87c93e28957e6dc2bb92d31727a863c9bf4a4daa080f0995fdcb0242c9166 \ + --hash=sha256:65e937576141ac50fac32fa9736dbb3e0e4a7769f91be004011a09216c6ab05f \ + --hash=sha256:66027e401c70d0cf55021b91b8a7ec2cd5810e20602f945b4fe6ae8b37ed232b \ + --hash=sha256:6644eecd99c4e8cdf72e646e17b391bb70b054f44becde37dcdc48cbec7e1527 \ + --hash=sha256:6705af9d1005127302c04d9f1533ba56e289336c465923ef878430e99295a0b9 \ + --hash=sha256:6a8248b1ba8861e48e4c7ced3895a1264d2a3b617de72ec94172ce6e64bf16c7 \ + --hash=sha256:6b14914e67044724c31a9360c7b89e9d070117c630746eefdc1620d967545acd \ + --hash=sha256:6b7b540bb321f7f0a7b8f5d107f941bb361e3a154c0b594dae0394997b91b743 \ + --hash=sha256:71250e010eede33c1d638e2c7a98288520d67f782fd5eceecaeca3129233a09a \ + --hash=sha256:722ab747ff5dc28070b1a8c020dba6466f13f2ab56bc251ad301fefa2454aa1e \ + --hash=sha256:724224d4c457207271239114bf43afafc3eca4a1b4661ccdd428d11869a03bbd \ + --hash=sha256:729989759feb81ff6493879a0d60a0bd7d2eee41001bbec0381a080318439382 \ + --hash=sha256:740bba4223fa5216d4d6c5dcd29e697f37229b8a61b69e1a50527b0ea993ab2c \ + --hash=sha256:7bdc658c9474864893e0b6b94f1390bf6a912952df1870dd93bb7d1a902b5628 \ + --hash=sha256:7d53fc4b90ef6e11de639dc18744d9be8cbd70ccb5ce80027aaffe7b1ab0215a \ + --hash=sha256:7fa69212ea60ea24e2d8aff23fe102b1f87f1b27a320f92d2e7b3468997a496a \ + --hash=sha256:80326bc571164bbbb6bead2e35d0c943b23b24fde629032197c58151529d24d8 \ + --hash=sha256:80578e91764bfca297ddca322c00a3136d993b829e9839e7fa378a19c1fcc25d \ + --hash=sha256:80a6e37e235780f189558ab804f3cbc01c18f5c578995739b49613d544d34ebf \ + --hash=sha256:8135a549e27e9941a70e2d2156676ef2ffeff4b06e9d564dbaaae963482afdef \ + --hash=sha256:82e60c00cfb8493a5ba27cdf9b3695fdd153f666f73e11f18968f78ead1d8705 \ + --hash=sha256:856bbb24f2ee6436af7609267e73309e4bb0a0f9982f79f1c2e01019279c6671 \ + --hash=sha256:85d6daa3a7ca404570ae130babbeb347ef9d3faca5300fb13cfa4bb48c3d30d1 \ + --hash=sha256:8879b6a4040ee925c638548238ab7fe612cf5fa8752fe847346da0da8be81c3f \ + --hash=sha256:8b3bede7d583bc9f47d8693a29984882e59077db1a5238ad977fc70ae8169741 \ + --hash=sha256:8dbb3862aa649de95660b6337d97101977f160b5b354d07b82fe309874d2c8bc \ + --hash=sha256:8f244dfee05ea29b8064701880865eb1684f3a704a5665bd10b64694444dbbd4 \ + --hash=sha256:8f7469408740cf06a79b01605f2ae0b2acc8839c03fcbbf05da49baee12d3302 \ + --hash=sha256:90cc9973c51811918247a78d9ab2f54871296e4e37e69a98aba4d3c9acedbf62 \ + --hash=sha256:90d8c4fa5ba293cf92de351fbc292cdaadb1a1d58d031a1a97ff9b8fb6250ec7 \ + --hash=sha256:94c02b0702656f083c098bdbd93bce22a7aa7a2b34ba56d2472efb67dab7cf58 \ + --hash=sha256:9662a9fe457820c60365b3209e03cbd8e4a817195d55de65a80fa680b4bea136 \ + --hash=sha256:997617dd7856c5dad742bf6893b872319156dd43f47587541a59e4a852d52bbc \ + --hash=sha256:99c5637b74e2805ee4de9e2a43c8b45f678e50abb1c2da9880f0372f6498ba2f \ + --hash=sha256:9a215749c9a88ef2842e880215193e56ae60b856773c99702b078236bc6d4adb \ + --hash=sha256:9bde5d82d7ac4859e0cb3a35e148f537a8d4e56aa43f42582b5f263982f095ae \ + --hash=sha256:a004c1fe5a72c496f28c6c1d8c2dc5cfe3278d99478e18f8b838c70a912b366f \ + --hash=sha256:a0e2e7c2c1a4f25db7b6596738d810c10732e77f852247300a1bc49e364d6dc9 \ + --hash=sha256:a22c4956a4d858200dba83f0691bb692d77b0eeddfca80157820d20b85dea4f1 \ + --hash=sha256:a7f391838e8adcf1f2fe71561d9e5a9f74ea7ffa5052eb2db329aac677383136 \ + --hash=sha256:a9eacb382f400bb5865e7e6d24bbe709a84510291e707f169dd4a24ee4a5ac6a \ + --hash=sha256:ace8dd1efa5f44c482d091aa38322e20e3ffe5adcdba6b67ad9f4e7da58f3f68 \ + --hash=sha256:ade76f8c84ade325bbed96806357fffd353fcc29a5166d6dc6d39381f6d9f76c \ + --hash=sha256:af37a1e6b6bd506b597d5fb4cf111fb527d1b2223608a5d441d4738827cfd166 \ + --hash=sha256:af7c474f0d8f4d2a196370e85033873c7b063a0b5da9a4f143aa023e3bd25407 \ + --hash=sha256:b225c656efa2dedecc81651d2290170af25887b7de4f38c5efc7c5ae2b9442ab \ + --hash=sha256:b459d29c8b82a420a98fa0ce8e9cf787ce0062d98cc90604ef9574fb3cc26ff8 \ + --hash=sha256:bc6f87d03e3691154673aaa8f08991723d6e3394c83ef2875c7a9049eac9595c \ + --hash=sha256:bc8fd230ef013951900c1fda100eebe4a4b1b407dde2a44186995042dda2c628 \ + --hash=sha256:bd645b5d2f810a181c44f4a71135e7c365b44f4be9ed3d1892635be13a857553 \ + --hash=sha256:be7be57097a65a8fab3d0f0a10d12931deb626a7d8082eb732b1ae41f3a202e3 \ + --hash=sha256:c15a165673bb3fb57facbf040edf5c0556d20ccd2e0ccc5f23ebb69fb715fdc1 \ + --hash=sha256:c4b3f2c7d84f303da873b634d0beef83b5e1137294c3de4e23102841dac0cf4a \ + --hash=sha256:c534c9524a2acd0b97b7022074bec3784ac273c683cf9535087e437672ea8531 \ + --hash=sha256:c95d25b5382ee0e6191ecafdeabefdc1f7bff8dfee29239b9b79e760ef3b04bd \ + --hash=sha256:c9d06edfd974e918b40e0662a7e76dac13125529ee5ec9fbe42dc0d6faa209c3 \ + --hash=sha256:cdb8e4b0f6a41b857c74c03116e8ffbef59ef8e533e934b532843178990d01d5 \ + --hash=sha256:ce39283ef08b0f28478e920d4d89a1fc169176e568e8a6a821332673145522eb \ + --hash=sha256:cfef7f7aa434a1e632e0ebda14b39f4f1d7975e0fce5c77ea7b43156fabf9bd8 \ + --hash=sha256:d2605a3b2af8d66f8b75312d223642c065ec773446c04566d9030ad5e2db13c9 \ + --hash=sha256:d338813a440bf6d2f0df55140d3f3080284410b1ebfe9df50e3847bcdb1a17bb \ + --hash=sha256:d355b22270be8062360b9df2ee0b10277590789a1059257916fb9e256a0cff80 \ + --hash=sha256:d68da281ac208b6236ffa8beef23fe7c7c956d8bf09b421d447232fd94441b35 \ + --hash=sha256:dafbb58a149d1ebd5767b3f064f911c5ee0897608005e51c3904b0ada2a8d99c \ + --hash=sha256:dc2b495a92b3971966bf102e73589f950db3ec1ca6713265355ce7771d97b199 \ + --hash=sha256:dd0f3037d501178a761fd2814370aca547f79e99a200a61a1f6e7b79ec3d78b9 \ + --hash=sha256:de2c12555bf05546f3ee75733f2c81de70aab7f530e86d0d2162b0ddcb3c39ae \ + --hash=sha256:e164c4a6868d11988c7f2386e3278ccff21c665cd7d08279b62709dd37a3167c \ + --hash=sha256:ed59cb4ee258674a6ed9b8515e42079db02cb3be94f133fdd3f200183a40c49d \ + --hash=sha256:ef07e0b6b879f699a16fc7a513b974246fdeaef44908a021130cf4afe6a1c89c \ + --hash=sha256:ef2950896673a9809e8e5ab857525924fbcacfc98b858b0509ccf50d341f80f4 \ + --hash=sha256:f09f145b89ea292cf6c25fd7c8c0d90207fd6a9e54c7828499669996a1e5bdcc \ + --hash=sha256:f27e7e8a85512b6a4b490d5756edde8e6772883a5800e1476172e44eda3f7d7c \ + --hash=sha256:f300f28dff74ff4148c21e6971e4de6ca0c34e0086a3d95c017c5174ebf85ca5 \ + --hash=sha256:f3ed747b7651668131029c720fbc32c7ee2d129bdad200e6c8f4037b20f7913a \ + --hash=sha256:f87f02afe05c1c246012e3de100a77cfbabd17db709fe93cf074271ab239b312 \ + --hash=sha256:fa87ea2ad64e59fe5622c1dc5716106b6d6e7e4b26b848cb5f5710a75c292ef5 \ + --hash=sha256:fc23e72bd7f8a8e540fed2244524d0a9aeb6ae3325f3d56f04a7a3debe5b04c2 \ + --hash=sha256:fdd00937f15f5f93c8e53d8060a8c4c010431ab595a9b5b50ddd569a04603944 \ + --hash=sha256:fe7a168d805c8a7bf1c2de55b7ab2bc958586bfd88d0e87235a92cac66b590a7 + # via esp-pylib diff --git a/scripts/build.py b/scripts/build.py index 3995fbac5..a0b45823e 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -16,17 +16,16 @@ python scripts/build.py --variant esp32s3-6832dbd5 app bootloader partition-table """ +import argparse import os import shutil import subprocess import sys from pathlib import Path -from utils.boards import board_fragment, find_variant, kconfig_lines, read_board_fragment +from utils.boards import ROOT, board_fragment, find_variant, kconfig_lines, read_board_fragment from utils.files import write_if_changed -ROOT = Path(__file__).resolve().parent.parent - def resolve_build_dir(name: str) -> Path: """``build/``, or OPENSHOCK_BUILD_DIR. CI fixes the path: it is on the compiler command line, so it is in ccache's hash.""" @@ -37,44 +36,24 @@ def resolve_build_dir(name: str) -> Path: return ROOT / 'build' / name -def main(argv: list[str]) -> int: - if not argv or argv[0] == '--variant' and len(argv) < 2: - raise SystemExit(__doc__) +def variant_inputs(variant_id: str) -> tuple[str, str, Path, Path, str]: + """Returns (target, board, build_dir, fragment path, fragment body); board is empty: a variant is not stamped.""" + variant = find_variant(variant_id) + build_dir = resolve_build_dir(variant.id) + body = f'# Build variant {variant.id} ({", ".join(variant.boards)})\n' + variant.sdkconfig_fragment() + return variant.target, '', build_dir, build_dir / 'variant.defaults', body - if argv[0] == '--variant': - variant = find_variant(argv[1]) - targets = argv[2:] or ['build'] - target, board = variant.target, '' - build_dir = resolve_build_dir(variant.id) - frag = build_dir / 'variant.defaults' - body = f'# Build variant {variant.id} ({", ".join(variant.boards)})\n' + variant.sdkconfig_fragment() - else: - board = argv[0] - targets = argv[1:] or ['build'] - fragment = board_fragment(board) - target = read_board_fragment(fragment).target # parsed whole, so a typo fails now rather than at the stamp - build_dir = resolve_build_dir(board) - frag = build_dir / 'board.defaults' - # Only the Kconfig lines: the OPENSHOCK_* pins are stamped, and Kconfig would reject them. - body = f'# Board {board}, without its pins\n' + '\n'.join(kconfig_lines(fragment)) + '\n' - # Generated, and written only on change so an unchanged fragment does not force a reconfigure. - write_if_changed(frag, '# Generated by scripts/build.py - do not edit.\n' + body) - # Relative to the project where it can be; ESP-IDF takes an absolute entry too, for a build dir outside the repo. - frag_rel = (frag.relative_to(ROOT) if frag.is_relative_to(ROOT) else frag).as_posix() - idf_args = [ - '-C', - str(ROOT), - '-B', - str(build_dir), - f'-DIDF_TARGET={target}', - f'-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;{frag_rel}', - f'-DSDKCONFIG={build_dir / "sdkconfig"}', - # Empty for a variant, which also clears a board an earlier configure cached. - f'-DOPENSHOCK_BOARD={board}', - *targets, - ] +def board_inputs(board: str) -> tuple[str, str, Path, Path, str]: + fragment = board_fragment(board) + target = read_board_fragment(fragment).target # parsed whole, so a typo fails now rather than at the stamp + build_dir = resolve_build_dir(board) + # Only the Kconfig lines: the OPENSHOCK_* pins are stamped, and Kconfig would reject them. + body = f'# Board {board}, without its pins\n' + '\n'.join(kconfig_lines(fragment)) + '\n' + return target, board, build_dir, build_dir / 'board.defaults', body + +def run_idf(idf_args: list[str]) -> int: # An idf.py wrapper on PATH (install-esp-idf-action, an activated ESP-IDF shell) sets up its own # environment. On Windows a bare idf.py is not executable, so run tools/idf.py with this interpreter. idf_wrapper = shutil.which('idf.py') if os.name == 'posix' else None @@ -90,5 +69,39 @@ def main(argv: list[str]) -> int: return subprocess.call(cmd) +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--variant', help='build this variant (one unstamped compile) instead of a board') + ap.add_argument('args', nargs=argparse.REMAINDER, help=' (unless --variant), then idf.py targets/args') + args = ap.parse_args(argv) + + if args.variant: + targets = args.args or ['build'] + target, board, build_dir, frag, body = variant_inputs(args.variant) + else: + if not args.args: + ap.error('a board or --variant is required') + targets = args.args[1:] or ['build'] + target, board, build_dir, frag, body = board_inputs(args.args[0]) + # Generated, and written only on change so an unchanged fragment does not force a reconfigure. + write_if_changed(frag, '# Generated by scripts/build.py - do not edit.\n' + body) + # Relative to the project where it can be; ESP-IDF takes an absolute entry too, for a build dir outside the repo. + frag_rel = (frag.relative_to(ROOT) if frag.is_relative_to(ROOT) else frag).as_posix() + + return run_idf( + [ + '-C', + str(ROOT), + '-B', + str(build_dir), + f'-DIDF_TARGET={target}', + f'-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;{frag_rel}', + f'-DSDKCONFIG={build_dir / "sdkconfig"}', + f'-DOPENSHOCK_BOARD={board}', # empty for a variant, which also clears a board an earlier configure cached + *targets, + ] + ) + + if __name__ == '__main__': - raise SystemExit(main(sys.argv[1:])) + raise SystemExit(main()) diff --git a/scripts/fetch_flatc.py b/scripts/fetch_flatc.py index d2b9b56f7..82747fcd3 100644 --- a/scripts/fetch_flatc.py +++ b/scripts/fetch_flatc.py @@ -6,7 +6,6 @@ fetch_flatc.py fetch flatc matching the submodule fetch_flatc.py --upgrade move the submodule to the latest flatbuffers release, then fetch its flatc - (aliases: --update, --latest) Each download is checked against the SHA-256 digest GitHub publishes for the release asset. """ @@ -22,7 +21,8 @@ import requests -ROOT = Path(__file__).resolve().parent.parent +from utils.boards import ROOT + SUBMODULE = ROOT / 'components' / 'flatbuffers' / 'flatbuffers' BASE_H = SUBMODULE / 'include' / 'flatbuffers' / 'base.h' @@ -112,13 +112,13 @@ def fetch_binaries(release: dict, output_dir: Path) -> None: print(f'Extracted {asset["name"]} to {output_dir}') -def main(): +def main(argv=None) -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('output_dir', nargs='?', default=str(Path(__file__).resolve().parent), help='directory to extract flatc into (default: scripts/)') - parser.add_argument('--upgrade', '--update', '--latest', dest='upgrade', action='store_true', + parser.add_argument('--upgrade', dest='upgrade', action='store_true', help='move the flatbuffers submodule to the latest release and fetch its flatc') - args = parser.parse_args() + args = parser.parse_args(argv) output_dir = Path(args.output_dir).resolve() output_dir.mkdir(parents=True, exist_ok=True) @@ -141,7 +141,8 @@ def main(): if args.upgrade: print(f'\nflatbuffers is now at {release["tag_name"]}. Next: run scripts/generate_schemas.py so the generated code ' 'matches, then commit the submodule bump together with the regenerated sources.') + return 0 if __name__ == '__main__': - main() + raise SystemExit(main()) diff --git a/scripts/gen_dep_graph.py b/scripts/gen_dep_graph.py index e2dc077d5..10a75d229 100644 --- a/scripts/gen_dep_graph.py +++ b/scripts/gen_dep_graph.py @@ -19,11 +19,12 @@ """ import argparse +import heapq import re import sys from pathlib import Path -ROOT = Path(__file__).resolve().parent.parent +from utils.boards import ROOT # idf_component_register() argument keywords. A token equal to one of these ends # the current value list and (for REQUIRES/PRIV_REQUIRES) starts a new one. @@ -91,53 +92,43 @@ def parse_requires(cmake_text: str, include_priv: bool) -> list[str]: tokens = block.split() deps: list[str] = [] - active = None # None | 'collect' + active = False for tok in tokens: if tok in CMAKE_KEYWORDS: - if tok == 'REQUIRES' or (tok == 'PRIV_REQUIRES' and include_priv): - active = 'collect' - else: - active = None + active = tok == 'REQUIRES' or (tok == 'PRIV_REQUIRES' and include_priv) continue - if active == 'collect': + if active: deps.append(tok) - # de-dupe while preserving order - seen = set() - ordered = [] - for d in deps: - if d not in seen: - seen.add(d) - ordered.append(d) - return ordered + return list(dict.fromkeys(deps)) def topo_order(nodes: set[str], edges: set[tuple[str, str]]) -> list[str]: """Dependencies first. Edge (dep, dependent). Alphabetical tie-break; any nodes left in a cycle are appended in sorted order so output stays total.""" indeg = {n: 0 for n in nodes} - adj = {n: [] for n in nodes} + adj: dict[str, list[str]] = {n: [] for n in nodes} for dep, dependent in edges: adj[dep].append(dependent) indeg[dependent] += 1 - ready = sorted(n for n in nodes if indeg[n] == 0) + ready = [n for n in nodes if indeg[n] == 0] + heapq.heapify(ready) order = [] while ready: - n = ready.pop(0) + n = heapq.heappop(ready) order.append(n) for m in sorted(adj[n]): indeg[m] -= 1 if indeg[m] == 0: - ready.append(m) - ready.sort() + heapq.heappush(ready, m) if len(order) < len(nodes): order.extend(sorted(nodes - set(order))) return order -def main(argv: list[str]) -> int: +def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--components-dir', default=str(ROOT / 'components'), @@ -166,8 +157,7 @@ def main(argv: list[str]) -> int: nodes: set[str] = set(local) for name, deps in local.items(): for dep in deps: - is_local = dep in local - if not is_local and not args.external: + if dep not in local and not args.external: continue edges.add((dep, name)) # dependency -> dependent nodes.add(dep) @@ -194,4 +184,4 @@ def main(argv: list[str]) -> int: if __name__ == '__main__': - raise SystemExit(main(sys.argv[1:])) + raise SystemExit(main()) diff --git a/scripts/gen_env_header.py b/scripts/gen_env_header.py index 255a6dd30..df73789da 100644 --- a/scripts/gen_env_header.py +++ b/scripts/gen_env_header.py @@ -53,20 +53,48 @@ def git_commit() -> str | None: return commit try: return subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True, stderr=subprocess.DEVNULL).strip() - except Exception: + except (OSError, subprocess.CalledProcessError): return None -def resolve_version() -> str: +def resolve_version(commit: str | None) -> str: version = os.environ.get('OPENSHOCK_FW_VERSION') if version: return version if github.is_ci(): raise SystemExit('error: OPENSHOCK_FW_VERSION must be set for CI builds') - commit = git_commit() return '0.0.0-local' + (f'+{commit[:7]}' if commit else '') +def board_defines(chip: str) -> dict[str, int | str]: + mode, log_level = github.build_mode() + chip_macro = chip.upper().replace('-', '').replace('_', '') # ESP32-S3 -> ESP32S3 + return { + 'OPENSHOCK_FW_CHIP': chip, + f'OPENSHOCK_FW_CHIP_{chip_macro}': 1, + 'OPENSHOCK_FW_MODE': mode, + 'OPENSHOCK_LOG_LEVEL': log_level, + } + + +def version_defines(commit: str | None) -> dict[str, int | str]: + version = resolve_version(commit) + major, minor, patch, prerelease, build = split_semver(version) + defines: dict[str, int | str] = { + 'OPENSHOCK_FW_VERSION': version, + 'OPENSHOCK_FW_VERSION_MAJOR': major, + 'OPENSHOCK_FW_VERSION_MINOR': minor, + 'OPENSHOCK_FW_VERSION_PATCH': patch, + } + if prerelease is not None: + defines['OPENSHOCK_FW_VERSION_PRERELEASE'] = prerelease + if build is not None: + defines['OPENSHOCK_FW_VERSION_BUILD'] = build + if commit is not None: + defines['OPENSHOCK_FW_GIT_COMMIT'] = commit + return defines + + def render(title: str, defines: dict[str, int | str]) -> str: lines = ['// Generated by scripts/gen_env_header.py - do not edit.', f'// {title}', '#pragma once', ''] lines += [f'#define {key} {serialize(defines[key])}' for key in sorted(defines)] @@ -85,33 +113,10 @@ def main(argv=None) -> int: args = ap.parse_args(argv) chip = boards.chip_for_target(args.target) - mode, log_level = github.build_mode() - board = { - 'OPENSHOCK_FW_CHIP': chip, - 'OPENSHOCK_FW_CHIP_' + chip.upper().replace('-', '').replace('_', ''): 1, - 'OPENSHOCK_FW_MODE': mode, - 'OPENSHOCK_LOG_LEVEL': log_level, - } - - version = resolve_version() - major, minor, patch, prerelease, build = split_semver(version) - version_defines: dict[str, int | str] = { - 'OPENSHOCK_FW_VERSION': version, - 'OPENSHOCK_FW_VERSION_MAJOR': major, - 'OPENSHOCK_FW_VERSION_MINOR': minor, - 'OPENSHOCK_FW_VERSION_PATCH': patch, - } - if prerelease is not None: - version_defines['OPENSHOCK_FW_VERSION_PRERELEASE'] = prerelease - if build is not None: - version_defines['OPENSHOCK_FW_VERSION_BUILD'] = build - commit = git_commit() - if commit is not None: - version_defines['OPENSHOCK_FW_GIT_COMMIT'] = commit - - headers = [('openshock_version.h', 'Firmware version and git commit. Changes every commit.', version_defines)] + headers = [] if not args.version_only: - headers.insert(0, ('openshock_board.h', 'Chip identity, build mode and log level.', board)) + headers.append(('openshock_board.h', 'Chip identity, build mode and log level.', board_defines(chip))) + headers.append(('openshock_version.h', 'Firmware version and git commit. Changes every commit.', version_defines(git_commit()))) out_dir = Path(args.out_dir) written = [name for name, title, defines in headers if write_if_changed(out_dir / name, render(title, defines))] diff --git a/scripts/gen_staticfs.py b/scripts/gen_staticfs.py index 3d79119b2..affbb3fec 100644 --- a/scripts/gen_staticfs.py +++ b/scripts/gen_staticfs.py @@ -2,7 +2,7 @@ """Build the captive-portal static filesystem image (native ESP-IDF). Pipeline: - 1. Build the SvelteKit frontend (pnpm) when its sources are newer than the last build (never in CI). + 1. Build the SvelteKit frontend (pnpm) when its sources are newer than the last build (unless --skip-frontend-build, which CI passes). 2. Gzip every built asset into /www/.gz. 3. Pack /www into a littlefs image sized for the `static0` partition. @@ -17,7 +17,6 @@ import os import shutil import subprocess -import sys from pathlib import Path from utils import partitions @@ -27,41 +26,41 @@ BLOCK_SIZE = 4096 READ_SIZE = PROG_SIZE = 128 CACHE_SIZE = 512 - - -def log(msg): - print(f'[gen_staticfs] {msg}', flush=True) - +LOOKAHEAD_SIZE = 128 # Inputs of the frontend build; a change to any of them makes frontend/build stale. FRONTEND_INPUTS = ('src', 'static', 'packages', 'index.html', 'package.json', 'pnpm-lock.yaml', 'vite.config.ts', 'tsconfig.json') FRONTEND_IGNORED_DIRS = {'node_modules', 'build', 'dist', '.svelte-kit'} +def log(msg): + print(f'[gen_staticfs] {msg}', flush=True) + + def newest_mtime(paths): newest = 0.0 for path in paths: - if os.path.isfile(path): - newest = max(newest, os.path.getmtime(path)) + if path.is_file(): + newest = max(newest, path.stat().st_mtime) continue for root, dirs, files in os.walk(path): dirs[:] = [d for d in dirs if d not in FRONTEND_IGNORED_DIRS] for name in files: - newest = max(newest, os.path.getmtime(os.path.join(root, name))) + newest = max(newest, (Path(root) / name).stat().st_mtime) return newest def frontend_build_is_current(frontend_dir, build_dir): - index = os.path.join(build_dir, 'index.html') - if not os.path.isfile(index): + index = build_dir / 'index.html' + if not index.is_file(): return False - inputs = [os.path.join(frontend_dir, p) for p in FRONTEND_INPUTS if os.path.exists(os.path.join(frontend_dir, p))] - return newest_mtime(inputs) <= os.path.getmtime(index) + inputs = [frontend_dir / p for p in FRONTEND_INPUTS if (frontend_dir / p).exists()] + return newest_mtime(inputs) <= index.stat().st_mtime def build_frontend(frontend_dir): """Run the SvelteKit production build when frontend/build is missing or older than its sources.""" - build_dir = os.path.join(frontend_dir, 'build') + build_dir = frontend_dir / 'build' if frontend_build_is_current(frontend_dir, build_dir): log('frontend/build is up to date, skipping pnpm') return build_dir @@ -75,32 +74,27 @@ def build_frontend(frontend_dir): def stage_assets(build_dir, staging_dir): """Mirror build_dir into /www, gzipping everything that isn't already .gz.""" - www_dir = os.path.join(staging_dir, 'www') - if os.path.exists(www_dir): + www_dir = staging_dir / 'www' + if www_dir.exists(): shutil.rmtree(www_dir) count = 0 for root, dirs, files in os.walk(build_dir): dirs.sort() - rel = os.path.relpath(root, build_dir) - dst_root = www_dir if rel == '.' else os.path.join(www_dir, rel) - os.makedirs(dst_root, exist_ok=True) + dst_root = www_dir / Path(root).relative_to(build_dir) + dst_root.mkdir(parents=True, exist_ok=True) for name in sorted(files): - src = os.path.join(root, name) + src = Path(root) / name if name.endswith('.gz'): - dst = os.path.join(dst_root, name) - shutil.copyfile(src, dst) + shutil.copyfile(src, dst_root / name) else: - dst = os.path.join(dst_root, name + '.gz') - # mtime=0 and no filename in the header: the gzip bytes depend only on the content - with open(src, 'rb') as f_in, open(dst, 'wb') as raw_out: - with gzip.GzipFile(filename='', mode='wb', fileobj=raw_out, mtime=0) as f_out: - f_out.write(f_in.read()) + # mtime=0: the gzip bytes depend only on the content + (dst_root / (name + '.gz')).write_bytes(gzip.compress(src.read_bytes(), mtime=0)) count += 1 log(f'Staged {count} gzipped assets into {www_dir}') - if not os.path.exists(os.path.join(www_dir, 'index.html.gz')): + if not (www_dir / 'index.html.gz').exists(): raise SystemExit(f'{www_dir}/index.html.gz missing: frontend build produced no index.html') return www_dir @@ -110,7 +104,7 @@ def pack_image(staging_dir, output, partition_size): try: from littlefs import LittleFS except ImportError: - raise SystemExit("littlefs-python is required (pip install -r requirements.txt)") + raise SystemExit('littlefs-python is required (pip install --require-hashes -r requirements.txt)') from None if partition_size % BLOCK_SIZE != 0: raise SystemExit(f'partition size {partition_size} is not a multiple of block size {BLOCK_SIZE}') @@ -121,31 +115,28 @@ def pack_image(staging_dir, output, partition_size): read_size=READ_SIZE, prog_size=PROG_SIZE, cache_size=CACHE_SIZE, - lookahead_size=128, + lookahead_size=LOOKAHEAD_SIZE, ) # Only www/ is packed, so stray files in the staging directory never end up in the image. file_count = 0 - for root, dirs, files in os.walk(os.path.join(staging_dir, 'www')): + for root, dirs, files in os.walk(staging_dir / 'www'): dirs.sort() - rel_root = os.path.relpath(root, staging_dir).replace(os.sep, '/') + rel_root = Path(root).relative_to(staging_dir).as_posix() fs.makedirs(rel_root, exist_ok=True) for name in sorted(files): - rel_path = name if rel_root == '.' else f'{rel_root}/{name}' - with open(os.path.join(root, name), 'rb') as f_in: - with fs.open(rel_path, 'wb') as f_out: - f_out.write(f_in.read()) + with fs.open(f'{rel_root}/{name}', 'wb') as f_out: + f_out.write((Path(root) / name).read_bytes()) file_count += 1 - os.makedirs(os.path.dirname(os.path.abspath(output)), exist_ok=True) - with open(output, 'wb') as f: - f.write(bytes(fs.context.buffer)) + output.resolve().parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(bytes(fs.context.buffer)) log(f'Packed {file_count} files into {output} ' f'({partition_size} bytes, {block_count} x {BLOCK_SIZE}-byte blocks)') -def main(): +def main(argv=None) -> int: p = argparse.ArgumentParser(description='Build the OpenShock static filesystem image') p.add_argument('--frontend-dir', required=True, help='Path to the frontend/ project') p.add_argument('--staging-dir', required=True, help='Where to stage gzipped assets (contains www/)') @@ -155,14 +146,15 @@ def main(): size.add_argument('--partition-csv', help='partition table CSV to read the static0 size from') p.add_argument('--skip-frontend-build', action='store_true', help='Reuse existing frontend/build without invoking pnpm') - args = p.parse_args() + args = p.parse_args(argv) + frontend_dir, staging_dir, output = Path(args.frontend_dir), Path(args.staging_dir), Path(args.output) if args.skip_frontend_build: - build_dir = os.path.join(args.frontend_dir, 'build') - if not os.path.isdir(build_dir): + build_dir = frontend_dir / 'build' + if not build_dir.is_dir(): raise SystemExit(f'--skip-frontend-build set but {build_dir} does not exist') else: - build_dir = build_frontend(args.frontend_dir) + build_dir = build_frontend(frontend_dir) partition_size = args.partition_size if partition_size is None: @@ -171,9 +163,10 @@ def main(): raise SystemExit(f'{args.partition_csv} has no static0 partition') partition_size = table['static0']['size'] - stage_assets(build_dir, args.staging_dir) - pack_image(args.staging_dir, args.output, partition_size) + stage_assets(build_dir, staging_dir) + pack_image(staging_dir, output, partition_size) + return 0 if __name__ == '__main__': - sys.exit(main()) + raise SystemExit(main()) diff --git a/scripts/generate_schemas.py b/scripts/generate_schemas.py index 045dac34a..51531375a 100644 --- a/scripts/generate_schemas.py +++ b/scripts/generate_schemas.py @@ -1,27 +1,30 @@ -import os +#!/usr/bin/env python3 +"""Generate the TypeScript and C++ flatbuffers sources from schemas/ with flatc.""" + import shutil import subprocess import sys import tempfile +from pathlib import Path + +from utils.boards import ROOT -def flatc_works(path: str) -> bool: +def flatc_works(path: str | Path) -> bool: try: - return subprocess.run([path, '--version'], capture_output=True).returncode == 0 - except OSError: + subprocess.run([str(path), '--version'], capture_output=True, check=True) + return True + except (OSError, subprocess.CalledProcessError): return False def get_flatc_path(): - script_dir = os.path.dirname(os.path.realpath(__file__)) - - # Pick the right binary name for the platform. flatc_name = 'flatc.exe' if sys.platform == 'win32' else 'flatc' # Check in the working directory, then the scripts directory. - for directory in [os.getcwd(), script_dir]: - path = os.path.join(directory, flatc_name) - if os.path.exists(path) and flatc_works(path): + for directory in [Path.cwd(), Path(__file__).resolve().parent]: + path = directory / flatc_name + if path.exists() and flatc_works(path): return path # Fall back to whatever is on PATH. @@ -31,78 +34,59 @@ def get_flatc_path(): return None -def resolve_path(path): - script_path = os.path.dirname(os.path.realpath(__file__)) - return os.path.normpath(os.path.join(script_path, path)) - - def generate(flatc_path, language_args, schema_files, output_path, label): """Generate into a temporary directory and only replace output_path once flatc succeeded, so a failing flatc never leaves the tree without generated sources.""" - parent = os.path.dirname(output_path) - os.makedirs(parent, exist_ok=True) + output_path.parent.mkdir(parents=True, exist_ok=True) - tmp = tempfile.mkdtemp(dir=parent) + tmp = tempfile.mkdtemp(dir=output_path.parent) try: print(f'Compiling schemas for {label}') - result = subprocess.run([flatc_path, *language_args, '-o', tmp, *schema_files]) - if result.returncode != 0: - sys.exit(f'flatc failed for {label} (exit code {result.returncode}); {output_path} left unchanged') + try: + subprocess.run([str(flatc_path), *language_args, '-o', tmp, *schema_files], check=True) + except subprocess.CalledProcessError as err: + sys.exit(f'flatc failed for {label} (exit code {err.returncode}); {output_path} left unchanged') - if os.path.exists(output_path): + if output_path.exists(): print(f'Replacing old {label} schemas') shutil.rmtree(output_path) shutil.move(tmp, output_path) tmp = None finally: - if tmp is not None and os.path.exists(tmp): + if tmp is not None and Path(tmp).exists(): shutil.rmtree(tmp) -def main(): +def main() -> int: flatc_path = get_flatc_path() if flatc_path is None: sys.exit('flatc not found. Please install flatc and make sure it is in your PATH.') - schemas_path = resolve_path('../schemas') - ts_output_path = resolve_path('../frontend/src/lib/_fbs') - cpp_output_path = resolve_path('../components/serialization/include/serialization/_fbs') + schemas_path = ROOT / 'schemas' + ts_output_path = ROOT / 'frontend' / 'src' / 'lib' / '_fbs' + cpp_output_path = ROOT / 'components' / 'serialization' / 'include' / 'serialization' / '_fbs' - # Get all the schema files. - schema_files = [] - for root, _, files in os.walk(schemas_path): - for filename in sorted(files): - if filename.endswith('.fbs') and not filename.startswith('Deprecated'): - schema_files.append(os.path.join(root, filename)) + schema_files = [str(p) for p in sorted(schemas_path.rglob('*.fbs')) if not p.name.startswith('Deprecated')] if not schema_files: sys.exit(f'No .fbs schemas found in {schemas_path}') - ts_args = [ - # Compile for TypeScript. - '--ts', - # Don't add .ts extension to imports. - '--ts-no-import-ext', - ] + ts_args = ['--ts', '--ts-no-import-ext'] cpp_args = [ - # Compile for C++. '--cpp', - # Don't prefix enum values in generated C++ by their enum type. '--no-prefix', - # Use C++11 style scoped and strongly typed enums in generated C++. This also implies --no-prefix. '--scoped-enums', - # Generate type name functions for C++. '--gen-name-strings', - # Don't construct custom string types by passing std::string from Flatbuffers, but (char* + length). This allows efficient construction of custom string types, including zero-copy construction. + # (char* + length) string construction instead of std::string, for efficient custom string types. '--cpp-str-flex-ctor', - # use C++17 features in generated code (experimental). '--cpp-std', 'c++17', ] generate(flatc_path, ts_args, schema_files, ts_output_path, 'TypeScript') generate(flatc_path, cpp_args, schema_files, cpp_output_path, 'C++') + return 0 if __name__ == '__main__': - main() + raise SystemExit(main()) diff --git a/scripts/merge_image.py b/scripts/merge_image.py index e507994e2..2005fc81c 100644 --- a/scripts/merge_image.py +++ b/scripts/merge_image.py @@ -36,7 +36,7 @@ def merge_one(board: str, bindir: Path, staticfs: str, output: str) -> None: esptool.main(argv) -def main(argv=None) -> None: +def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--board', required=True, action='append', help='board name(s); repeatable, whitespace-separated') ap.add_argument('--bindir', required=True, help='dir with app.bin, bootloader.bin, partition-table.bin; {board} is expanded') @@ -50,7 +50,7 @@ def main(argv=None) -> None: for template in (args.bindir, args.output): template.format(board=names[0]) except (KeyError, IndexError, ValueError) as e: - raise SystemExit(f'error: cannot expand --bindir or --output ({type(e).__name__}: {e}); only {{board}} is expanded') + raise SystemExit(f'error: cannot expand --bindir or --output ({type(e).__name__}: {e}); only {{board}} is expanded') from e if len(names) > 1 and len({args.output.format(board=b) for b in names}) < len(names): raise SystemExit('error: --output expands to the same path for different boards; include {board}') @@ -71,7 +71,8 @@ def main(argv=None) -> None: if failed: raise SystemExit(f'error: failed to merge: {", ".join(failed)}') + return 0 if __name__ == '__main__': - main() + raise SystemExit(main()) diff --git a/scripts/utils/boards.py b/scripts/utils/boards.py index 3ad7e18d5..bb356c607 100644 --- a/scripts/utils/boards.py +++ b/scripts/utils/boards.py @@ -10,6 +10,9 @@ from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path +from typing import Any + +from .espimage import DESC_BOARD_LEN ROOT = Path(__file__).resolve().parent.parent.parent @@ -29,9 +32,11 @@ KNOWN_KEYS = PIN_KEYS + BOOL_KEYS + LIST_KEYS NO_PIN = -1 +MAX_GPIO = 127 +MAX_BYPASS_GPIO = 63 -# The longest board name the image descriptor holds: espimage.DESC_BOARD_LEN less its NUL terminator. -NAME_MAX_LEN = 47 +# The longest board name the image descriptor holds, less its NUL terminator. +NAME_MAX_LEN = DESC_BOARD_LEN - 1 # Descriptor flag bits (BoardDescriptor.h: OPENSHOCK_BOARD_FLAG_*). FLAG_LED_SWAP_RG = 1 << 0 @@ -61,7 +66,7 @@ def gpio_bypass_mask(self) -> int: def board_fragment(board: str) -> Path: fragment = ROOT / 'boards' / f'{board}.defaults' if not fragment.is_file(): - avail = ', '.join(sorted(p.stem for p in (ROOT / 'boards').glob('*.defaults'))) + avail = ', '.join(all_boards()) raise SystemExit(f'error: unknown board {board!r}\n available: {avail}') return fragment @@ -99,15 +104,44 @@ def _parse_pin(fragment: Path, lineno: int, key: str, value: str) -> int: try: pin = int(value) except ValueError: - raise SystemExit(f'error: {fragment}:{lineno}: {key} expects an integer GPIO, got {value!r}') - if pin < NO_PIN or pin > 127: + raise SystemExit(f'error: {fragment}:{lineno}: {key} expects an integer GPIO, got {value!r}') from None + if pin < NO_PIN or pin > MAX_GPIO: raise SystemExit(f'error: {fragment}:{lineno}: {key} must be -1 (no pin) or a GPIO number, got {pin}') return pin +def _parse_bool(fragment: Path, lineno: int, key: str, value: str) -> bool: + if value not in ('y', 'n'): + raise SystemExit(f'error: {fragment}:{lineno}: {key} is a boolean, expected y or n, got {value!r}') + return value == 'y' + + +def _parse_pin_list(fragment: Path, lineno: int, key: str, value: str) -> tuple[int, ...]: + pins = [] + for item in filter(None, (s.strip() for s in value.split(','))): + pin = _parse_pin(fragment, lineno, key, item) + if pin == NO_PIN or pin > MAX_BYPASS_GPIO: + raise SystemExit( + f'error: {fragment}:{lineno}: {key} entries must be GPIO numbers 0-{MAX_BYPASS_GPIO}, got {pin}' + ) + pins.append(pin) + if len(set(pins)) != len(pins): + raise SystemExit(f'error: {fragment}:{lineno}: {key} lists a pin twice') + return tuple(sorted(pins)) + + +def _check_name(fragment: Path) -> str: + name = fragment.stem + if not name.isascii() or len(name) > NAME_MAX_LEN: + raise SystemExit( + f'error: {fragment}: a board name must be at most {NAME_MAX_LEN} ASCII characters to fit the descriptor, got {name!r}' + ) + return name + + def read_board_fragment(fragment: Path) -> BoardSpec: """Parse a fragment's ``OPENSHOCK_*=`` assignments and IDF target. Anything unrecognised is an error, so a typo cannot pass silently.""" - values: dict[str, object] = {key: NO_PIN for key in PIN_KEYS} + values: dict[str, Any] = {key: NO_PIN for key in PIN_KEYS} values.update({key: False for key in BOOL_KEYS}) values.update({key: () for key in LIST_KEYS}) @@ -125,28 +159,13 @@ def read_board_fragment(fragment: Path) -> BoardSpec: ) if key in BOOL_KEYS: - if value not in ('y', 'n'): - raise SystemExit(f'error: {fragment}:{lineno}: {key} is a boolean, expected y or n, got {value!r}') - values[key] = value == 'y' + values[key] = _parse_bool(fragment, lineno, key, value) elif key in LIST_KEYS: - pins = [] - for item in filter(None, (s.strip() for s in value.split(','))): - pin = _parse_pin(fragment, lineno, key, item) - if pin == NO_PIN or pin > 63: - raise SystemExit(f'error: {fragment}:{lineno}: {key} entries must be GPIO numbers 0-63, got {pin}') - pins.append(pin) - if len(set(pins)) != len(pins): - raise SystemExit(f'error: {fragment}:{lineno}: {key} lists a pin twice') - values[key] = tuple(sorted(pins)) + values[key] = _parse_pin_list(fragment, lineno, key, value) else: values[key] = _parse_pin(fragment, lineno, key, value) - name = fragment.stem - if not name.isascii() or len(name) > NAME_MAX_LEN: - raise SystemExit( - f'error: {fragment}: a board name must be at most {NAME_MAX_LEN} ASCII characters to fit the descriptor, got {name!r}' - ) - + name = _check_name(fragment) target = read_target(fragment) return BoardSpec( name=name, @@ -169,8 +188,6 @@ def all_boards() -> list[str]: return sorted(p.stem for p in (ROOT / 'boards').glob('*.defaults')) -# --- Flash size -------------------------------------------------------------------------- - # ESP-IDF's own default when no sdkconfig input picks a CONFIG_ESPTOOLPY_FLASHSIZE_* choice. DEFAULT_FLASH_SIZE = '2MB' @@ -219,27 +236,8 @@ def flash_size_bytes(size: str) -> int: return int(size.removesuffix('MB')) * 1024 * 1024 -# --- Build variants ------------------------------------------------------------------- -# A variant is one compile, and every board whose fragment differs from it only in what -# is settled after linking. That is two things: -# -# - the OPENSHOCK_* lines, stamped into the app image by utils/stamping.py; -# - the flash size. It feeds the partition table's fit check and the image headers' -# flash-size field, and nothing else: no board's partition table, and no firmware -# code, uses flash past VARIANT_FLASH_SIZE, the size partitions/ota_4mb.csv is laid -# out for. So a variant compiles at that size and every board ships its images as -# they are; a bigger board's bootloader logs that the chip is larger than its header -# says and uses the header's size. A board that wants its extra flash needs a bigger -# partition table, which is Kconfig and so a variant of its own - and the fit check -# fails that variant's build until it compiles at the board's size. -# -# Every other Kconfig line is compiled in and splits the variant: the S3 boards on the -# USB-Serial-JTAG console and the ones on the UART cannot share an image, because the -# serial driver and the bootloader's console are both chosen at compile time. -# -# Variants are derived from whichever boards a build asks for, never declared, so a board -# that changes a Kconfig setting moves into its own variant by itself. - +# A variant is one compile shared by boards that differ only in the OPENSHOCK_* lines and flash size. +# It compiles at the flash size partitions/ota_4mb.csv is laid out for; a board needing more fails the fit check. VARIANT_FLASH_SIZE = '4MB' diff --git a/scripts/utils/espimage.py b/scripts/utils/espimage.py index 414f21d06..dc57ab5cf 100644 --- a/scripts/utils/espimage.py +++ b/scripts/utils/espimage.py @@ -27,6 +27,9 @@ IMAGE_MAGIC = 0xE9 HEADER_LEN = 24 +MAX_SEGMENTS = 16 +CHIP_ID_OFFSET = 0x0C +HASH_FLAG_OFFSET = 0x17 SEGMENT_HEADER_LEN = 8 CHECKSUM_INIT = 0xEF DIGEST_LEN = 32 @@ -46,7 +49,6 @@ 'esp32h2': 0x0010, } -# --- Board descriptor ---------------------------------------------------------------- # Mirrors OpenShock::BoardDescriptor (components/common/include/BoardDescriptor.h): # u32 magic, u16 schema_version, u16 length, char board[48], char chip[16], # i8 rf_tx_gpio, i8 estop_gpio, i8 led_gpio, i8 ws2812b_gpio, @@ -60,7 +62,13 @@ DESC_BOARD_LEN = 48 DESC_CHIP_LEN = 16 -assert struct.calcsize(DESC_FORMAT) == DESC_LEN + +class DescriptorError(ValueError): + pass + + +class ImageError(ValueError): + pass @dataclass(frozen=True) @@ -103,11 +111,6 @@ def pack(self) -> bytes: return body[:DESC_CRC_LEN] + struct.pack(' Descriptor: """Decode and validate 256 descriptor bytes. Raises DescriptorError when they are not a descriptor.""" if len(data) < DESC_LEN: @@ -137,13 +140,6 @@ def unpack_descriptor(data: bytes) -> Descriptor: ) -# --- Image ------------------------------------------------------------------------- - - -class ImageError(ValueError): - pass - - @dataclass(frozen=True) class Segment: load_addr: int @@ -178,9 +174,8 @@ def verify(self) -> None: stored = self.data[self.checksum_offset] if stored != self.checksum(): raise ImageError(f'checksum mismatch: stored 0x{stored:02X}, computed 0x{self.checksum():02X}') - if self.hash_appended: - if self.data[self.digest_offset :] != self.digest(): - raise ImageError('sha256 digest mismatch') + if self.hash_appended and self.data[self.digest_offset :] != self.digest(): + raise ImageError('sha256 digest mismatch') def descriptor_bytes(self) -> bytes: return self.data[DESC_OFFSET : DESC_OFFSET + DESC_LEN] @@ -212,10 +207,10 @@ def parse(data: bytes) -> Image: if data[0] != IMAGE_MAGIC: raise ImageError(f'bad image magic 0x{data[0]:02X}') segment_count = data[1] - if segment_count == 0 or segment_count > 16: + if segment_count == 0 or segment_count > MAX_SEGMENTS: raise ImageError(f'unreasonable segment count {segment_count}') - chip_id = struct.unpack_from(' Image: cursor += length # The checksum byte is the last byte of the 16-byte-aligned block the segments end in. - checksum_offset = ((cursor + 1 + 15) // 16) * 16 - 1 + checksum_offset = cursor + (-(cursor + 1) % 16) expected_len = checksum_offset + 1 + (DIGEST_LEN if hash_appended else 0) if len(data) != expected_len: raise ImageError(f'image is {len(data)} bytes, expected {expected_len} from its segment table') diff --git a/scripts/utils/partitions.py b/scripts/utils/partitions.py index bee238a03..6cbf8e080 100644 --- a/scripts/utils/partitions.py +++ b/scripts/utils/partitions.py @@ -2,6 +2,7 @@ import re from pathlib import Path +from typing import TypedDict from .boards import ROOT, sdkconfig_value @@ -9,6 +10,13 @@ DEFAULT_PARTITION_TABLE_OFFSET = 0x8000 +class Partition(TypedDict): + type: str + subtype: str + offset: int + size: int + + def partition_csv_for(board: str) -> Path: name = sdkconfig_value(board, 'CONFIG_PARTITION_TABLE_CUSTOM_FILENAME') if not name: @@ -21,9 +29,9 @@ def partition_table_offset(board: str) -> int: return int(value, 0) if value else DEFAULT_PARTITION_TABLE_OFFSET -def read_partitions(csv_path: Path) -> dict[str, dict]: - """Partitions by name: {'type', 'subtype', 'offset', 'size'}. Every entry must give an explicit offset.""" - partitions = {} +def read_partitions(csv_path: Path) -> dict[str, Partition]: + """Partitions by name. Every entry must give an explicit offset.""" + partitions: dict[str, Partition] = {} for raw in csv_path.read_text(encoding='utf-8').splitlines(): line = raw.split('#', 1)[0].strip() if not line: @@ -46,7 +54,7 @@ def _parse_size(size: str) -> int: return value * {'': 1, 'K': 1024, 'M': 1024 * 1024}[m.group(2)] -def first_app_partition(partitions: dict[str, dict]) -> dict: +def first_app_partition(partitions: dict[str, Partition]) -> Partition: apps = [p for p in partitions.values() if p['type'] == 'app'] if not apps: raise SystemExit('error: partition table has no app partition') diff --git a/scripts/utils/stamping.py b/scripts/utils/stamping.py index 42e5072f3..050c5bffa 100644 --- a/scripts/utils/stamping.py +++ b/scripts/utils/stamping.py @@ -9,9 +9,6 @@ from . import boards, espimage -# boards.py checks names where fragments are read, so it must agree with the descriptor's field. -assert boards.NAME_MAX_LEN == espimage.DESC_BOARD_LEN - 1 - def descriptor_for(spec: boards.BoardSpec) -> espimage.Descriptor: return espimage.Descriptor( @@ -44,7 +41,7 @@ def read_descriptor(image: espimage.Image, path) -> espimage.Descriptor: except espimage.DescriptorError as err: raise SystemExit( f'error: {path} carries no board descriptor at 0x{espimage.DESC_OFFSET:X} ({err}); was it built with BoardDescriptor.cpp?' - ) + ) from err def stamp_image(image: espimage.Image, spec: boards.BoardSpec, path) -> espimage.Image: From b4cffcfa510dd37cdd15b2d0788823c85f6e74e6 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 23 Oct 2026 14:18:05 +0200 Subject: [PATCH 2/5] test: split script tests per tree and cover publish, vars and gen paths - move tests of scripts/ into scripts/tests, keep CI-script tests under .github/scripts/tests - add tests for get_vars inputs, repo server upload/init/discard paths, merge_one, gen_staticfs, gen_env_header and cert_bundle ci: open the pinned-cert PR with the release app token so its checks run --- .github/scripts/requirements-dev.txt | 2 +- .github/scripts/tests/test_get_vars.py | 39 +++++ .../scripts/tests/test_repo_server_publish.py | 120 ++++++++++++++ .github/workflows/pinned-cert-audit.yml | 14 +- .github/workflows/script-tests.yml | 2 +- pyproject.toml | 4 +- {.github/scripts => scripts}/tests/support.py | 2 +- .../tests/test_build_cli.py | 0 .../tests/test_build_scripts.py | 48 ++++++ .../tests/test_build_variants.py | 0 scripts/tests/test_cert_bundle.py | 149 ++++++++++++++++++ scripts/tests/test_gen_env_header.py | 65 ++++++++ .../tests/test_merge_image.py | 25 +++ .../tests/test_package_boards.py | 0 .../tests/test_repository_boards.py | 0 .../tests/test_stamp_board.py | 0 16 files changed, 462 insertions(+), 8 deletions(-) rename {.github/scripts => scripts}/tests/support.py (97%) rename {.github/scripts => scripts}/tests/test_build_cli.py (100%) rename {.github/scripts => scripts}/tests/test_build_scripts.py (59%) rename {.github/scripts => scripts}/tests/test_build_variants.py (100%) create mode 100644 scripts/tests/test_cert_bundle.py create mode 100644 scripts/tests/test_gen_env_header.py rename {.github/scripts => scripts}/tests/test_merge_image.py (64%) rename {.github/scripts => scripts}/tests/test_package_boards.py (100%) rename {.github/scripts => scripts}/tests/test_repository_boards.py (100%) rename {.github/scripts => scripts}/tests/test_stamp_board.py (100%) diff --git a/.github/scripts/requirements-dev.txt b/.github/scripts/requirements-dev.txt index 4809ba12a..462ea3b56 100644 --- a/.github/scripts/requirements-dev.txt +++ b/.github/scripts/requirements-dev.txt @@ -6,7 +6,7 @@ # # python -m venv .venv # .venv/bin/pip install --require-hashes -r .github/scripts/requirements-dev.txt -# .venv/bin/python -m pytest .github/scripts/tests +# .venv/bin/python -m pytest -r requirements.txt diff --git a/.github/scripts/tests/test_get_vars.py b/.github/scripts/tests/test_get_vars.py index 1506685fd..52b871aab 100644 --- a/.github/scripts/tests/test_get_vars.py +++ b/.github/scripts/tests/test_get_vars.py @@ -106,3 +106,42 @@ def test_main_channel_input(run_main, monkeypatch, channel, nightly, expected): monkeypatch.setenv('NIGHTLY_INPUT', nightly) outputs, _ = run_main('refs/heads/develop') assert outputs['release-channel'] == expected + + +def test_main_on_a_pull_request_names_the_version_after_the_head_branch(run_main, monkeypatch): + monkeypatch.setenv('GITHUB_HEAD_REF', 'feature/rf-fix') + outputs, _ = run_main('refs/pull/12/merge') + assert outputs['version'] == '1.4.0-rf-fix+01234567' + assert outputs['release-channel'] == 'develop' + + +@pytest.mark.parametrize('ref', ['refs/pull/12/head', 'refs/remotes/origin/x', 'develop']) +def test_main_refuses_a_ref_that_is_not_a_branch_tag_or_merge_ref(run_main, ref): + with pytest.raises(SystemExit): + run_main(ref) + + +@pytest.mark.parametrize( + 'ref, skip, expected', + [ + ('refs/heads/develop', '', '1.5.0-develop+01234567'), + ('refs/heads/develop', 'true', '1.4.0-develop+01234567'), + ('refs/tags/1.4.1', '', '1.4.1'), + ], +) +def test_main_next_version_applies_to_branch_builds_only(run_main, monkeypatch, ref, skip, expected): + monkeypatch.setenv('RELEASE_NEXT_VERSION', '1.5.0') + monkeypatch.setenv('RELEASE_SKIP', skip) + outputs, _ = run_main(ref) + assert outputs['version'] == expected + + +def test_main_refuses_a_channel_the_server_does_not_know(run_main, monkeypatch): + monkeypatch.setenv('CHANNEL_INPUT', 'nightly') + with pytest.raises(SystemExit): + run_main('refs/heads/develop') + + +def test_main_refuses_an_unknown_board(run_main): + with pytest.raises(SystemExit): + run_main('refs/heads/develop', 'OpenShock-Core-V2, No-Such-Board') diff --git a/.github/scripts/tests/test_repo_server_publish.py b/.github/scripts/tests/test_repo_server_publish.py index 55a933f8e..715b0d4ab 100644 --- a/.github/scripts/tests/test_repo_server_publish.py +++ b/.github/scripts/tests/test_repo_server_publish.py @@ -1,7 +1,12 @@ +import hashlib +import json + import pytest import repo_server_publish import requests +_REAL_UPLOAD_BOARD = repo_server_publish.upload_board + class FakeResponse: def __init__(self, status_code): @@ -109,3 +114,118 @@ def unreachable(url, **kwargs): session = FakeSession(publish=201) session.delete = unreachable assert repo_server_publish.discard(session, 'https://repo', 'rel-1') == 'failed' + + +# --- upload_board, init_release, extract_changelog --- + + +class ScriptedSession: + """Plays back one scripted outcome per call: a status code, or an exception to raise.""" + + def __init__(self, *outcomes): + self.outcomes = list(outcomes) + self.calls = [] + + def _answer(self, url, kwargs): + self.calls.append((url, kwargs)) + outcome = self.outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + return FakeResponse(outcome) + + put = post = delete = lambda self, url, **kwargs: self._answer(url, kwargs) + + +@pytest.fixture +def real_upload(monkeypatch, tmp_path): + monkeypatch.setattr(repo_server_publish, 'upload_board', _REAL_UPLOAD_BOARD) + monkeypatch.setattr(repo_server_publish.time, 'sleep', lambda seconds: None) + write_board_artifacts(tmp_path, 'BoardA') + return tmp_path + + +def test_upload_retries_a_5xx_then_succeeds(real_upload): + session = ScriptedSession(503, requests.ConnectionError('reset'), 200) + repo_server_publish.upload_board(session, 'https://repo', 'rel-1', 'BoardA', real_upload) + assert [url for url, _ in session.calls] == ['https://repo/2/firmware/releases/rel-1/boards/BoardA'] * 3 + + +def test_upload_gives_up_after_the_last_5xx(real_upload): + session = ScriptedSession(500, 500, 500) + with pytest.raises(SystemExit): + repo_server_publish.upload_board(session, 'https://repo', 'rel-1', 'BoardA', real_upload) + assert len(session.calls) == repo_server_publish.UPLOAD_RETRIES + + +def test_upload_does_not_retry_a_4xx(real_upload): + session = ScriptedSession(422) + with pytest.raises(SystemExit): + repo_server_publish.upload_board(session, 'https://repo', 'rel-1', 'BoardA', real_upload) + assert len(session.calls) == 1 + + +def test_upload_gives_up_after_repeated_transport_errors(real_upload): + session = ScriptedSession(*[requests.ConnectionError('down')] * repo_server_publish.UPLOAD_RETRIES) + with pytest.raises(SystemExit): + repo_server_publish.upload_board(session, 'https://repo', 'rel-1', 'BoardA', real_upload) + assert not session.outcomes + + +def test_upload_sends_a_sha256_manifest_of_every_artifact(real_upload): + session = ScriptedSession(200) + repo_server_publish.upload_board(session, 'https://repo', 'rel-1', 'BoardA', real_upload) + kwargs = session.calls[0][1] + manifest = json.loads(kwargs['data']['sha256']) + assert set(manifest) == set(kwargs['files']) == {*repo_server_publish.BUILD_ARTIFACTS, 'staticfs', 'merged'} + assert manifest['merged'] == hashlib.sha256(b'merged').hexdigest() + + +class CreatedResponse(FakeResponse): + def json(self): + return {'id': 'rel-9', 'status': 'staged'} + + +def test_init_release_returns_the_staged_id(): + session = FakeSession(publish=201) + session.post = lambda url, **kwargs: CreatedResponse(201) + release_id = repo_server_publish.init_release( + session, 'https://repo', version='1.5.0', channel='beta', boards=['BoardA'], changelog='# 1.5.0' + ) + assert release_id == 'rel-9' + + +def test_init_release_dies_on_anything_but_201(): + session = FakeSession(publish=200) + session.post = lambda url, **kwargs: FakeResponse(409) + with pytest.raises(SystemExit): + repo_server_publish.init_release(session, 'https://repo', version='1.5.0', channel='beta', boards=['BoardA'], changelog='x') + + +def test_extract_changelog_takes_the_newest_section_only(tmp_path): + changelog = tmp_path / 'CHANGELOG.md' + changelog.write_text('preamble\n# 1.5.0\n- new\n\n## Fixes\n- a\n# 1.4.0\n- old\n', encoding='utf-8') + assert repo_server_publish.extract_changelog(changelog) == '# 1.5.0\n- new\n\n## Fixes\n- a' + + +@pytest.mark.parametrize('body', [None, 'no heading here\n', '## only a subheading\n']) +def test_extract_changelog_fails_without_a_section(tmp_path, body): + changelog = tmp_path / 'CHANGELOG.md' + if body is not None: + changelog.write_text(body, encoding='utf-8') + with pytest.raises(SystemExit): + repo_server_publish.extract_changelog(changelog) + + +@pytest.mark.parametrize('status, closed', [(404, True), (409, True), (500, False)]) +def test_abort_mode_reports_but_never_fails(monkeypatch, capsys, status, closed): + monkeypatch.setenv('RELEASE_ID', 'rel-1') + session = ScriptedSession(status) + assert repo_server_publish.do_abort(session, 'https://repo') == 0 + assert ('already closed' in capsys.readouterr().out) == closed + + +def test_abort_mode_without_a_release_does_nothing(monkeypatch): + monkeypatch.setenv('RELEASE_ID', '') + session = ScriptedSession() + assert repo_server_publish.do_abort(session, 'https://repo') == 0 + assert session.calls == [] diff --git a/.github/workflows/pinned-cert-audit.yml b/.github/workflows/pinned-cert-audit.yml index 3746bd920..da84cc806 100644 --- a/.github/workflows/pinned-cert-audit.yml +++ b/.github/workflows/pinned-cert-audit.yml @@ -17,8 +17,7 @@ concurrency: cancel-in-progress: false permissions: - contents: write # push the retire-pin branch - pull-requests: write # open the retire-pin PR + contents: read defaults: run: @@ -31,8 +30,17 @@ jobs: timeout-minutes: 10 steps: + # An App token, not GITHUB_TOKEN, so the retire-pin PR triggers its required checks. + - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + id: app-token + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_KEY }} + - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + token: ${{ steps.app-token.outputs.token }} - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -69,7 +77,7 @@ jobs: - name: Open PR to retire unneeded pinned root(s) if: steps.audit.outputs.removable != '' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} REMOVABLE: ${{ steps.audit.outputs.removable }} REF_NAME: ${{ github.ref_name }} RUN_ID: ${{ github.run_id }} diff --git a/.github/workflows/script-tests.yml b/.github/workflows/script-tests.yml index 60c8f1a1f..68a2d4e4f 100644 --- a/.github/workflows/script-tests.yml +++ b/.github/workflows/script-tests.yml @@ -1,4 +1,4 @@ -# Unit tests for the CI/release scripts (.github/scripts) and the build helpers in scripts/. +# Unit tests for the CI/release scripts (.github/scripts/tests) and the build helpers (scripts/tests). name: script-tests on: diff --git a/pyproject.toml b/pyproject.toml index 115da66c0..956cb014e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ version = '0.0.0' requires-python = '>=3.10' [tool.pytest.ini_options] -pythonpath = ['.github/scripts', 'scripts', '.github/scripts/tests'] -testpaths = ['.github/scripts/tests'] +pythonpath = ['.github/scripts', 'scripts', 'certificates'] +testpaths = ['.github/scripts/tests', 'scripts/tests'] [tool.ruff] target-version = 'py310' diff --git a/.github/scripts/tests/support.py b/scripts/tests/support.py similarity index 97% rename from .github/scripts/tests/support.py rename to scripts/tests/support.py index a3059548c..991eba743 100644 --- a/.github/scripts/tests/support.py +++ b/scripts/tests/support.py @@ -6,7 +6,7 @@ from utils import espimage -ROOT = Path(__file__).resolve().parents[3] +ROOT = Path(__file__).resolve().parents[2] def blank(chip: str = 'ESP32-S3') -> bytes: diff --git a/.github/scripts/tests/test_build_cli.py b/scripts/tests/test_build_cli.py similarity index 100% rename from .github/scripts/tests/test_build_cli.py rename to scripts/tests/test_build_cli.py diff --git a/.github/scripts/tests/test_build_scripts.py b/scripts/tests/test_build_scripts.py similarity index 59% rename from .github/scripts/tests/test_build_scripts.py rename to scripts/tests/test_build_scripts.py index 4ce9b5215..0eab632fb 100644 --- a/.github/scripts/tests/test_build_scripts.py +++ b/scripts/tests/test_build_scripts.py @@ -1,5 +1,6 @@ """Tests for the build helpers in scripts/ (partition table reading, reproducible staticfs staging).""" +import gzip import os from pathlib import Path @@ -77,3 +78,50 @@ def test_frontend_build_is_current_tracks_source_changes(tmp_path): os.utime(source, (3000, 3000)) assert not gen_staticfs.frontend_build_is_current(frontend, frontend / 'build') + + +def test_staged_assets_gzip_everything_but_prebuilt_gz(tmp_path): + build = tmp_path / 'build' + build.mkdir() + (build / 'index.html').write_text('x', encoding='utf-8') + (build / 'font.woff2.gz').write_bytes(b'already') + + www = gen_staticfs.stage_assets(build, tmp_path / 'staging') + + assert sorted(p.name for p in www.iterdir()) == ['font.woff2.gz', 'index.html.gz'] + assert (www / 'font.woff2.gz').read_bytes() == b'already' + assert gzip.decompress((www / 'index.html.gz').read_bytes()) == b'x' + + +def test_staging_a_build_without_index_html_fails(tmp_path): + build = tmp_path / 'build' + build.mkdir() + (build / 'app.js').write_text('1', encoding='utf-8') + with pytest.raises(SystemExit, match='index.html.gz missing'): + gen_staticfs.stage_assets(build, tmp_path / 'staging') + + +def test_packed_image_is_partition_sized_and_holds_only_www(tmp_path): + from littlefs import LittleFS + + staging = tmp_path / 'staging' + (staging / 'www').mkdir(parents=True) + (staging / 'www' / 'index.html.gz').write_bytes(b'page') + (staging / 'stray.txt').write_text('x', encoding='utf-8') + output = tmp_path / 'out' / 'staticfs.bin' + + gen_staticfs.pack_image(staging, output, 64 * 1024) + + image = output.read_bytes() + assert len(image) == 64 * 1024 + fs = LittleFS(block_size=gen_staticfs.BLOCK_SIZE, block_count=16, disk_version=0) + fs.context.buffer[:] = image + fs.mount() + assert fs.listdir('/') == ['www'] + with fs.open('www/index.html.gz', 'rb') as f: + assert f.read() == b'page' + + +def test_packing_refuses_a_size_that_is_not_a_block_multiple(tmp_path): + with pytest.raises(SystemExit, match='not a multiple of block size'): + gen_staticfs.pack_image(tmp_path, tmp_path / 'out.bin', 5000) diff --git a/.github/scripts/tests/test_build_variants.py b/scripts/tests/test_build_variants.py similarity index 100% rename from .github/scripts/tests/test_build_variants.py rename to scripts/tests/test_build_variants.py diff --git a/scripts/tests/test_cert_bundle.py b/scripts/tests/test_cert_bundle.py new file mode 100644 index 000000000..f0ef5cb47 --- /dev/null +++ b/scripts/tests/test_cert_bundle.py @@ -0,0 +1,149 @@ +"""Tests for the offline parts of certificates/cert_bundle.py: PEM loading and validation, expiry, merging.""" + +import datetime + +import cert_bundle +import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID + +NOW = datetime.datetime.now(datetime.timezone.utc) + + +def make_cert(name='Root A', *, ca=True, days_left=365): + key = ec.generate_private_key(ec.SECP256R1()) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, name)]) + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(NOW - datetime.timedelta(days=400)) + .not_valid_after(NOW + datetime.timedelta(days=days_left)) + .add_extension(x509.BasicConstraints(ca=ca, path_length=None), critical=True) + .sign(key, hashes.SHA256()) + ) + return cert, cert.public_bytes(serialization.Encoding.PEM) + + +@pytest.mark.parametrize( + 'content, expected', + [ + (b'a' * 64 + b' cacert.pem\n', 'a' * 64), + (b'A' * 64 + b' *cacert.pem', 'a' * 64), + (b'b' * 64 + b'\nsecond line ignored', 'b' * 64), + (b'c' * 64 + b' /some/dir/cacert.pem', 'c' * 64), + ], +) +def test_parse_sha256_file(content, expected): + assert cert_bundle.parse_sha256_file(content, expect_filename='cacert.pem') == expected + + +@pytest.mark.parametrize('content', [b'', b' \n', b'not-a-hash cacert.pem', b'a' * 63, b'a' * 64 + b' other.pem']) +def test_parse_sha256_file_rejects(content): + with pytest.raises(cert_bundle.InputError): + cert_bundle.parse_sha256_file(content, expect_filename='cacert.pem') + + +def test_a_single_cert_block_is_canonicalised_and_labels_are_dropped(): + _, pem = make_cert() + block = cert_bundle._load_single_pem_cert_block_from_bytes_strict(b'Some Label\n=====\n' + pem, source='x') + assert block == pem + + +@pytest.mark.parametrize( + 'mutate', + [ + lambda pem: pem + pem, + lambda pem: pem + b'trailing', + lambda pem: pem + b'-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY-----\n', + lambda pem: b'', + lambda pem: pem.replace(b'MII', b'!!!', 1), + ], +) +def test_a_cert_file_must_hold_exactly_one_clean_certificate(mutate): + _, pem = make_cert() + with pytest.raises(cert_bundle.InputError): + cert_bundle._load_single_pem_cert_block_from_bytes_strict(mutate(pem), source='x') + + +def test_extract_pem_cert_blocks(): + _, a = make_cert('A') + _, b = make_cert('B') + assert len(cert_bundle.extract_pem_cert_blocks(b'## banner\n' + a + b, source='x')) == 2 + with pytest.raises(cert_bundle.InputError): + cert_bundle.extract_pem_cert_blocks(b'nothing', source='x') + + +def test_expiry_is_classified_against_the_warning_window(): + certs = [make_cert(f'C{days}', days_left=days)[0] for days in (-5, 30, 400)] + expired, expiring = cert_bundle.check_certs_expiry(certs, ['a', 'b', 'c'], warn_days=90) + assert [d.split(' ')[0] for d in expired] == ['CN=C-5'] + assert [d.split(' ')[0] for d in expiring] == ['CN=C30'] + + +@pytest.mark.parametrize('raw, expected', [(None, 90), ('', 90), ('14', 14), ('0', 0)]) +def test_expiry_warn_days_from_env(monkeypatch, raw, expected): + monkeypatch.delenv('PINNED_CERT_EXPIRY_WARN_DAYS', raising=False) + if raw is not None: + monkeypatch.setenv('PINNED_CERT_EXPIRY_WARN_DAYS', raw) + assert cert_bundle._expiry_warn_days() == expected + + +@pytest.mark.parametrize('raw', ['soon', '-1']) +def test_expiry_warn_days_rejects_bad_values(monkeypatch, raw): + monkeypatch.setenv('PINNED_CERT_EXPIRY_WARN_DAYS', raw) + with pytest.raises(cert_bundle.InputError): + cert_bundle._expiry_warn_days() + + +def test_non_ca_certificates_are_refused(): + ca, _ = make_cert('CA') + leaf, _ = make_cert('Leaf', ca=False) + cert_bundle._reject_non_ca([ca], ['ca.pem'], kind='pinned') + with pytest.raises(cert_bundle.InputError, match='non-CA pinned'): + cert_bundle._reject_non_ca([ca, leaf], ['ca.pem', 'leaf.pem'], kind='pinned') + + +def test_duplicate_subjects_are_refused(): + a1, _ = make_cert('Same') + a2, _ = make_cert('Same') + b, _ = make_cert('Other') + cert_bundle.validate_unique_subjects([a1, b]) + with pytest.raises(cert_bundle.InputError, match='Duplicate certificate subject'): + cert_bundle.validate_unique_subjects([a1, b, a2]) + + +def test_load_all_certs_reports_an_unparseable_block(): + _, pem = make_cert() + assert len(cert_bundle.load_all_certs([pem])) == 1 + with pytest.raises(cert_bundle.InputError, match='invalid CA certificate'): + cert_bundle.load_all_certs([cert_bundle.PEM_BEGIN + b'\nAAAA\n' + cert_bundle.PEM_END + b'\n']) + + +def test_append_pem_blocks_separates_blocks_with_a_blank_line(): + assert cert_bundle._append_pem_blocks(b'base', [b'one\n', b'two\n']) == b'base\n\none\n\ntwo\n' + assert cert_bundle._append_pem_blocks(b'', [b'one\n']) == b'\none\n' + + +def test_atomic_write_replaces_the_file_and_leaves_no_temp(tmp_path): + target = tmp_path / 'sub' / 'bundle.pem' + cert_bundle.atomic_write(target, b'old') + cert_bundle.atomic_write(target, b'new') + assert [p.name for p in target.parent.iterdir()] == ['bundle.pem'] + assert target.read_bytes() == b'new' + + +def test_load_dir_cert_blocks_handles_a_missing_or_empty_dir(tmp_path, monkeypatch): + monkeypatch.setattr(cert_bundle, 'HERE', tmp_path) + assert cert_bundle.load_dir_cert_blocks('nope', '*.pem') == ([], []) + (tmp_path / 'pins').mkdir() + assert cert_bundle.load_dir_cert_blocks('pins', '*.pem') == ([], []) + _, pem = make_cert() + (tmp_path / 'pins' / 'a.pem').write_bytes(pem) + blocks, paths = cert_bundle.load_dir_cert_blocks('pins', '*.pem') + assert blocks == [pem] + assert [p.name for p in paths] == ['a.pem'] diff --git a/scripts/tests/test_gen_env_header.py b/scripts/tests/test_gen_env_header.py new file mode 100644 index 000000000..a2fbb0742 --- /dev/null +++ b/scripts/tests/test_gen_env_header.py @@ -0,0 +1,65 @@ +"""Tests for the pure helpers of scripts/gen_env_header.py: version parsing and C header rendering.""" + +import gen_env_header +import pytest + + +@pytest.mark.parametrize( + 'version, expected', + [ + ('1.5.0', (1, 5, 0, None, None)), + ('v0.8.1', (0, 8, 1, None, None)), + ('1.5.0-rc.2+abc123', (1, 5, 0, 'rc.2', 'abc123')), + ('0.0.0-local+1234567', (0, 0, 0, 'local', '1234567')), + ], +) +def test_split_semver(version, expected): + assert gen_env_header.split_semver(version) == expected + + +@pytest.mark.parametrize('version', ['1.5', '1.5.0-', 'latest', '']) +def test_split_semver_rejects(version): + with pytest.raises(ValueError, match='Invalid semver'): + gen_env_header.split_semver(version) + + +def test_serialize_quotes_strings_even_when_numeric(): + assert gen_env_header.serialize(5) == '5' + assert gen_env_header.serialize('1') == '"1"' + assert gen_env_header.serialize('a"b\\c') == '"a\\"b\\\\c"' + + +def test_version_defines_include_only_the_parts_present(monkeypatch): + monkeypatch.setenv('OPENSHOCK_FW_VERSION', '1.5.0-rc.2+abc') + defines = gen_env_header.version_defines('deadbeef') + assert defines['OPENSHOCK_FW_VERSION_MAJOR'] == 1 + assert defines['OPENSHOCK_FW_VERSION_PRERELEASE'] == 'rc.2' + assert defines['OPENSHOCK_FW_VERSION_BUILD'] == 'abc' + assert defines['OPENSHOCK_FW_GIT_COMMIT'] == 'deadbeef' + + monkeypatch.setenv('OPENSHOCK_FW_VERSION', '1.5.0') + defines = gen_env_header.version_defines(None) + assert not {'OPENSHOCK_FW_VERSION_PRERELEASE', 'OPENSHOCK_FW_VERSION_BUILD', 'OPENSHOCK_FW_GIT_COMMIT'} & set(defines) + + +def test_resolve_version_falls_back_locally_but_not_in_ci(monkeypatch): + monkeypatch.delenv('OPENSHOCK_FW_VERSION', raising=False) + monkeypatch.setattr(gen_env_header.github, 'is_ci', lambda: False) + assert gen_env_header.resolve_version('0123456789') == '0.0.0-local+0123456' + assert gen_env_header.resolve_version(None) == '0.0.0-local' + + monkeypatch.setattr(gen_env_header.github, 'is_ci', lambda: True) + with pytest.raises(SystemExit, match='OPENSHOCK_FW_VERSION must be set'): + gen_env_header.resolve_version(None) + + +def test_board_defines_name_the_chip_macro(monkeypatch): + monkeypatch.setattr(gen_env_header.github, 'build_mode', lambda: ('release', 3)) + defines = gen_env_header.board_defines('ESP32-S3') + assert defines['OPENSHOCK_FW_CHIP_ESP32S3'] == 1 + assert (defines['OPENSHOCK_FW_MODE'], defines['OPENSHOCK_LOG_LEVEL']) == ('release', 3) + + +def test_render_sorts_defines_under_a_pragma_once(): + text = gen_env_header.render('Title.', {'B_VALUE': 'x', 'A_VALUE': 1}) + assert text.splitlines()[2:] == ['#pragma once', '', '#define A_VALUE 1', '#define B_VALUE "x"'] diff --git a/.github/scripts/tests/test_merge_image.py b/scripts/tests/test_merge_image.py similarity index 64% rename from .github/scripts/tests/test_merge_image.py rename to scripts/tests/test_merge_image.py index 021351eb2..f662c851a 100644 --- a/.github/scripts/tests/test_merge_image.py +++ b/scripts/tests/test_merge_image.py @@ -39,3 +39,28 @@ def test_an_unwritable_output_dir_fails_that_board_and_continues(tmp_path, monke with pytest.raises(SystemExit, match=r'failed to merge: OpenShock-Core-V2$'): merge_image.main(args) assert merged == ['OpenShock-Core-V1'] + + +def test_merge_one_places_each_image_at_its_partition_offset(monkeypatch, tmp_path): + import esptool + + calls = [] + monkeypatch.setattr(esptool, 'main', calls.append) + merge_image.merge_one('OpenShock-Core-V2', tmp_path / 'bin', 'staticfs.bin', 'out.bin') + + argv = calls[0] + assert argv[:2] == ['--chip', 'esp32s3'] + assert argv[argv.index('--flash-size') + 1] == 'keep' + pairs = dict(zip(argv[7::2], argv[8::2], strict=True)) + assert pairs == { + '0x0': str(tmp_path / 'bin' / 'bootloader.bin'), + '0x8000': str(tmp_path / 'bin' / 'partition-table.bin'), + '0x10000': str(tmp_path / 'bin' / 'app.bin'), + '0x353000': 'staticfs.bin', + } + + +def test_merge_one_refuses_a_table_without_static0(monkeypatch, tmp_path): + monkeypatch.setattr(merge_image.partitions, 'read_partitions', lambda path: {'app0': {}}) + with pytest.raises(SystemExit, match='no static0 partition'): + merge_image.merge_one('OpenShock-Core-V2', tmp_path, 'staticfs.bin', 'out.bin') diff --git a/.github/scripts/tests/test_package_boards.py b/scripts/tests/test_package_boards.py similarity index 100% rename from .github/scripts/tests/test_package_boards.py rename to scripts/tests/test_package_boards.py diff --git a/.github/scripts/tests/test_repository_boards.py b/scripts/tests/test_repository_boards.py similarity index 100% rename from .github/scripts/tests/test_repository_boards.py rename to scripts/tests/test_repository_boards.py diff --git a/.github/scripts/tests/test_stamp_board.py b/scripts/tests/test_stamp_board.py similarity index 100% rename from .github/scripts/tests/test_stamp_board.py rename to scripts/tests/test_stamp_board.py From abfcc9a806f6b29cdb6f36212295fb33976d4c29 Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 23 Oct 2026 14:22:46 +0200 Subject: [PATCH 3/5] ci: expand host test ELF glob into an array to satisfy shellcheck SC2211 --- .github/workflows/host-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/host-tests.yml b/.github/workflows/host-tests.yml index 72bbd62d8..409102cc8 100644 --- a/.github/workflows/host-tests.yml +++ b/.github/workflows/host-tests.yml @@ -77,7 +77,8 @@ jobs: name="$(basename "$(dirname "$dir")")" echo "::group::$name" if idf.py -C "$dir" --preview set-target linux build \ - && "$dir"/build/*_host_test.elf; then + && elfs=("$dir"/build/*_host_test.elf) \ + && "${elfs[0]}"; then echo "::endgroup::" else echo "::endgroup::" From 4fb5a6456a477664e12d0f0947993492882d2cbd Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 23 Oct 2026 14:36:36 +0200 Subject: [PATCH 4/5] ci: address CodeRabbit review feedback - fix stale dry-run claim in release note - document root requirements install and correct pip-compile output path - clear GITHUB_STEP_SUMMARY in get_vars test fixture - keep gzip header identical across Python versions via GzipFile - scope app token permissions in pinned-cert-audit --- .changes/ci-repository-server-only-deploy.md | 2 +- .github/scripts/requirements-dev.txt | 1 + .github/scripts/requirements.txt | 2 +- .github/scripts/tests/test_get_vars.py | 2 +- .github/workflows/pinned-cert-audit.yml | 2 ++ scripts/gen_staticfs.py | 3 ++- 6 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.changes/ci-repository-server-only-deploy.md b/.changes/ci-repository-server-only-deploy.md index 2ec17ca82..c40f633a4 100644 --- a/.changes/ci-repository-server-only-deploy.md +++ b/.changes/ci-repository-server-only-deploy.md @@ -9,4 +9,4 @@ CI no longer uploads firmware to the CDN or advances the channel pointer files i What remains of that path moved out of YAML into `.github/scripts/repo_server_publish.py` and `.github/scripts/repo_server_token.py`, joining `get_vars.py` and the build scripts. All three now share `.github/scripts/gha.py` for workflow commands, step outputs and the interpreter floor. HTTP and multipart are `requests`, and `get_vars.py` parses tags with the `semver` package instead of a local regex and comparison class. Dependencies are pinned by digest in `.github/scripts/requirements.txt` and installed with `--require-hashes`, so the job holding the publish credential cannot pick up whatever the index happens to serve. The staged release is a resource with a lifetime, and a `try`/`except` that discards on failure expresses that directly, where the four YAML steps it replaces had to coordinate through `GITHUB_ENV` and an `always()` cleanup step to get the same guarantee. -The `cdn-deploy` and `cdn-bump` jobs are gone, along with the `cdn-upload-firmware`, `cdn-upload-version-info`, `cdn-bump-version` and `sftp-mirror` composite actions and the `cdn` environment they read their Bunny credentials from. Publishing is now driven by the deploy gate instead of the `OPENSHOCK_REPO_SERVER_PUBLISH` variable, which is retired: a release tag or a nightly/manual develop deploy publishes, and every other build does the same init-upload-discard dry run as before. The staged-draft-release check that guarded the CDN upload now guards the publish. The GitHub release and the API announcement still run last, in that order, each chained on the publish succeeding. +The `cdn-deploy` and `cdn-bump` jobs are gone, along with the `cdn-upload-firmware`, `cdn-upload-version-info`, `cdn-bump-version` and `sftp-mirror` composite actions and the `cdn` environment they read their Bunny credentials from. Publishing is now driven by the deploy gate instead of the `OPENSHOCK_REPO_SERVER_PUBLISH` variable, which is retired: a release tag or a nightly/manual develop deploy publishes, and every other build does not upload to the server. The staged-draft-release check that guarded the CDN upload now guards the publish. The GitHub release and the API announcement still run last, in that order, each chained on the publish succeeding. diff --git a/.github/scripts/requirements-dev.txt b/.github/scripts/requirements-dev.txt index 462ea3b56..7523487cd 100644 --- a/.github/scripts/requirements-dev.txt +++ b/.github/scripts/requirements-dev.txt @@ -5,6 +5,7 @@ # requirements.txt is; refresh the same way. # # python -m venv .venv +# .venv/bin/pip install --require-hashes -r requirements.txt # root build lock; scripts/tests need it # .venv/bin/pip install --require-hashes -r .github/scripts/requirements-dev.txt # .venv/bin/python -m pytest diff --git a/.github/scripts/requirements.txt b/.github/scripts/requirements.txt index a95e7d5fc..8594ff7d4 100644 --- a/.github/scripts/requirements.txt +++ b/.github/scripts/requirements.txt @@ -1,5 +1,5 @@ # Hashed for --require-hashes (this job holds the publish credential). Direct deps: requests, semver; the rest is transitive. -# Refresh with: pip-compile --generate-hashes --output-file requirements.txt <(printf 'requests\nsemver\n') +# Refresh from the repository root with: pip-compile --generate-hashes --output-file .github/scripts/requirements.txt <(printf 'requests\nsemver\n') # --- direct --- diff --git a/.github/scripts/tests/test_get_vars.py b/.github/scripts/tests/test_get_vars.py index 52b871aab..0d77b4245 100644 --- a/.github/scripts/tests/test_get_vars.py +++ b/.github/scripts/tests/test_get_vars.py @@ -53,7 +53,7 @@ def test_sanitize(name, expected): def run_main(monkeypatch): """Runs get-vars main() for a ref and board selection, git's tag list stubbed out; returns its outputs.""" monkeypatch.setattr(get_vars.subprocess, 'check_output', lambda *a, **kw: '1.4.0\n1.3.0\n') - for name in ('GITHUB_HEAD_REF', 'RELEASE_NEXT_VERSION', 'RELEASE_SKIP', 'CHANNEL_INPUT', 'NIGHTLY_INPUT'): + for name in ('GITHUB_HEAD_REF', 'RELEASE_NEXT_VERSION', 'RELEASE_SKIP', 'CHANNEL_INPUT', 'NIGHTLY_INPUT', 'GITHUB_STEP_SUMMARY'): monkeypatch.delenv(name, raising=False) def run(ref, boards_input=''): diff --git a/.github/workflows/pinned-cert-audit.yml b/.github/workflows/pinned-cert-audit.yml index da84cc806..633cd9536 100644 --- a/.github/workflows/pinned-cert-audit.yml +++ b/.github/workflows/pinned-cert-audit.yml @@ -36,6 +36,8 @@ jobs: with: app-id: ${{ secrets.RELEASE_APP_ID }} private-key: ${{ secrets.RELEASE_APP_KEY }} + permission-contents: write + permission-pull-requests: write - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/scripts/gen_staticfs.py b/scripts/gen_staticfs.py index affbb3fec..5807c8a5e 100644 --- a/scripts/gen_staticfs.py +++ b/scripts/gen_staticfs.py @@ -90,7 +90,8 @@ def stage_assets(build_dir, staging_dir): shutil.copyfile(src, dst_root / name) else: # mtime=0: the gzip bytes depend only on the content - (dst_root / (name + '.gz')).write_bytes(gzip.compress(src.read_bytes(), mtime=0)) + with open(dst_root / (name + '.gz'), 'wb') as raw, gzip.GzipFile(filename='', mode='wb', fileobj=raw, mtime=0) as gz: + gz.write(src.read_bytes()) count += 1 log(f'Staged {count} gzipped assets into {www_dir}') From bcf6d852c8d12bd8f11a668b44b84bdf22f1e90c Mon Sep 17 00:00:00 2001 From: HeavenVR Date: Fri, 23 Oct 2026 15:27:23 +0200 Subject: [PATCH 5/5] chore(scripts): restore flatc flag comments in generate_schemas.py --- scripts/generate_schemas.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/generate_schemas.py b/scripts/generate_schemas.py index 51531375a..13b55cced 100644 --- a/scripts/generate_schemas.py +++ b/scripts/generate_schemas.py @@ -71,14 +71,24 @@ def main() -> int: if not schema_files: sys.exit(f'No .fbs schemas found in {schemas_path}') - ts_args = ['--ts', '--ts-no-import-ext'] + ts_args = [ + # Compile for TypeScript. + '--ts', + # Don't add .ts extension to imports. + '--ts-no-import-ext', + ] cpp_args = [ + # Compile for C++. '--cpp', + # Don't prefix enum values in generated C++ by their enum type. '--no-prefix', + # Use C++11 style scoped and strongly typed enums in generated C++. This also implies --no-prefix. '--scoped-enums', + # Generate type name functions for C++. '--gen-name-strings', - # (char* + length) string construction instead of std::string, for efficient custom string types. + # Don't construct custom string types by passing std::string from Flatbuffers, but (char* + length). This allows efficient construction of custom string types, including zero-copy construction. '--cpp-str-flex-ctor', + # use C++17 features in generated code (experimental). '--cpp-std', 'c++17', ]