diff --git a/.changes/alpine-in-the-build-stage-from.md b/.changes/alpine-in-the-build-stage-from.md new file mode 100644 index 0000000..6c204b7 --- /dev/null +++ b/.changes/alpine-in-the-build-stage-from.md @@ -0,0 +1,4 @@ +--- +type: changed +--- +Alpine in the build stage from 3.24.3 to 3.24.4 diff --git a/.changes/health-check-failing-with-database-is.md b/.changes/health-check-failing-with-database-is.md new file mode 100644 index 0000000..24053a2 --- /dev/null +++ b/.changes/health-check-failing-with-database-is.md @@ -0,0 +1,7 @@ +--- +type: fixed +--- +Health check failing with `database is locked` while an application writes, and making the writes of an application without a busy timeout fail + +- The health check reads every database with `immutable=1`, which takes no lock +- When that read fails, it opens the database as before, with a busy timeout of 3 seconds diff --git a/.changes/home-of-nonroot-from-var-lib.md b/.changes/home-of-nonroot-from-var-lib.md new file mode 100644 index 0000000..6b9541d --- /dev/null +++ b/.changes/home-of-nonroot-from-var-lib.md @@ -0,0 +1,7 @@ +--- +type: security +--- +Moved the home of `nonroot` from `/var/lib/sqlite` to `/home/nonroot`, `sqlite3` runs the `.sqliterc` of the home on every start, and every user can write `/var/lib/sqlite` + +- `sqlite3` reads its startup file from `/home/nonroot/.sqliterc`, which only `nonroot` can write +- The working directory stays `/var/lib/sqlite` diff --git a/.changes/image-labels-to-one-set-that.md b/.changes/image-labels-to-one-set-that.md new file mode 100644 index 0000000..42524f2 --- /dev/null +++ b/.changes/image-labels-to-one-set-that.md @@ -0,0 +1,9 @@ +--- +type: changed +--- +Image labels to one set that every image uses + +- `dev.peakimages.image` gains `distro`, `variant`, `flavor`, `shell` and `package-manager` +- `dev.peakimages.image.upstream.version` replaces `dev.peakimages.sqlite.version` +- `dev.peakimages.sqlite.extensions`, `dev.peakimages.tini.version` and `dev.peakimages.busybox.version` are removed as these labels do not scale to bigger images +- `org.opencontainers.image.licenses` names the license of the image and of SQLite, and `base.name` is empty with an empty `base.digest` for an image built from `scratch` diff --git a/.changes/package-metadata-instead-of-the-sbom-document.md b/.changes/package-metadata-instead-of-the-sbom-document.md new file mode 100644 index 0000000..e068d6e --- /dev/null +++ b/.changes/package-metadata-instead-of-the-sbom-document.md @@ -0,0 +1,10 @@ +--- +type: fixed +--- +Scanners listing the wrong packages and the wrong OS, `syft` found only `busybox 1.37.0`, `grype` reported two BusyBox CVEs Alpine has fixed and none of SQLite, and Trivy found nothing + +- `sqlite3` carries a package note and the image the apk package database records of tini and BusyBox, `syft` and `grype` list all three, Trivy all but SQLite +- A scanner reports the image as Alpine, the release tini and BusyBox are taken from, instead of BusyBox or no OS, the same way Docker Hardened Images does +- The SBOM attestation lists the same three packages as before, it was already correct, but a scanner does not read it and makes its own list from the image +- The syft document in `/var/lib/db/sbom` that BuildKit made the SBOM attestation from is gone, BuildKit runs syft, which reads the package note and the apk package database natively +- `syft` no longer needs the `--select-catalogers +sbom-cataloger` flag to read the custom SBOM in the syft JSON file, as its default list is now correct too diff --git a/.changes/vendor-and-authors-labels-from-peakimages.md b/.changes/vendor-and-authors-labels-from-peakimages.md new file mode 100644 index 0000000..9cc9119 --- /dev/null +++ b/.changes/vendor-and-authors-labels-from-peakimages.md @@ -0,0 +1,4 @@ +--- +type: changed +--- +Vendor and authors labels from `peakimages` to `PeakImages`, the name as it is written everywhere but in a URL or an image name diff --git a/.gitattributes b/.gitattributes index 8a01f67..e2545e1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,6 +3,7 @@ *.md diff=markdown CHANGELOG.md merge=union +/.changes export-ignore /.github export-ignore .editorconfig export-ignore .gitattributes export-ignore diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index b7ef44f..225e010 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -19,11 +19,11 @@ workflows: '.github/workflows/pull-request.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' - - 'docker/login-action@v4.7.0' + - 'docker/login-action@v4.8.0' '.github/workflows/release.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' - - 'docker/login-action@v4.7.0' + - 'docker/login-action@v4.8.0' '.github/workflows/tests.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' @@ -49,9 +49,9 @@ dependencies: commit: 'sha1-c3c9e263c25d99ce0380d002d59b67737d91b0dc' owner_id: 5429470 repo_id: 241092383 - 'docker/login-action@v4.7.0': - ref: 'v4.7.0' - commit: 'sha1-81c6f93d4bc991e510b342a00b419b3bb4efc5c6' + 'docker/login-action@v4.8.0': + ref: 'v4.8.0' + commit: 'sha1-8aa5831fb8d48d8f76594c7ddde61ae58019f550' owner_id: 5429470 repo_id: 287743349 'docker/setup-buildx-action@v4.4.1': diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml index 4a1ee32..1c42b5f 100644 --- a/.github/workflows/clean-manifests.yaml +++ b/.github/workflows/clean-manifests.yaml @@ -23,6 +23,7 @@ jobs: if: ${{ github.repository == 'peakimages/sqlite' }} permissions: contents: read + actions: write # disable this workflow after a failed run environment: name: cleanup deployment: false @@ -41,11 +42,19 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CLEANUP_TOKEN }} run: scripts/clean-manifests.sh --delete 200 - - name: Upload the list of deleted manifests + - name: Upload the lists of manifests if: ${{ always() }} uses: actions/upload-artifact@v7.0.2 with: - name: deleted-manifests - path: scripts/deleted-manifests.tsv + name: manifests + path: | + scripts/manifests-to-delete.tsv + scripts/deleted-manifests.tsv if-no-files-found: ignore retention-days: 90 + + - name: Disable this workflow after a failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ github.token }} + run: gh workflow disable clean-manifests.yaml --repo "$GITHUB_REPOSITORY" diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index d488618..3ce4d3a 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -67,7 +67,7 @@ jobs: path: ${{ runner.temp }}/images - name: Log in to ghcr.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f50019a..df1ebda 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,14 +88,14 @@ jobs: path: ${{ runner.temp }}/images - name: Log in to ghcr.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to docker.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -164,14 +164,14 @@ jobs: echo "tags=$tags" | tee -a "$GITHUB_OUTPUT" - name: Log in to ghcr.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to docker.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5322ecc..a1142c1 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -35,9 +35,9 @@ jobs: - name: Verify the action lockfile env: GH_TOKEN: ${{ github.token }} - # https://github.com/github/gh-actions-lock/releases/tag/v0.4.0 + # https://github.com/github/gh-actions-lock/releases/tag/v0.4.1 run: | - gh extension install github/gh-actions-lock --pin v0.4.0 + gh extension install github/gh-actions-lock --pin v0.4.1 gh actions-lock --verify test: diff --git a/README.md b/README.md index fa53581..06739e7 100644 --- a/README.md +++ b/README.md @@ -122,8 +122,36 @@ The defaults are set in the image, override them with `-e` or the `environment` | `SQLITE_PAGE_SIZE` | `4096` | Page size of a new database, a power of two from `512` to `65536`. **Fixed once the database has content** | | `SQLITE_TMPDIR` | unset | Directory for the temporary files of sorts and vacuums. Default `/tmp`, or the data directory when `/tmp` is not writable | +## Attestations + +Every image carries an SBOM and a provenance attestation. + +The SBOM lists the packages of the image: + +```sh +docker buildx imagetools inspect peakimages/sqlite:latest --format '{{ json .SBOM }}' +``` + +The provenance records how the image was built: + +```sh +docker buildx imagetools inspect peakimages/sqlite:latest --format '{{ json .Provenance }}' +``` + +A scanner lists the same packages, Trivy all but SQLite: + +```sh +syft peakimages/sqlite:latest +grype peakimages/sqlite:latest +trivy image peakimages/sqlite:latest +``` + ## Releases Every SQLite release becomes a new version of the image. A check for new SQLite versions runs twice a day. The image version follows semantic versioning independently of SQLite. Changes to the image are listed in [CHANGELOG.md](CHANGELOG.md). + +## Trademarks + +This image is built and published by PeakImages. All third-party product names, logos and trademarks are the property of their respective owners and are used solely for identification. PeakImages claims no interest in those marks, and no affiliation, sponsorship or endorsement is implied. diff --git a/RELEASE.md b/RELEASE.md index 80e9246..5264aaf 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -3,7 +3,7 @@ Every release is tagged `v-v`, such as `v3.54.1-v1.4.0-beta.1`. > [!IMPORTANT] -> A push that changes `release.json` or `src/upstream.json` publishes every release of it whose tag does not exist yet. Check what a push publishes with `scripts/plan-version.sh` first. +> A push that changes `release.json` or `src/upstream.json` releases the versions they set, unless they are released already. Run `scripts/plan-version.sh` first to see what a push releases. ## Rules @@ -17,6 +17,7 @@ Every release is tagged `v-v`, such as `v3.54.1-v | `v1.2.x` | backports and hotfixes, cut from the newest tag of v1.2 | stable `1.2.x` versions | - `main` takes fixes only. A new minor or major reaches it as a merge of `next` or a release branch +- Merge `next` or a release branch into `main` only right before you release it, on the same day. To freeze a minor for longer, publish an RC on `next` or cut `release/v1.4` - `next` and the release branches publish prereleases only, their stable versions are released on `main` - Name a branch after what it changes: `fix/`, `feat/`, and `fix/-v1.2.2` for a backport, after the version that releases it - Squash merge pull requests from contributors and pull requests with a single change, such as SQLite updates @@ -35,11 +36,12 @@ Every release is tagged `v-v`, such as `v3.54.1-v > [!NOTE] > `next` publishes prereleases only. For its stable version, merge `next` into `main` with a merge commit and create the release on `main`. -1. Describe the changes under `## [Unreleased]` in `CHANGELOG.md`, they become the release notes -2. Check what the branch releases: +1. Describe the changes in change files with `scripts/change.sh`, they become the release notes, see [Change files](#change-files) +2. Check what the branch releases and the entries it lists: ```sh scripts/plan-version.sh + scripts/change.sh --notes ``` 3. Select the stage and the version, the script commits them and asks to push: @@ -57,6 +59,27 @@ Every release is tagged `v-v`, such as `v3.54.1-v > scripts/undo-release.sh > ``` +## Change files + +Every change to the image has a file in `.changes/`, `AGENTS.md` describes the types and the wording: + +```text +--- +type: fixed +commit: 1a2b3c4 +--- +Missing description and license on ghcr.io + +- The release index carries the labels of the image as annotations +``` + +- `scripts/change.sh` writes a file, it asks for what its flags leave out, and `--notes` prints the entries of the next release +- `scripts/release.sh` preselects the version the files need and writes them into `CHANGELOG.md`, the release workflow reads its notes from there +- A prerelease writes the files under `## [Unreleased]`, keeps them and adds its version to the files that have none +- A stable release writes the files under `## [v1.4.0] - ` and removes them +- `scripts/undo-release.sh` brings the files back as they were +- A new SQLite version needs no file, the release workflow writes its notes + ## Backport a fix `main` is on 1.3, and 1.1 and 1.2 still get fixes. A fix reaches each of them through a patch branch and a pull request of its own: @@ -90,11 +113,11 @@ Every release is tagged `v-v`, such as `v3.54.1-v 1. Pin the line on a branch from `main`: ```sh - git switch -c feat/sqlite-3.53 origin/main && scripts/check-upstream.sh 3.53 + git switch -c feat/sqlite-3.53 origin/main && scripts/check-upstream.sh v3.53 ``` 2. Squash merge a pull request as `feat(release): add the SQLite 3.53 line`, the push publishes `v3.53.4-v1.3.0` -3. To stop the line, remove `3.53` from `src/upstream.json`, its tags stay where they are +3. To stop the line, remove `v3.53` from `src/upstream.json`, its tags stay where they are ## Release a major version @@ -130,8 +153,11 @@ Every release is tagged `v-v`, such as `v3.54.1-v ## Pull requests -The pull request workflow builds every pull request and publishes the image of the line `latest` of one from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`, versioned `-pr.`: pull request 1234 into `next` is `1.4.0-beta.1-pr.1234` after `1.4.0-beta.1` and `1.5.0-alpha.0-pr.1234` before the first alpha of 1.5.0, into `main` `1.3.4-pr.1234`. PHP's `version_compare` sorts it after its base and before the next prerelease or release. +The pull request workflow builds every pull request and publishes the image of the line `latest` of a pull request from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`. Its version is `-pr.`: +- Into `next`: `1.4.0-beta.1-pr.1234` after `1.4.0-beta.1`, or `1.5.0-alpha.0-pr.1234` before the first alpha of 1.5.0 +- Into `main`: `1.3.4-pr.1234` +- PHP's `version_compare` sorts it after its base and before the next prerelease or release - Pull requests from forks and Dependabot are built, not published - A push to a pull request from `main`, `next`, a release branch or a patch branch that changes `release.json` or `src/upstream.json` is not built, the release workflow builds it - ghcr.io only, no git tag and no GitHub release, the clean manifests workflow keeps the image while the pull request is open diff --git a/scripts/change.sh b/scripts/change.sh new file mode 100755 index 0000000..d8ac894 --- /dev/null +++ b/scripts/change.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +CHANGE_DIRECTORY=.changes +TYPES='added,changed,deprecated,removed,fixed,security,refactored,maintenance' +SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Refactored,Maintenance' + +BOLD=$'\033[1m' GREEN=$'\033[32m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' + +heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} +confirm() { + local answer + printf '\n %s [y/N] ' "$*" + read -r answer + [[ "$answer" =~ ^[Yy]$ ]] +} + +cd "$(dirname "$0")/.." +trap 'tput cnorm 2> /dev/null || true' EXIT + +main() { + read_arguments "$@" + if [ -n "$notes" ]; then + print_notes + return + fi + [ -n "$kind" ] && [ -n "$summary" ] || ask_questions + write_change +} + +read_arguments() { + kind='' summary='' commit='' breaking='' notes='' details=() + while [ $# -gt 0 ]; do + case "$1" in + --notes) notes=true ;; + --breaking) breaking=true ;; + --type | --summary | --detail | --commit) + [ $# -ge 2 ] || die "$1 has no value, expected one" + case "$1" in + --type) kind="$2" ;; + --summary) summary="$2" ;; + --detail) details+=("$2") ;; + --commit) commit="$2" ;; + esac + shift + ;; + *) die "'$1' is not an option, expected --type, --summary, --detail, --breaking, --commit or --notes" ;; + esac + shift + done +} + +ask_questions() { + local selected detail types + [ -t 0 ] || die "--type and --summary are required without a terminal" + if [ -z "$kind" ]; then + heading "Select the type" + IFS=, read -ra types <<< "$TYPES" + selected="$(menu 0 "${types[@]}")" + kind="${types[$selected]}" + fi + if [ -z "$summary" ]; then + heading "Summary, one sentence with the heading" + printf ' ' + read -r summary + fi + heading "Details, an empty line ends them" + while printf ' - ' && read -r detail && [ -n "$detail" ]; do + details+=("$detail") + done + if confirm "Breaking change?"; then + breaking=true + fi + printf '\n' +} + +write_change() { + local path detail + [[ ",$TYPES," == *",$kind,"* ]] || die "the type is '$kind', expected ${TYPES//,/, }" + [ -n "$summary" ] || die "the summary is empty, expected one sentence" + if [ -n "$commit" ]; then + git rev-parse --verify --quiet "$commit^{commit}" > /dev/null || die "$commit is not a commit of this repository" + commit="$(git rev-parse --short "$commit")" + fi + mkdir -p "$CHANGE_DIRECTORY" + path="$CHANGE_DIRECTORY/$(file_name).md" + { + printf -- '---\ntype: %s\n' "$kind" + [ -z "$breaking" ] || printf 'breaking: true\n' + [ -z "$commit" ] || printf 'commit: %s\n' "$commit" + printf -- '---\n%s\n' "$summary" + [ "${#details[@]}" = 0 ] || printf '\n' + for detail in ${details[@]+"${details[@]}"}; do + printf -- '- %s\n' "$detail" + done + } > "$path" + ok "$path describes the $BOLD$kind$RESET change" +} + +print_notes() { + local section entries sections + compgen -G "$CHANGE_DIRECTORY/*.md" > /dev/null || die "$CHANGE_DIRECTORY has no change files" + IFS=, read -ra sections <<< "$SECTIONS" + for section in "${sections[@]}"; do + entries="$(awk -v section="$section" ' + FNR == 1 { fences = 0; type = ""; breaking = 0 } + fences < 2 { + if ($0 == "---") fences++ + else if ($1 == "type:") type = $2 + else if ($0 == "breaking: true") breaking = 1 + next + } + NF && tolower(section) == (breaking ? "breaking changes" : type) { + prefix = /^- / ? " " : "- " + print prefix $0 + } + ' "$CHANGE_DIRECTORY"/*.md)" + [ -z "$entries" ] || printf '### %s\n\n%s\n\n' "$section" "$entries" + done +} + +file_name() { + local stem name number=2 + stem="$(printf '%s' "$summary" | LC_ALL=C tr -cs '[:alnum:]' '\n' | LC_ALL=C tr '[:upper:]' '[:lower:]' | awk 'NF && ++count <= 6' | paste -sd - -)" + name="$stem" + while [ -e "$CHANGE_DIRECTORY/$name.md" ]; do + name="$stem-$number" + number=$((number + 1)) + done + echo "$name" +} + +menu() { + local selected="$1" options key i + shift + options=("$@") + tput civis >&2 2> /dev/null || true + while true; do + for i in "${!options[@]}"; do + if [ "$i" = "$selected" ]; then + printf ' %s❯ %s%s\n' "$GREEN" "${options[$i]}" "$RESET" >&2 + else + printf ' %s %s%s\n' "$DIM" "${options[$i]}" "$RESET" >&2 + fi + done + IFS= read -rsn1 key + if [ "$key" = $'\x1b' ]; then + IFS= read -rsn2 key + if [ "$key" = '[A' ] && [ "$selected" -gt 0 ]; then + selected=$((selected - 1)) + elif [ "$key" = '[B' ] && [ "$selected" -lt $((${#options[@]} - 1)) ]; then + selected=$((selected + 1)) + fi + elif [ -z "$key" ]; then + break + fi + printf '\033[%dA\033[J' "${#options[@]}" >&2 + done + tput cnorm >&2 2> /dev/null || true + echo "$selected" +} + +main "$@" diff --git a/scripts/check-upstream.sh b/scripts/check-upstream.sh index 61944c2..b26c73f 100755 --- a/scripts/check-upstream.sh +++ b/scripts/check-upstream.sh @@ -19,7 +19,7 @@ cd "$(dirname "$0")/.." trap 'rm -rf "$WORK_DIR"' EXIT main() { - [[ "$OPTION" =~ ^(latest|[0-9]+(\.[0-9]+)*)?$ ]] || die "'$OPTION' is not an option, expected a line such as 3.53, latest or none" + [[ "$OPTION" =~ ^(latest|v[0-9]+(\.[0-9]+)?)?$ ]] || die "'$OPTION' is not an option, expected a line such as v3.53, latest or none" read_newest_release compare_with_pinned download_amalgamation @@ -34,7 +34,7 @@ read_newest_release() { local release year number fetch "$WORK_DIR/chronology.html" https://sqlite.org/chronology.html release="$(sed -nE "s|.*>([0-9]{4})-[0-9]{2}-[0-9]{2}\$|year \1|p; s|.*data-sortkey='([0-9]+)'>($VERSION_PATTERN).*|version \1 \2|p" "$WORK_DIR/chronology.html" \ - | awk -v line="$LINE." '$1 == "year" { year = $2 } $1 == "version" && (line == "latest." || index($3 ".", line) == 1) { print year, $2, $3 }' \ + | awk -v line="${LINE#v}." '$1 == "year" { year = $2 } $1 == "version" && (line == "latest." || index($3 ".", line) == 1) { print year, $2, $3 }' \ | sort -k3,3V | tail -n 1)" [ -n "$release" ] || die "sqlite.org/chronology.html lists no SQLite $LINE release" read -r year number new_version <<< "$release" diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index 964f394..7423473 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -4,10 +4,14 @@ set -euo pipefail OPTION="$*" LIMIT="${2:-}" IMAGE=peakimages/sqlite +VARIANTS="" KEEP_TAG_PATTERN='^(latest|preview|[0-9]+(\.[0-9]+){0,2}|[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?)$' KEEP_TAG_EXCEPTIONS="" +MIN_AGE_DAYS=1 DOCKERHUB_USERNAME="${DOCKERHUB_USERNAME:-}" DOCKERHUB_TOKEN="${DOCKERHUB_TOKEN:-}" +GHCR_TOKEN="" +MANIFEST_TYPES='application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' TO_DELETE_FILE=scripts/manifests-to-delete.tsv DELETED_FILE=scripts/deleted-manifests.tsv WORK_DIR="$(mktemp -d)" @@ -28,14 +32,17 @@ trap 'rm -rf "$WORK_DIR"' EXIT main() { [[ "$OPTION" =~ ^(--delete [1-9][0-9]*)?$ ]] || die "'$OPTION' is not an option, expected --delete with a count or none" + [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to read docker.io/$IMAGE" + GHCR_TOKEN="$(gh auth token)" || die "gh is not signed in with a token that has read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" heading "Reading ghcr.io/$IMAGE and docker.io/$IMAGE" read_ghcr_versions read_dockerhub_tags + read_dockerhub_manifests_deleted_on_ghcr + read_dockerhub_untagged_manifests select_kept_tags read_manifests select_kept_manifests heading "Selecting what to delete" - find_dockerhub_only_manifests write_list delete_manifests verify_kept_manifests @@ -48,134 +55,186 @@ read_ghcr_versions() { list_ghcr_versions "state=deleted&page=$page" done | awk -v window="$window" 'NR <= window' > "$WORK_DIR/ghcr-deleted.tsv" cut -f 1 "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/on-ghcr.txt" - ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and $(count < "$WORK_DIR/ghcr-deleted.tsv") of the versions it deleted last are checked on docker.io, twice as many as the run deletes" + ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged" } read_dockerhub_tags() { - local token page last="" - token="$(registry_token docker.io pull)" - while :; do - page="$(curl -sSf --retry 3 -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/tags/list?n=1000${last:+&last=$last}" | jq -r '.tags // [] | .[]')" \ - || die "could not list the tags of docker.io/$IMAGE" - [ -n "$page" ] || break - echo "$page" >> "$WORK_DIR/dockerhub-tags.txt" - last="$(tail -n 1 <<< "$page")" + local hub url page + hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME" + url="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags?page_size=100" + : > "$WORK_DIR/hub-tags.tsv" + : > "$WORK_DIR/hub-images.tsv" + : > "$WORK_DIR/dockerhub-images.tsv" + while [ -n "$url" ]; do + page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" || die "could not list the tags of docker.io/$IMAGE from the Docker Hub API" + jq -r '.results[] | [.name, (.digest // ""), (.media_type // ""), (.tag_last_pushed // .last_updated // ""), (.full_size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-tags.tsv" + jq -r '.results[] | .name as $tag | .images[]? | select(.digest) | [$tag, .digest, ([.os, .architecture] | join("/")), (.last_pushed // ""), (.size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-images.tsv" + url="$(jq -r '.next // empty' <<< "$page")" done - request_manifests docker.io "$WORK_DIR/dockerhub-tags.txt" -f --head -o /dev/null -w '%header{docker-content-digest}\t%header{content-type}\t{}\n' \ - > "$WORK_DIR/dockerhub.tsv" || die "could not read the digests of the tags of docker.io/$IMAGE" - ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(cut -f 1 "$WORK_DIR/dockerhub.tsv" | sort -u | count) manifests" + awk -F '\t' '$2 == "" || $3 == "" { print $1 }' "$WORK_DIR/hub-tags.tsv" > "$WORK_DIR/old-tags.txt" + request_manifests docker.io "$WORK_DIR/old-tags.txt" -f --head -o /dev/null -w '{}\t%header{docker-content-digest}\t%header{content-type}\n' > "$WORK_DIR/old-tags.tsv" \ + || die "could not read the digests of the old tags of docker.io/$IMAGE, the Docker Hub API lists them without one" + awk -F '\t' -v OFS='\t' -v images="$WORK_DIR/dockerhub-images.tsv" ' + FILENAME == ARGV[1] { old_digest[$1] = $2; old_type[$1] = $3; next } + FILENAME == ARGV[2] { if ($2 == "") $2 = old_digest[$1]; if ($3 == "") $3 = old_type[$1]; root[$1] = $2; tags[$2] = ($2 in tags) ? tags[$2] "," $1 : $1; type[$2] = $3; if ($4 > pushed[$2]) pushed[$2] = $4; next } + $2 == root[$1] { platform[$2] = $3; size[$2] = $5; next } + !((root[$1], $2) in pair) { pair[root[$1], $2]; print root[$1], $2 > images } + !($2 in tags) && !seen[$2]++ { print $2, "", "", $4, $5, $3 } + END { for (d in tags) print d, type[d], tags[d], pushed[d], (type[d] ~ /index|list/ ? 0 : size[d] + 0), platform[d] }' \ + "$WORK_DIR/old-tags.tsv" "$WORK_DIR/hub-tags.tsv" "$WORK_DIR/hub-images.tsv" > "$WORK_DIR/dockerhub-manifests.tsv" + awk -F '\t' -v OFS='\t' '$3 != "" { n = split($3, tags, ","); for (i = 1; i <= n; i++) print $1, $2, tags[i] }' "$WORK_DIR/dockerhub-manifests.tsv" > "$WORK_DIR/dockerhub.tsv" + grep -q $'\tlatest$' "$WORK_DIR/dockerhub.tsv" || die "docker.io/$IMAGE lists no latest tag, expected the Docker Hub API to return every tag" + awk -F '\t' '$1 !~ /^sha256:/ || $4 == "" { exit 1 }' "$WORK_DIR/dockerhub-manifests.tsv" || die "docker.io/$IMAGE lists manifests without a digest or push date, expected both from the Docker Hub API" + ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(awk -F '\t' '$3 != ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) manifests, which reference $(awk -F '\t' '$3 == ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) more" +} + +read_dockerhub_manifests_deleted_on_ghcr() { + awk -F '\t' 'FILENAME == ARGV[1] { listed[$1]; next } !($1 in listed) && !seen[$1]++ { print $1 }' "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/ghcr-deleted.tsv" > "$WORK_DIR/deleted-on-ghcr.txt" + request_manifests docker.io "$WORK_DIR/deleted-on-ghcr.txt" --head -o /dev/null -w '%{http_code}\t{}\t%header{content-type}\n' > "$WORK_DIR/deleted-on-ghcr.tsv" \ + || die "could not check which of the versions that ghcr.io deleted last docker.io still has" + awk '$1 != 200 && $1 != 404 { exit 1 }' "$WORK_DIR/deleted-on-ghcr.tsv" || die "docker.io answered neither 200 nor 404 for a version that ghcr.io deleted last" + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { created[$1] = $4; next } $1 == 200 { print $2, $3, "", created[$2], "", "" }' "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/deleted-on-ghcr.tsv" >> "$WORK_DIR/dockerhub-manifests.tsv" + ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/deleted-on-ghcr.tsv" | count) of the $(count < "$WORK_DIR/ghcr-deleted.tsv") versions that ghcr.io deleted last and no tag references" +} + +# The manifests endpoint is undocumented, Docker Hub uses it for image management, so it can change without notice. It only adds the untagged manifests that no tag references +read_dockerhub_untagged_manifests() { + local hub base url page key + hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME" + base="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/manifests?page_size=100" + url="$base" + : > "$WORK_DIR/listing.tsv" + while [ -n "$url" ]; do + if ! page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" \ + || ! jq -r '.manifests[] | [.manifest_digest, .media_type, (.tags // [] | join(",")), .last_pushed, (if (.media_type // "" | test("index|list")) then .size else .total_size end), ([.os, .arch] | map(select(. != null and . != "")) | join("/"))] | @tsv' <<< "$page" >> "$WORK_DIR/listing.tsv"; then + warn "the undocumented manifests listing of the Docker Hub API failed, the untagged manifests that no tag references wait for a later run" + return + fi + key="$(jq -r 'select(.manifests | length > 0) | .last_evaluated_key // empty' <<< "$page")" + url="${key:+$base&last_evaluated_key=$(jq -rn --arg key "$key" '$key | @uri')}" + done + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { known[$1]; next } !($1 in known) && $3 == "" { print $1, $2, "", $4, $5, $6 }' "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/listing.tsv" > "$WORK_DIR/untagged.tsv" + cat "$WORK_DIR/untagged.tsv" >> "$WORK_DIR/dockerhub-manifests.tsv" + ok "the undocumented manifests listing adds $(count < "$WORK_DIR/untagged.tsv") untagged manifests that no tag references" } select_kept_tags() { { cut -f 3 "$WORK_DIR/ghcr-active.tsv" | tr ',' '\n' && cut -f 3 "$WORK_DIR/dockerhub.tsv"; } | sed '/^$/d' | sort -u > "$WORK_DIR/tags.txt" git ls-remote --tags "https://github.com/$IMAGE.git" | sed -n 's|.*refs/tags/||p' | sed 's/\^{}$//; p; s/^v//' > "$WORK_DIR/keep.txt" \ || die "could not list the git tags of github.com/$IMAGE" - gh pr list --repo "$IMAGE" --state open --limit 1000 --json number --jq '.[] | "pr-\(.number)"' >> "$WORK_DIR/keep.txt" \ + gh pr list --repo "$IMAGE" --state open --limit 1000 --json number --jq '.[] | "pr-\(.number)"' > "$WORK_DIR/pull-requests.txt" \ || die "gh could not list the open pull requests of $IMAGE" - tr ' ' '\n' <<< "$KEEP_TAG_EXCEPTIONS" >> "$WORK_DIR/keep.txt" + awk -v variants="$VARIANTS" '{ print; n = split(variants, variant, ","); for (i = 1; i <= n; i++) print $0 "-" variant[i] }' "$WORK_DIR/pull-requests.txt" >> "$WORK_DIR/keep.txt" + tr ',' '\n' <<< "$KEEP_TAG_EXCEPTIONS" >> "$WORK_DIR/keep.txt" { grep -E "$KEEP_TAG_PATTERN" "$WORK_DIR/tags.txt" || true grep -xF -f "$WORK_DIR/keep.txt" "$WORK_DIR/tags.txt" || true } | sort -u > "$WORK_DIR/kept-tags.txt" grep -qx latest "$WORK_DIR/kept-tags.txt" || die "found no latest tag, expected it among the tags to keep" - ok "keeping $(count < "$WORK_DIR/kept-tags.txt") tags: every tag KEEP_TAG_PATTERN matches or KEEP_TAG_EXCEPTIONS lists, every git tag of github.com/$IMAGE and the tags of open pull requests" + ok "keeping $(count < "$WORK_DIR/kept-tags.txt") tags: the ones KEEP_TAG_PATTERN matches or KEEP_TAG_EXCEPTIONS lists, the git tags of github.com/$IMAGE and the tags of open pull requests" } read_manifests() { - local digest tag mkdir "$WORK_DIR/manifests" request_manifests ghcr.io "$WORK_DIR/on-ghcr.txt" -f -o "$WORK_DIR/manifests/{}" -w '{}\t%{size_download}\n' > "$WORK_DIR/bytes.tsv" \ || die "could not read the manifests of ghcr.io/$IMAGE" - awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && !seen[$1]++ { print $1, $2, $3 }' \ - "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-roots.tsv" - awk -F '\t' '$2 ~ /oci\.image\.index/ { print $1 }' "$WORK_DIR/dockerhub-roots.tsv" > "$WORK_DIR/dockerhub-indexes.txt" - request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the indexes kept tags point at on docker.io" - while IFS=$'\t' read -r digest tag; do - curl -sSf --retry 3 "https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags/$tag" | jq -r --arg root "$digest" '.images[] | [$root, .digest] | @tsv' \ - || die "could not read the manifests $tag points at on docker.io" - done < <(awk -F '\t' -v OFS='\t' '$2 !~ /oci\.image\.index/ { print $1, $3 }' "$WORK_DIR/dockerhub-roots.tsv") > "$WORK_DIR/children.tsv" + awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && $2 ~ /oci\.image\.index/ && !seen[$1]++ { print $1 }' \ + "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-indexes.txt" + request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the kept indexes that only docker.io has" manifest_fields '.manifests[]? | [manifest, .digest, (.platform | [.os, .architecture, .variant] | map(values) | join("/")), .annotations["vnd.docker.reference.digest"]]' \ - >> "$WORK_DIR/children.tsv" + > "$WORK_DIR/children.tsv" manifest_fields '(.config // empty), .layers[]? | [manifest, .digest, .size]' > "$WORK_DIR/blobs.tsv" manifest_fields '[manifest, if .manifests then "index" elif any(.layers[]?; .mediaType == "application/vnd.in-toto+json") then "attestation" else "image" end]' \ > "$WORK_DIR/kinds.tsv" - ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests of ghcr.io/$IMAGE and the $(count < "$WORK_DIR/dockerhub-roots.tsv") that kept tags point at on docker.io alone" + ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests on ghcr.io and the $(count < "$WORK_DIR/dockerhub-indexes.txt") kept indexes that only docker.io has, each with a GET that counts as a pull" } select_kept_manifests() { local kinds - kept_manifests "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/kept-ghcr.io.txt" - kept_manifests "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/kept-docker.io.txt" + manifests_of_kept_tags "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/kept-ghcr.io.txt" + manifests_of_kept_tags "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/kept-docker.io.txt" sort -u "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kept-docker.io.txt" > "$WORK_DIR/kept.txt" awk 'FILENAME == ARGV[1] { ghcr[$1]; next } !($1 in ghcr) { exit 1 }' "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/kept-ghcr.io.txt" \ - || die "manifests the kept tags point at are missing from the versions gh listed on ghcr.io, expected every one of them" - kinds="$(awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } ($1 in kept) { count[$2]++ } - END { printf "%d indexes, %d images and %d attestations", count["index"], count["image"], count["attestation"] }' "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kinds.tsv")" - ok "keeping $(count < "$WORK_DIR/kept-ghcr.io.txt") manifests on ghcr.io, $kinds, and $(count < "$WORK_DIR/kept-docker.io.txt") on docker.io: the kept tags and every manifest they point at" -} - -find_dockerhub_only_manifests() { - awk -F '\t' 'FILENAME == ARGV[1] || FILENAME == ARGV[2] { skip[$1]; next } !($1 in skip) && !seen[$1]++ { print $1 }' \ - "$WORK_DIR/kept.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" "$WORK_DIR/ghcr-deleted.tsv" > "$WORK_DIR/candidates.txt" - request_manifests docker.io "$WORK_DIR/candidates.txt" --head -o /dev/null -w '%{http_code}\t{}\t%header{content-type}\n' \ - > "$WORK_DIR/dockerhub-only.tsv" || die "could not check which manifests docker.io has" - awk '$1 != 200 && $1 != 404 { exit 1 }' "$WORK_DIR/dockerhub-only.tsv" \ - || die "could not check docker.io for the manifests ghcr.io does not have, expected 200 or 404" - ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/dockerhub-only.tsv" | count) of the $(count < "$WORK_DIR/candidates.txt") unused manifests ghcr.io does not have" + || die "ghcr.io lists no version for some kept manifests, expected one for each" + kinds="$(awk -F '\t' 'function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } + FILENAME == ARGV[1] { kept[$1]; next } ($1 in kept) { count[$2]++ } + END { print amount(count["index"], "index", "indexes") ", " amount(count["image"], "image", "images") ", " amount(count["attestation"], "attestation", "attestations") }' "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kinds.tsv")" + ok "keeping $(count < "$WORK_DIR/kept-ghcr.io.txt") manifests on ghcr.io ($kinds) and $(count < "$WORK_DIR/kept-docker.io.txt") on docker.io" } write_list() { - awk -F '\t' -v OFS='\t' 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } + local cutoff + cutoff="$(jq -nr --argjson days "$MIN_AGE_DAYS" 'now - $days * 86400 | strftime("%Y-%m-%dT%H:%M:%SZ")')" + ok "keeping the $(awk -F '\t' -v cutoff="$cutoff" '$4 >= cutoff && !seen[$1]++' "$WORK_DIR/ghcr-active.tsv" | count) versions on ghcr.io and the $(awk -F '\t' -v cutoff="$cutoff" '$4 >= cutoff' "$WORK_DIR/dockerhub-manifests.tsv" | count) manifests on docker.io pushed since $cutoff, a running build may still need them" + awk -F '\t' -v OFS='\t' -v cutoff="$cutoff" 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { kind[$1] = $2; next } FILENAME == ARGV[3] { size[$1] = $2; next } FILENAME == ARGV[4] { size[$1] += $3; next } FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next } - FILENAME == ARGV[6] { tags[$1] = tags[$1] (tags[$1] == "" ? "" : ",") $3; next } - FILENAME == ARGV[7] { version[$1] = $2; created[$1] = $4; next } - FILENAME == ARGV[8] { if ($1 == 200) print "no", "yes", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } - !($1 in kept) { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ - "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ - "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k1,1 -k6,6 > "$WORK_DIR/rows.tsv" + FILENAME == ARGV[6] { + on_docker_hub[$1] + if (!($1 in kept) && !($1 in kind) && $4 != "" && $4 < cutoff && !seen[$1]++) + print "no", "yes", ($2 ~ /index|list/ ? "index" : ($6 == "unknown/unknown" ? "attestation" : "image")), ($6 == "unknown/unknown" ? "" : $6), $3, $4, $5, "", $1 + next + } + !($1 in kept) && $4 < cutoff && !seen[$1]++ { print "yes", (($1 in on_docker_hub) ? "yes" : "no"), kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ + "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/ghcr-active.tsv" \ + | sort -t $'\t' -k3,3r -k6,6 > "$WORK_DIR/sorted.tsv" + awk -F '\t' 'function emit(digest, list, n, i) { printed[digest]; print line[digest]; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) if ((list[i] in line) && !(list[i] in printed) && --parents[list[i]] == 0) emit(list[i]) } + FILENAME == ARGV[1] || FILENAME == ARGV[2] { if (!(($1, $2) in pair)) { pair[$1, $2]; children[$1] = children[$1] " " $2 } next } + { order[++rows] = $9; line[$9] = $0; type[$9] = $3 } + END { + for (i = 1; i <= rows; i++) if (order[i] in children) { n = split(children[order[i]], list, " "); for (j = 1; j <= n; j++) if (list[j] in line) parents[list[j]]++ } + for (i = 1; i <= rows; i++) if (type[order[i]] == "index" && !(order[i] in children)) emit(order[i]) + for (i = 1; i <= rows; i++) if (type[order[i]] == "image" && !(order[i] in parents) && !(order[i] in printed)) emit(order[i]) + for (i = 1; i <= rows; i++) if (!(order[i] in printed)) emit(order[i]) + }' "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-images.tsv" "$WORK_DIR/sorted.tsv" > "$WORK_DIR/rows.tsv" write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" - ok "$(awk -F '\t' 'END { print $9 " to delete, " $3 ", they free " $7 }' "$TO_DELETE_FILE")" - ok "$(awk -F '\t' '$1 == "no"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" - info "$TO_DELETE_FILE lists every manifest to delete with its type, platform, tags, date and size, indexes first" + ok "$(awk -F '\t' 'END { print ($9 + 0 ? $9 " to delete (" $3 "), they free " $7 : "nothing to delete") }' "$TO_DELETE_FILE")" + info "$TO_DELETE_FILE lists every manifest to delete in the order it deletes them, with its type, platform, tags, date and size" } delete_manifests() { - local on_ghcr type tags version digest rest name deleted ghcr docker_hub status + local on_ghcr on_docker_hub type tags version digest rest name deleted ghcr docker_hub status gone=0 if [ -z "$LIMIT" ]; then - info "nothing is deleted, a run with --delete and a count deletes that many from the top of the list" + info "a dry run deletes nothing, --delete with a count deletes that many manifests from the top of the list" return fi - [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to delete manifests on docker.io" heading "Deleting" - write_table /dev/null "$DELETED_FILE" - while IFS='|' read -r on_ghcr type tags version digest rest; do + head -n 1 "$TO_DELETE_FILE" > "$DELETED_FILE" + : > "$WORK_DIR/deleted.tsv" + while IFS='|' read -r on_ghcr on_docker_hub type tags version digest rest; do + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || die "the list names '$digest', expected a sha256 digest" name="$digest${tags:+ ($tags)}" - deleted="" ghcr=no docker_hub=no + deleted="" ghcr=no docker_hub=no status="" if [ "$on_ghcr" = yes ]; then delete_ghcr_version "$version" "$type $name" || continue deleted=ghcr.io ghcr=yes fi - status="$(delete_dockerhub_manifest "$digest")" - if [[ "$status" == 2?? ]]; then - deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + if [ "$on_docker_hub" = yes ]; then + status="$(delete_dockerhub_manifest "$digest")" + if [[ "$status" == 2?? ]]; then + deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + elif [ "$status" = 404 ]; then + info "docker.io had already deleted the $type $BOLD$name$RESET together with its index" + gone=$((gone + 1)) + fi fi if [ -n "$deleted" ]; then ok "deleted the $type $BOLD$name$RESET from $deleted" - printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" >> "$WORK_DIR/deleted.tsv" - write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" | tee -a "$WORK_DIR/deleted.tsv" >> "$DELETED_FILE" fi case "$status" in - 2?? | 404) ;; + "" | 2?? | 404) ;; 403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;; *) die "could not delete the $type $name from docker.io, it answered $status" ;; esac - done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $3, $5, $8, $9, rest }') - ok "$(awk -F '\t' 'END { print "deleted " $9 ", " $3 ", they free " $7 }' "$DELETED_FILE")" - info "$DELETED_FILE lists what this run deleted and where, ghcr.io restores it for 30 days, docker.io deletes for good" + done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $2, $3, $5, $8, $9, rest }') + write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + ok "$(awk -F '\t' 'END { print ($9 + 0 ? "deleted " $9 " (" $3 "), they free " $7 : "deleted nothing") }' "$DELETED_FILE")" + [ "$gone" = 0 ] || info "docker.io had already deleted $gone of the listed manifests together with their index, it deletes the children of an index that nothing else references" + info "$DELETED_FILE lists what this run deleted and where, ghcr.io can restore them for 30 days, docker.io deletes them permanently" } verify_kept_manifests() { @@ -184,9 +243,9 @@ verify_kept_manifests() { for registry in ghcr.io docker.io; do missing="$(request_manifests "$registry" "$WORK_DIR/kept-$registry.txt" --head -o /dev/null -w '%{http_code}\t{}\n' | awk '$1 != 200 { print $2 }')" \ || die "could not check the kept manifests on $registry" - [ -z "$missing" ] || die "$(count <<< "$missing") manifests the kept tags point at are gone from $registry, expected all of them: $(head -n 3 <<< "$missing" | paste -sd ' ' -)" + [ -z "$missing" ] || die "$(count <<< "$missing") kept manifests are gone from $registry: $(head -n 3 <<< "$missing" | paste -sd ' ' -)" done - ok "every kept tag and every manifest it points at is still on ghcr.io and docker.io" + ok "every kept manifest is still on ghcr.io and docker.io" } list_ghcr_versions() { @@ -194,7 +253,7 @@ list_ghcr_versions() { shift gh api "$@" "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&$query" \ --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' 2> /dev/null \ - || die "gh could not list the versions of ghcr.io/$IMAGE with $query, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" + || die "gh could not list the versions of ghcr.io/$IMAGE, its token needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" } request_manifests() { @@ -204,9 +263,7 @@ request_manifests() { split -l 1000 "$list" "$WORK_DIR/chunk-" for chunk in "$WORK_DIR"/chunk-*; do token="$(registry_token "$registry" pull)" - xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" \ - -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ - "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 + xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" -H "Accept: $MANIFEST_TYPES" "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 rm "$chunk" done } @@ -215,11 +272,11 @@ manifest_fields() { find "$WORK_DIR/manifests" -type f -exec jq -r 'def manifest: input_filename | sub(".*/"; ""); '"$1"' | @tsv' {} + } -kept_manifests() { +manifests_of_kept_tags() { awk -F '\t' 'function keep(digest, list, n, i) { if (digest in kept) return; kept[digest]; print digest; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) keep(list[i]) } - FILENAME == ARGV[1] { children[$1] = children[$1] " " $2; next } - FILENAME == ARGV[2] { tag[$1]; next } - { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/kept-tags.txt" "$1" + FILENAME == ARGV[1] || FILENAME == ARGV[2] { children[$1] = children[$1] " " $2; next } + FILENAME == ARGV[3] { tag[$1]; next } + { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-images.tsv" "$WORK_DIR/kept-tags.txt" "$1" } write_table() { @@ -227,22 +284,26 @@ write_table() { function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } BEGIN { split("B kB MB GB TB", units, " "); print "ghcr", "docker_hub", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } FILENAME == ARGV[1] { removed[$9]; types[$3]++; manifests++; if ($7 != "") $7 = human($7); print; next } - FILENAME == ARGV[2] { before += $2; if (!($1 in removed)) after += $2; next } - { blob[$2] = $3; if (!($1 in removed)) used[$2] } + FILENAME == ARGV[2] { bytes[$1] = $2; before += $2; if (!($1 in removed)) after += $2; next } + FILENAME == ARGV[3] { if ($1 in bytes) on_hub[$1]; size = ($1 in bytes) ? bytes[$1] : $5; hub_before += size; if (!($1 in removed)) hub_after += size; next } + { blob[$2] = $3; if (!($1 in removed)) used[$2]; if ($1 in on_hub) { hub_blob[$2] = $3; if (!($1 in removed)) hub_used[$2] } } END { for (digest in blob) { before += blob[digest]; if (digest in used) after += blob[digest] } + for (digest in hub_blob) { hub_before += hub_blob[digest]; if (digest in hub_used) hub_after += hub_blob[digest] } print "total", "", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), - "", "", "", human(before - after) ", " human(before) " → " human(after) " on ghcr.io", "", amount(manifests, "manifest", "manifests") - }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" > "$2" + "", "", "", human(before - after) " on ghcr.io (" human(before) " → " human(after) ") and ~" human(hub_before - hub_after) " on docker.io (~" human(hub_before) " → ~" human(hub_after) ")", + "", amount(manifests, "manifest", "manifests") + }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/blobs.tsv" > "$2" } delete_ghcr_version() { local error + [[ "$1" =~ ^[0-9]+$ ]] || die "the version id of the $2 is '$1', expected a number from the GitHub API" until error="$(gh api -X DELETE "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions/$1" 2>&1 > /dev/null)"; do case "$error" in *"rate limit"*) sleep 60 ;; *downloads*) - warn "ghcr.io and docker.io keep the $2, it has more than 5,000 downloads on ghcr.io" + warn "ghcr.io and docker.io keep the $2, ghcr.io deletes no version with more than 5,000 downloads" return 1 ;; *) die "could not delete the $2 from ghcr.io: $error" ;; @@ -251,22 +312,38 @@ delete_ghcr_version() { sleep 1 } +# docker.io answers 202 to the deletion of any digest, so a HEAD tells first whether the manifest is still there delete_dockerhub_manifest() { - local status - while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $(registry_token docker.io pull,push,delete)" \ - "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do + local token status + token="$(registry_token docker.io pull,push,delete)" + status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' --head -H "Authorization: Bearer $token" -H "Accept: $MANIFEST_TYPES" "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" + [ "$status" = 200 ] || { + echo "$status" + return + } + while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do sleep 60 + token="$(registry_token docker.io pull,push,delete)" done echo "$status" } registry_token() { case "$1" in - ghcr.io) curl -sSf --retry 3 "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; - docker.io) curl -sSf --retry 3 ${DOCKERHUB_USERNAME:+-u "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN"} "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; + ghcr.io) basic_auth "x:$GHCR_TOKEN" | curl -sSf --retry 3 -K - "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; + docker.io) basic_auth "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN" | curl -sSf --retry 3 -K - "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; esac | jq -r .token } +basic_auth() { + printf 'header = "Authorization: Basic %s"\n' "$(printf '%s' "$1" | jq -Rrj @base64)" +} + +hub_token() { + jq -n --arg identifier "$DOCKERHUB_USERNAME" --arg secret "$DOCKERHUB_TOKEN" '{identifier: $identifier, secret: $secret}' \ + | curl -sSf --retry 3 -X POST -H 'Content-Type: application/json' --data-binary @- https://hub.docker.com/v2/auth/token | jq -er .access_token +} + count() { awk 'END { print NR }'; } main diff --git a/scripts/lint.sh b/scripts/lint.sh index 0624685..b696e30 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -8,8 +8,8 @@ HADOLINT=hadolint/hadolint:v2.15.1 SHELLCHECK=koalaman/shellcheck:v0.11.0 # https://github.com/mvdan/sh/releases/tag/v3.14.1 SHFMT=mvdan/shfmt:v3.14.1 -# https://github.com/jdx/jactionlint/releases/tag/v2.4.0 -JACTIONLINT=ghcr.io/jdx/jactionlint:2.4.0 +# https://github.com/jdx/jactionlint/releases/tag/v2.5.0 +JACTIONLINT=ghcr.io/jdx/jactionlint:2.5.0 # https://github.com/editorconfig-checker/editorconfig-checker/releases/tag/v4.0.2 EDITORCONFIG_CHECKER=mstruebing/editorconfig-checker:4.0.2 diff --git a/scripts/plan-version.sh b/scripts/plan-version.sh index 1675a45..c5287dd 100755 --- a/scripts/plan-version.sh +++ b/scripts/plan-version.sh @@ -59,9 +59,13 @@ print_build_version() { } fetch_origin() { - local current behind + local current behind fetched stale [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" - git fetch --quiet --prune --tags origin || die "could not fetch origin" + fetched="$(git fetch --porcelain --no-progress --prune --tags origin)" || { + stale="$(awk '$1 == "!" { sub("^refs/tags/", "", $4); print $4 }' <<< "$fetched" | paste -sd ' ' -)" + [ -z "$stale" ] || die "the local tag(s) $stale differ from origin, check them with git ls-remote --tags origin, then run git fetch --tags --force origin" + die "could not fetch origin" + } current="$(git branch --show-current)" if [ -n "$current" ] && behind="$(git rev-list --count "HEAD..origin/$current" 2> /dev/null)"; then [ "$behind" = 0 ] || die "$current is $behind commit(s) behind origin/$current, pull first" @@ -126,21 +130,21 @@ check_version() { } select_releases() { - local name sqlite tag previous older moves listed added="" + local name upstream tag previous older moves listed added="" releases= [ "$publish" = true ] || return 0 - while read -r name sqlite; do - tag="v${sqlite:+$sqlite-v}$version" + while read -r name upstream; do + tag="v${upstream:+$upstream-v}$version" if grep -qxF "$tag" <<< "$tags"; then info "$tag is published" continue fi previous= while read -r older; do - ! newer "$sqlite" "$older" || previous="$older" + ! newer "$upstream" "$older" || previous="$older" done < <(with_version "$version" <<< "$tags" | sed -nE 's/^v(.*)-v.*/\1/p') added+="$tag"$'\n' - releases+="$(jq -nc --arg name "$name" --arg version "$sqlite" --arg image "$version" --arg tag "$tag" --arg previous "$previous" \ + releases+="$(jq -nc --arg name "$name" --arg version "$upstream" --arg image "$version" --arg tag "$tag" --arg previous "$previous" \ '{$name, $version, $image, $tag, $previous}')"$'\n' done < <(upstream_lines) moves="$(moved_tags "$tags" "$tags"$'\n'"$added")" @@ -259,7 +263,7 @@ moved_tags() { } moving_tags() { - local tags releases latest preview base image sqlite prefix seen=" " + local tags releases latest preview base image upstream prefix seen=" " tags="$(cat)" releases="$(sed -nE 's/^v((.*)-v)?([^-]*)$/\3 \2/p' <<< "$tags" | sort -k1,1rV -k2,2rV)" latest="$(awk 'NR == 1 && NF { print "v" ($2 == "" ? "" : $2 "-v") $1 }' <<< "$releases")" @@ -273,10 +277,10 @@ moving_tags() { done < <(tag_versions <<< "$tags" | sed -n 's/-.*//p' | sort -uV) [ -z "$latest" ] || echo "latest $latest" [ -z "$preview" ] || echo "preview $preview" - while read -r image sqlite; do - prefix="$sqlite" + while read -r image upstream; do + prefix="$upstream" while [ -n "$prefix" ] && [[ "$seen" != *" $prefix "* ]]; do - echo "$prefix v$sqlite-v$image" + echo "$prefix v$upstream-v$image" seen+="$prefix " [[ "$prefix" == *.* ]] || break prefix="${prefix%.*}" diff --git a/scripts/release.sh b/scripts/release.sh index 7eaa44f..39f34f8 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,9 +3,10 @@ set -euo pipefail OPTION="$*" VERSION_FILE=release.json +CHANGE_DIRECTORY=.changes +SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Refactored,Maintenance' ARTIFACT=image UPSTREAM=sqlite -BREAKING_CHANGE='^([a-z]+(\([^)]*\))?!|BREAKING[ -]CHANGE): ' WORK_DIR="$(mktemp -d)" BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' @@ -38,13 +39,14 @@ main() { select_version heading "Releasing v$new_version" update_changelog + update_change_files update_version commit push } check_repository() { - local counts behind ahead entries + local counts behind ahead entries fetched stale heading "Checking the repository" git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" branch="$(git symbolic-ref --quiet --short HEAD)" || die "HEAD is detached, expected a branch" @@ -55,14 +57,18 @@ check_repository() { [[ ! "$branch" =~ ^(next|release/) ]] || channel=preview [ -z "$(git status --porcelain)" ] || die "the working tree is not clean, commit or stash first" [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" - git fetch --quiet --prune --tags origin || die "could not fetch origin" + fetched="$(git fetch --porcelain --no-progress --prune --tags origin)" || { + stale="$(awk '$1 == "!" { sub("^refs/tags/", "", $4); print $4 }' <<< "$fetched" | paste -sd ' ' -)" + [ -z "$stale" ] || die "the local tag(s) $stale differ from origin, check them with git ls-remote --tags origin, then run git fetch --tags --force origin" + die "could not fetch origin" + } counts="$(git rev-list --left-right --count "origin/$branch...HEAD" 2> /dev/null)" || die "origin has no branch $branch, push it first" read -r behind ahead <<< "$counts" [ "$behind$ahead" = 00 ] || die "$branch is $ahead commit(s) ahead of and $behind behind origin/$branch, expected it in sync" ok "$branch is clean and in sync with origin/$branch" - entries="$(awk '/^## \[Unreleased\]$/ { inside = 1; next } /^## / { inside = 0 } inside && /^- / { count++ } END { print count + 0 }' CHANGELOG.md)" - [ "$entries" != 0 ] || die "CHANGELOG.md has no entries under ## [Unreleased], describe the changes first" - ok "CHANGELOG.md lists $BOLD$entries$RESET change(s) under [Unreleased], the notes of this release" + entries="$(git ls-files "$CHANGE_DIRECTORY" | wc -l | tr -d ' ')" + [ "$entries" != 0 ] || die "$CHANGE_DIRECTORY has no change files, describe the changes first with scripts/change.sh" + ok "$CHANGE_DIRECTORY lists $BOLD$entries$RESET change(s), the notes of this release" } read_versions() { @@ -77,19 +83,17 @@ read_versions() { } read_changes() { - local tag range hash subject - tag="$(git describe --tags --abbrev=0 --match "v${UPSTREAM:+*-v}$stable" 2> /dev/null)" || true - range="${tag:+$tag..}HEAD" + local file required="patch" - if [ -n "$(git log -E --grep="$BREAKING_CHANGE" --format=%h "$range")" ]; then + if grep -qx 'breaking: true' "$CHANGE_DIRECTORY"/*.md; then required=major [ "${stable%%.*}" != 0 ] || required=minor - elif git log --format=%s "$range" | awk '/^feat(\([^)]*\))?!?: / && !/^feat\(release\): bump / { found = 1 } END { exit !found }'; then + elif grep -qxE 'type: (added|changed|deprecated|removed)' "$CHANGE_DIRECTORY"/*.md; then required=minor fi - info "the commits since ${tag:-the first commit} need a $BOLD$required$RESET release" - git log -E --grep="$BREAKING_CHANGE" --format='%h %s' "$range" | while read -r hash subject; do - warn "$DIM$hash$RESET $subject is a breaking change" + info "the change files need a $BOLD$required$RESET release" + for file in $(grep -lx 'breaking: true' "$CHANGE_DIRECTORY"/*.md || true); do + warn "$file is a breaking change" done } @@ -148,19 +152,59 @@ select_version() { } update_changelog() { - local today - if [ "$channel" = preview ]; then - info "CHANGELOG.md keeps the entries under [Unreleased] until the stable release dates them" - return - fi - today="$(date +%Y-%m-%d)" - awk -v heading="## [v$new_version] - $today" ' - /^#/ && NR > 1 && previous != "" { print "" } - { print; previous = $0 } - /^## \[Unreleased\]$/ { print ""; print heading; previous = heading } + local heading='' + [ "$channel" = preview ] || heading="## [v$new_version] - $(date +%Y-%m-%d)" + print_notes > "$WORK_DIR/notes.md" + awk -v heading="$heading" -v notes="$WORK_DIR/notes.md" ' + /^## / { inside = 0 } + /^## \[Unreleased\]$/ { + inside = 1 + print + print "" + if (heading != "") { print heading; print "" } + while ((getline line < notes) > 0) print line + next + } + !inside ' CHANGELOG.md > "$WORK_DIR/CHANGELOG.md" cp "$WORK_DIR/CHANGELOG.md" CHANGELOG.md - ok "CHANGELOG.md dates the unreleased entries as ${BOLD}v$new_version$RESET, $today" + ok "CHANGELOG.md lists the changes under $BOLD${heading:-## [Unreleased]}$RESET" +} + +print_notes() { + local section entries sections + IFS=, read -ra sections <<< "$SECTIONS" + for section in "${sections[@]}"; do + entries="$(awk -v section="$section" ' + FNR == 1 { fences = 0; type = ""; breaking = 0 } + fences < 2 { + if ($0 == "---") fences++ + else if ($1 == "type:") type = $2 + else if ($0 == "breaking: true") breaking = 1 + next + } + NF && tolower(section) == (breaking ? "breaking changes" : type) { + prefix = /^- / ? " " : "- " + print prefix $0 + } + ' "$CHANGE_DIRECTORY"/*.md)" + [ -z "$entries" ] || printf '### %s\n\n%s\n\n' "$section" "$entries" + done +} + +update_change_files() { + local file + if [ "$channel" = stable ]; then + rm "$CHANGE_DIRECTORY"/*.md + ok "$CHANGE_DIRECTORY is empty, CHANGELOG.md lists the changes now" + return + fi + for file in "$CHANGE_DIRECTORY"/*.md; do + ! grep -q '^version: ' "$file" || continue + awk -v version="$new_version" '/^---$/ && ++fences == 2 { print "version: " version } { print }' "$file" > "$WORK_DIR/change.md" + cp "$WORK_DIR/change.md" "$file" + done + ok "$CHANGE_DIRECTORY names ${BOLD}v$new_version$RESET in the change files that had no version" } update_version() { @@ -170,9 +214,14 @@ update_version() { } commit() { - git add "$VERSION_FILE" CHANGELOG.md - git commit --quiet -S --message "feat(release): bump $ARTIFACT version to v$new_version" - ok "signed commit $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" + git add --all -- "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY" + local sign=-S kind=signed + if ! git commit-tree -S -m 'release signing check' 'HEAD^{tree}' > /dev/null 2>&1 < /dev/null; then + sign=--no-gpg-sign kind=unsigned + warn "git cannot sign a commit (no user.signingkey, or no agent that holds the key), the release commit is unsigned" + fi + git commit --quiet "$sign" --message "feat(release): bump $ARTIFACT version to v$new_version" + ok "$kind commit $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" } push() { diff --git a/scripts/test.sh b/scripts/test.sh index f20fdc8..e378d45 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -27,6 +27,7 @@ main() { start_container check_databases check_healthcheck + check_locked_database check_restart check_other_user check_creation_settings @@ -43,9 +44,9 @@ main() { check_version() { local printed version source_id printed="$(docker run --rm "$IMAGE" --version)" - version="$(docker image inspect --format '{{ index .Config.Labels "dev.peakimages.sqlite.version" }}' "$IMAGE")" + version="${printed%% *}" source_id="$(jq -r --arg version "$version" '.sqlite[] | select(.version == $version) | .source_id' src/upstream.json)" - [[ "$printed" == "$version $source_id "* ]] || die "sqlite3 --version prints '$printed', expected '$version $source_id'" + [[ -n "$source_id" && "$printed" == "$version $source_id "* ]] || die "sqlite3 --version prints '$printed', expected a version with the source id src/upstream.json pins for it" ok "sqlite3 --version prints $version with the source id src/upstream.json pins for it" } @@ -65,9 +66,7 @@ start_container() { -e SQLITE_DATABASES="app.sqlite,jobs.sqlite" \ --health-interval 2s --health-start-period 1s "$IMAGE" > /dev/null wait_healthy "$CONTAINER" - # no exact match (-x), under QEMU ps shows "{sleep} /usr/bin/qemu-riscv64 /bin/sleep infinity" - docker exec "$CONTAINER" ps -o args | grep -q 'sleep infinity$' || die "the main process is not sleep infinity" - ok "healthy with a read-only root filesystem and all capabilities dropped, main process sleep infinity" + ok "healthy with a read-only root filesystem and all capabilities dropped" } check_databases() { @@ -85,6 +84,25 @@ check_healthcheck() { ok "healthcheck.sh passes for app.sqlite,jobs.sqlite and fails as soon as one file is missing" } +check_locked_database() { + local holder + touch "$WORK_DIR/hold" + ( + echo 'PRAGMA locking_mode = EXCLUSIVE; BEGIN EXCLUSIVE;' + while [ -e "$WORK_DIR/hold" ]; do sleep 1; done + ) | docker exec -i "$CONTAINER" sqlite3 jobs.sqlite > /dev/null & + holder=$! + for _ in $(seq 30); do + docker exec "$CONTAINER" sqlite3 -readonly jobs.sqlite 'PRAGMA schema_version;' > /dev/null 2>&1 || break + sleep 1 + done + docker exec "$CONTAINER" healthcheck.sh jobs.sqlite || die "healthcheck.sh failed for jobs.sqlite while another connection holds it locked, expected it to take no lock" + docker exec "$CONTAINER" sqlite3 -readonly jobs.sqlite 'PRAGMA schema_version;' > /dev/null 2>&1 && die "jobs.sqlite is not locked, expected a connection in exclusive locking mode to hold it" + rm "$WORK_DIR/hold" + wait "$holder" || die "the connection that held jobs.sqlite locked failed" + ok "healthcheck.sh passes for jobs.sqlite while another connection holds it in exclusive locking mode" +} + check_restart() { sql app.sqlite 'PRAGMA journal_mode=DELETE;' > /dev/null docker restart -t 5 "$CONTAINER" > /dev/null diff --git a/scripts/undo-release.sh b/scripts/undo-release.sh index 843e107..8444a4e 100755 --- a/scripts/undo-release.sh +++ b/scripts/undo-release.sh @@ -3,6 +3,7 @@ set -euo pipefail OPTION="$*" VERSION_FILE=release.json +CHANGE_DIRECTORY=.changes BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' @@ -26,7 +27,7 @@ main() { [ -z "$OPTION" ] || die "'$OPTION' is not an option, expected none" check_repository heading "Undoing $(git log -1 --format='%h %s')" - info "resets the commit and restores $VERSION_FILE and CHANGELOG.md" + info "resets the commit and restores $VERSION_FILE, CHANGELOG.md and $CHANGE_DIRECTORY" confirm "Continue?" || die "aborted, nothing changed" undo_commit } @@ -35,18 +36,18 @@ check_repository() { local branch git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" branch="$(git rev-parse --abbrev-ref HEAD)" - git diff --quiet HEAD~1 HEAD -- . ":(exclude)$VERSION_FILE" ":(exclude)CHANGELOG.md" || die "the last commit changes more than $VERSION_FILE and CHANGELOG.md, expected a release commit" + git diff --quiet HEAD~1 HEAD -- . ":(exclude)$VERSION_FILE" ":(exclude)CHANGELOG.md" ":(exclude)$CHANGE_DIRECTORY" || die "the last commit changes more than $VERSION_FILE, CHANGELOG.md and $CHANGE_DIRECTORY, expected a release commit" [ "$(versions_at HEAD~1)" != "$(versions_at HEAD)" ] || die "the last commit keeps the versions in $VERSION_FILE at $(versions_at HEAD), expected a release commit" - [ -z "$(git status --porcelain -- "$VERSION_FILE" CHANGELOG.md)" ] || die "$VERSION_FILE or CHANGELOG.md has uncommitted changes, commit or stash them first" + [ -z "$(git status --porcelain -- "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY")" ] || die "$VERSION_FILE, CHANGELOG.md or $CHANGE_DIRECTORY has uncommitted changes, commit or stash them first" git fetch --quiet origin || die "could not fetch origin" [ "$(git rev-list --count "origin/$branch..HEAD")" -ge 1 ] || die "the release commit is on origin/$branch already, nothing to undo locally" } undo_commit() { git reset --quiet --soft HEAD~1 - git restore --staged --worktree "$VERSION_FILE" CHANGELOG.md + git restore --staged --worktree "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY" ok "reset to $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" - ok "$VERSION_FILE is back to $BOLD$(versions_at HEAD)$RESET, the entries are back under [Unreleased]" + ok "$VERSION_FILE is back to $BOLD$(versions_at HEAD)$RESET, the change files are back" } versions_at() { diff --git a/src/Dockerfile b/src/Dockerfile index 8878e64..0712350 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -3,9 +3,10 @@ # Set by scripts/check-upstream.sh ARG SQLITE_VERSION=3.54.1 -# Build dependencies -# https://hub.docker.com/_/alpine/tags?name=3.24.3 -ARG BASE_IMAGE=alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a +# Dependencies +# https://hub.docker.com/_/alpine/tags?name=3.24.4 +ARG BASE_IMAGE=alpine:3.24.4@sha256:f59abe656b692d901e184be3445e084786602108599d2a51d229c0f5a48c71bf +ARG DISTRO=alpine-3.24 # https://pkgs.alpinelinux.org/package/v3.24/community/x86_64/tini-static ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static @@ -20,7 +21,6 @@ ARG GIT_SHA FROM ${BASE_IMAGE} AS build ARG SQLITE_VERSION -ARG IMAGE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION @@ -76,24 +76,21 @@ RUN gcc -static-pie -fPIE -Os \ shell.c sqlite3.c -lm -o sqlite3 \ && strip --strip-all sqlite3 -# Assemble the root filesystem of the final image in /rootfs COPY --chmod=0755 rootfs/ /rootfs/ COPY --chmod=0644 rootfs.sh /src/ RUN sh /src/rootfs.sh -# A scratch image has no package database for the SBOM scanner to read, this syft JSON document takes its place COPY --chmod=0644 sbom.sh /src/ -RUN sh /src/sbom.sh "peakimages/sqlite:${SQLITE_VERSION}-v${IMAGE_VERSION:?}" \ - "generic|sqlite|${SQLITE_VERSION}|blessing|$(jq -r .url /src/sqlite.json)|sqlite:sqlite" \ - "apk|tini-static|tini_project:tini|/sbin/tini" \ - "apk|busybox-static|busybox:busybox|/bin/busybox" +RUN sh /src/sbom.sh peakimages/sqlite "PeakImages SQLite" \ + "generic|sqlite|${SQLITE_VERSION}|blessing|sqlite:sqlite|/usr/local/bin/sqlite3" \ + "apk|tini-static|/sbin/tini" \ + "apk|busybox-static|/bin/busybox" FROM scratch AS final +ARG DISTRO ARG SQLITE_VERSION ARG IMAGE_VERSION -ARG TINI_VERSION -ARG BUSYBOX_VERSION ARG CREATED ARG GIT_SHA @@ -102,20 +99,23 @@ COPY --from=build /rootfs/ / LABEL org.opencontainers.image.title="SQLite" \ org.opencontainers.image.description="Minimal, production-ready, hardened and secure SQLite Docker image." \ org.opencontainers.image.version="${SQLITE_VERSION}-v${IMAGE_VERSION}" \ - org.opencontainers.image.created="${CREATED}" \ org.opencontainers.image.revision="${GIT_SHA}" \ - org.opencontainers.image.licenses="blessing AND MIT AND GPL-2.0-only AND AGPL-3.0-or-later" \ - org.opencontainers.image.authors="peakimages" \ - org.opencontainers.image.vendor="peakimages" \ + org.opencontainers.image.created="${CREATED}" \ + org.opencontainers.image.licenses="AGPL-3.0-or-later AND blessing" \ + org.opencontainers.image.authors="PeakImages" \ + org.opencontainers.image.vendor="PeakImages" \ org.opencontainers.image.url="https://hub.docker.com/r/peakimages/sqlite" \ - org.opencontainers.image.source="https://github.com/peakimages/sqlite" \ org.opencontainers.image.documentation="https://github.com/peakimages/sqlite#readme" \ - org.opencontainers.image.base.name="scratch" \ - dev.peakimages.sqlite.version="${SQLITE_VERSION}" \ - dev.peakimages.sqlite.extensions="fts3,fts4,fts5,rtree,geopoly,math,dbstat,dbpage,stmt,soundex" \ - dev.peakimages.tini.version="${TINI_VERSION}" \ - dev.peakimages.busybox.version="${BUSYBOX_VERSION}" \ - dev.peakimages.image.version="${IMAGE_VERSION}" + org.opencontainers.image.source="https://github.com/peakimages/sqlite" \ + org.opencontainers.image.base.name="" \ + org.opencontainers.image.base.digest="" \ + dev.peakimages.image.version="${IMAGE_VERSION}" \ + dev.peakimages.image.upstream.version="${SQLITE_VERSION}" \ + dev.peakimages.image.distro="${DISTRO}" \ + dev.peakimages.image.variant="runtime" \ + dev.peakimages.image.flavor="" \ + dev.peakimages.image.shell="busybox" \ + dev.peakimages.image.package-manager="" ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ SQLITE_DATABASES=database.sqlite \ diff --git a/src/rootfs.sh b/src/rootfs.sh index 5a61243..808e307 100644 --- a/src/rootfs.sh +++ b/src/rootfs.sh @@ -30,12 +30,13 @@ install_binaries() { create_user() { install -d /rootfs/etc - echo "nonroot:x:65532:65532::/var/lib/sqlite:/sbin/nologin" > /rootfs/etc/passwd + echo "nonroot:x:65532:65532:nonroot:/home/nonroot:/sbin/nologin" > /rootfs/etc/passwd echo "nonroot:x:65532:" > /rootfs/etc/group } create_directories() { install -d -m 0777 -o 65532 -g 65532 /rootfs/var/lib/sqlite + install -d -m 0755 -o 65532 -g 65532 /rootfs/home/nonroot install -d -m 1777 /rootfs/tmp } diff --git a/src/rootfs/usr/local/bin/healthcheck.sh b/src/rootfs/usr/local/bin/healthcheck.sh index 8134d39..c5aa5fd 100755 --- a/src/rootfs/usr/local/bin/healthcheck.sh +++ b/src/rootfs/usr/local/bin/healthcheck.sh @@ -6,8 +6,16 @@ DATA_DIR=/var/lib/sqlite main() { [ "$#" -gt 0 ] || set -- "${SQLITE_DATABASES:-database.sqlite}" for name in $(printf '%s' "$*" | tr ',' ' '); do - sqlite3 -readonly "$DATA_DIR/$name" 'PRAGMA schema_version;' > /dev/null + read_without_lock "$DATA_DIR/$name" || read_with_lock "$DATA_DIR/$name" done } +read_without_lock() { + sqlite3 -readonly "file:$1?immutable=1" 'PRAGMA schema_version;' > /dev/null 2>&1 +} + +read_with_lock() { + sqlite3 -readonly -cmd '.timeout 3000' "$1" 'PRAGMA schema_version;' > /dev/null +} + main "$@" diff --git a/src/sbom.sh b/src/sbom.sh index 318262f..d87937d 100644 --- a/src/sbom.sh +++ b/src/sbom.sh @@ -1,10 +1,12 @@ #!/bin/sh set -eu -USAGE="usage: sbom.sh IMAGE COMPONENT..., a component is apk|NAME|VENDOR:PRODUCT|PATH..., deb|NAME|LICENSE|VENDOR:PRODUCT|PATH... or generic|NAME|VERSION|LICENSE|URL|VENDOR:PRODUCT" +USAGE="usage: sbom.sh IMAGE FULL_NAME COMPONENT..., a component is apk|NAME|PATH,..., deb|NAME|PATH,... or generic|NAME|VERSION|LICENSE|VENDOR:PRODUCT|PATH" IMAGE="${1:-}" -REPOSITORY="${IMAGE%:*}" -OUTPUT_FILE="/rootfs/var/lib/db/sbom/${REPOSITORY##*/}-${IMAGE##*:}.syft.json" +FULL_NAME="${2:-}" +ROOT_DIR="${ROOT_DIR:-/rootfs}" +APK_DATABASE="$ROOT_DIR/lib/apk/db/installed" +DPKG_DATABASE="$ROOT_DIR/var/lib/dpkg/status.d" die() { echo "sbom: $*" >&2 @@ -12,64 +14,72 @@ die() { } main() { - [ "$#" -gt 1 ] || die "$USAGE" - shift - write_document "$@" - verify_files - echo "sbom: $OUTPUT_FILE lists $(jq -r '[.artifacts[] | "\(.name) \(.version)"] | join(", ")' "$OUTPUT_FILE")" + [ "$#" -gt 2 ] || die "$USAGE" + shift 2 + for component in "$@"; do + case "$component" in + apk\|*\|*) write_apk_record "${component#apk|}" ;; + deb\|*\|*) write_deb_record "${component#deb|}" ;; + generic\|*) write_note "${component#generic|}" ;; + *) die "'$component' is not a component, $USAGE" ;; + esac + done + write_os_release + echo "sbom: $ROOT_DIR declares $(printf '%s\n' "$@" | cut -d '|' -f 2 | paste -sd , - | sed 's/,/, /g')" } -write_document() { - mkdir -p "${OUTPUT_FILE%/*}" - cat /lib/apk/db/installed /var/lib/dpkg/status 2> /dev/null | jq -Rsc --rawfile os_release /etc/os-release --arg usage "$USAGE" ' - ([$os_release | split("\n")[] | capture("^(?[A-Z_]+)=\"?(?[^\"]*)")] | from_entries) as $os - | [ - split("\n\n")[] | [split("\n")[] | capture("^(?[^ :]+): ?(?.*)$")] | from_entries - | select(.Status | . == null or endswith(" installed")) - ] as $installed - | def installed($key; $name): first($installed[] | select(.[$key] == $name)) // error("sbom: \($name) is not installed"); - def purl($path; $version; $qualifiers): "pkg:\($path)@\($version)?" + ([$qualifiers | to_entries[] | select(.value | length > 0) | "\(.key)=\(.value | @uri)"] | join("&")); - def os_purl($type; $name; $version; $arch; $upstream): purl("\($type)/\($os.ID)/\($name)"; $version; { arch: $arch, distro: "\($os.ID)-\($os.VERSION_ID)", upstream: $upstream }); - def package($type; $name; $version; $license; $cpe; $purl): - ($cpe | select(test("^[a-z0-9._-]+:[a-z0-9._-]+$")) // error("sbom: \($cpe) is not a CPE, expected VENDOR:PRODUCT")) as $cpe - | { - name: $name, - version: $version, - type: $type, - licenses: [{ value: $license, spdxExpression: $license, type: "declared" }], - cpes: [{ cpe: "cpe:2.3:a:\($cpe):\($version | gsub("(?[^A-Za-z0-9._-])"; "\\\(.c)")):*:*:*:*:*:*:*", source: "declared" }], - purl: $purl - }; - def apk($name; $cpe; $paths): - installed("P"; $name) - | package("apk"; .P; .V; .L; $cpe; os_purl("apk"; .P; .V; .A; if .o != .P then .o else "" end)) + { - metadataType: "apk-db-entry", - metadata: { package: .P, originPackage: .o, version: .V, architecture: .A, files: [$paths[] | { path: . }] } - }; - def deb($name; $license; $cpe; $paths): - installed("Package"; $name) - | package("deb"; .Package; .Version; $license; $cpe; os_purl("deb"; .Package; .Version; .Architecture; .Source // "" | sub(" \\((?.*)\\)$"; "@\(.version)"))) + { - metadataType: "dpkg-db-entry", - metadata: { package: .Package, source: (.Source // "" | sub(" .*"; "")), version: .Version, architecture: .Architecture, files: [$paths[] | { path: . }] } - }; - def generic($name; $version; $license; $url; $cpe): - package("binary"; $name; $version; $license; $cpe; purl("generic/\($name)"; $version; { download_url: $url })); - { - artifacts: [ - $ARGS.positional[] | split("|") | . as [$type, $name] - | if $type == "apk" and length > 3 then apk($name; .[2]; .[3:]) - elif $type == "deb" and length > 4 then deb($name; .[2]; .[3]; .[4:]) - elif $type == "generic" and length == 6 then generic($name; .[2]; .[3]; .[4]; .[5]) - else error("sbom: \(join("|")) is not a component, \($usage)") end - ], - schema: { version: "16.1.10", url: "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-16.1.10.json" } - }' --args "$@" > "$OUTPUT_FILE" +write_apk_record() { + name="${1%%|*}" + mkdir -p "${APK_DATABASE%/*}" + { + awk -v RS= -v name="$name" 'index("\n" $0 "\n", "\nP:" name "\n") { sub(/\nF:.*/, ""); print; found = 1 } END { exit !found }' /lib/apk/db/installed \ + || die "$name is not installed" + for path in $(echo "${1#*|}" | tr ',' ' '); do + [ -e "$ROOT_DIR$path" ] || die "$path is not in $ROOT_DIR" + if [ -d "$ROOT_DIR$path" ]; then + echo "F:${path#/}" + else + printf 'F:%s\nR:%s\n' "$(dirname "${path#/}")" "${path##*/}" + fi + done + echo + } >> "$APK_DATABASE" } -verify_files() { - jq -r '.artifacts[].metadata.files[]?.path' "$OUTPUT_FILE" | while read -r path; do - [ -f "/rootfs$path" ] || die "$path is not a file in /rootfs" - done +write_deb_record() { + name="${1%%|*}" + mkdir -p "$DPKG_DATABASE" + awk -v RS= -v name="$name" 'index("\n" $0 "\n", "\nPackage: " name "\n") { sub(/\nConffiles:(\n [^\n]*)*/, ""); print; found = 1 } END { exit !found }' /var/lib/dpkg/status > "$DPKG_DATABASE/$name" \ + || die "$name is not installed" + for path in $(echo "${1#*|}" | tr ',' ' '); do + [ -e "$ROOT_DIR$path" ] || die "$path is not in $ROOT_DIR" + (cd "$ROOT_DIR" && find "${path#/}" -type f -exec md5sum {} +) + done > "$DPKG_DATABASE/$name.md5sums" + install -D -m 0644 "/usr/share/doc/$name/copyright" "$ROOT_DIR/usr/share/doc/$name/copyright" + install -D -m 0644 /etc/debian_version "$ROOT_DIR/etc/debian_version" +} + +write_note() { + path="${1##*|}" + [ -f "$ROOT_DIR$path" ] || die "$path is not in $ROOT_DIR" + note="$(echo "$1" | jq -Rce 'split("|") | select(length == 5 and (.[3] | test("^[a-z0-9._-]+:[a-z0-9._-]+$"))) | { type: "generic", name: .[0], version: .[1], license: .[2], appCpe: "cpe:2.3:a:\(.[3]):\(.[1]):*:*:*:*:*:*:*" }')" \ + || die "'generic|$1' is not a component, $USAGE" + as -o /src/note.o /dev/null + ld -shared --package-metadata="$note" -o /src/note.so /src/note.o + objcopy -O binary --only-section=.note.package /src/note.so /src/note + objcopy --add-section .note.package=/src/note "$ROOT_DIR$path" + objcopy --set-section-alignment .note.package=4 "$ROOT_DIR$path" +} + +write_os_release() { + [ -e "$APK_DATABASE" ] || [ -d "$DPKG_DATABASE" ] || return 0 + distribution="$(awk -F= '{ gsub(/"/, "", $2) } $1 == "NAME" { name = $2 } $1 == "VERSION_ID" { version = $2 } END { print name, version }' /etc/os-release)" + { + printf 'NAME="%s"\n' "$FULL_NAME" + grep -Ev '^(PRETTY_NAME|NAME|[A-Z_]+_URL)=' /etc/os-release + printf 'PRETTY_NAME="%s (packages from %s)"\n' "$FULL_NAME" "$distribution" + printf 'HOME_URL="https://github.com/%s"\nBUG_REPORT_URL="https://github.com/%s/issues"\n' "$IMAGE" "$IMAGE" + } > "$ROOT_DIR/etc/os-release" } main "$@"