From 837506058b0062028301804fa270a3f62d6f10c2 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:45:52 +0100 Subject: [PATCH 01/27] fix(release): name the local tags that differ from origin - a local tag that differs from origin failed the fetch with "could not fetch origin" and no reason, as --quiet also hid the rejected tags --- scripts/plan-version.sh | 8 ++++++-- scripts/release.sh | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/plan-version.sh b/scripts/plan-version.sh index 1675a45..5f7ab7f 100755 --- a/scripts/plan-version.sh +++ b/scripts/plan-version.sh @@ -59,9 +59,13 @@ print_build_version() { } fetch_origin() { - local current behind + local current behind fetched stale [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" - git fetch --quiet --prune --tags origin || die "could not fetch origin" + fetched="$(git fetch --porcelain --no-progress --prune --tags origin)" || { + stale="$(awk '$1 == "!" { sub("^refs/tags/", "", $4); print $4 }' <<< "$fetched" | paste -sd ' ' -)" + [ -z "$stale" ] || die "the local tag(s) $stale differ from origin, check them with git ls-remote --tags origin, then run git fetch --tags --force origin" + die "could not fetch origin" + } current="$(git branch --show-current)" if [ -n "$current" ] && behind="$(git rev-list --count "HEAD..origin/$current" 2> /dev/null)"; then [ "$behind" = 0 ] || die "$current is $behind commit(s) behind origin/$current, pull first" diff --git a/scripts/release.sh b/scripts/release.sh index 7eaa44f..f691c60 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -44,7 +44,7 @@ main() { } check_repository() { - local counts behind ahead entries + local counts behind ahead entries fetched stale heading "Checking the repository" git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" branch="$(git symbolic-ref --quiet --short HEAD)" || die "HEAD is detached, expected a branch" @@ -55,7 +55,11 @@ check_repository() { [[ ! "$branch" =~ ^(next|release/) ]] || channel=preview [ -z "$(git status --porcelain)" ] || die "the working tree is not clean, commit or stash first" [ "$(git rev-parse --is-shallow-repository)" = false ] || die "the clone is shallow, run git fetch --unshallow first" - git fetch --quiet --prune --tags origin || die "could not fetch origin" + fetched="$(git fetch --porcelain --no-progress --prune --tags origin)" || { + stale="$(awk '$1 == "!" { sub("^refs/tags/", "", $4); print $4 }' <<< "$fetched" | paste -sd ' ' -)" + [ -z "$stale" ] || die "the local tag(s) $stale differ from origin, check them with git ls-remote --tags origin, then run git fetch --tags --force origin" + die "could not fetch origin" + } counts="$(git rev-list --left-right --count "origin/$branch...HEAD" 2> /dev/null)" || die "origin has no branch $branch, push it first" read -r behind ahead <<< "$counts" [ "$behind$ahead" = 00 ] || die "$branch is $ahead commit(s) ahead of and $behind behind origin/$branch, expected it in sync" From 38bbcf7603f5a19e304819b7e95bec3750cfde41 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:37:05 +0100 Subject: [PATCH 02/27] ci: upload the list of the manifests to delete in every run --- .github/workflows/clean-manifests.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml index 4a1ee32..8e2f823 100644 --- a/.github/workflows/clean-manifests.yaml +++ b/.github/workflows/clean-manifests.yaml @@ -41,11 +41,13 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CLEANUP_TOKEN }} run: scripts/clean-manifests.sh --delete 200 - - name: Upload the list of deleted manifests + - name: Upload the lists of manifests if: ${{ always() }} uses: actions/upload-artifact@v7.0.2 with: - name: deleted-manifests - path: scripts/deleted-manifests.tsv + name: manifests + path: | + scripts/manifests-to-delete.tsv + scripts/deleted-manifests.tsv if-no-files-found: ignore retention-days: 90 From 387ba14ee0a13692277d179060b0106e496f2a0b Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:30 +0100 Subject: [PATCH 03/27] feat(scripts): sync the manifest cleanup with Coolify - delete the untagged manifests no tag references on docker.io, which the undocumented manifests endpoint of the Docker Hub API lists - delete on docker.io the versions ghcr.io deleted last, so a run that stops halfway leaves nothing behind - delete only manifests older than a day and a shared image after the last index that references it - sign in to ghcr.io and the Docker Hub API, with the logins on stdin and the ids and digests checked before a delete - keep the pull request images of the variants VARIANTS lists - show the sizes of both registries, every layer counted once --- scripts/clean-manifests.sh | 253 ++++++++++++++++++++++++------------- 1 file changed, 165 insertions(+), 88 deletions(-) diff --git a/scripts/clean-manifests.sh b/scripts/clean-manifests.sh index 964f394..7423473 100755 --- a/scripts/clean-manifests.sh +++ b/scripts/clean-manifests.sh @@ -4,10 +4,14 @@ set -euo pipefail OPTION="$*" LIMIT="${2:-}" IMAGE=peakimages/sqlite +VARIANTS="" KEEP_TAG_PATTERN='^(latest|preview|[0-9]+(\.[0-9]+){0,2}|[0-9]+\.[0-9]+\.[0-9]+-v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?)$' KEEP_TAG_EXCEPTIONS="" +MIN_AGE_DAYS=1 DOCKERHUB_USERNAME="${DOCKERHUB_USERNAME:-}" DOCKERHUB_TOKEN="${DOCKERHUB_TOKEN:-}" +GHCR_TOKEN="" +MANIFEST_TYPES='application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' TO_DELETE_FILE=scripts/manifests-to-delete.tsv DELETED_FILE=scripts/deleted-manifests.tsv WORK_DIR="$(mktemp -d)" @@ -28,14 +32,17 @@ trap 'rm -rf "$WORK_DIR"' EXIT main() { [[ "$OPTION" =~ ^(--delete [1-9][0-9]*)?$ ]] || die "'$OPTION' is not an option, expected --delete with a count or none" + [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to read docker.io/$IMAGE" + GHCR_TOKEN="$(gh auth token)" || die "gh is not signed in with a token that has read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" heading "Reading ghcr.io/$IMAGE and docker.io/$IMAGE" read_ghcr_versions read_dockerhub_tags + read_dockerhub_manifests_deleted_on_ghcr + read_dockerhub_untagged_manifests select_kept_tags read_manifests select_kept_manifests heading "Selecting what to delete" - find_dockerhub_only_manifests write_list delete_manifests verify_kept_manifests @@ -48,134 +55,186 @@ read_ghcr_versions() { list_ghcr_versions "state=deleted&page=$page" done | awk -v window="$window" 'NR <= window' > "$WORK_DIR/ghcr-deleted.tsv" cut -f 1 "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/on-ghcr.txt" - ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged, and $(count < "$WORK_DIR/ghcr-deleted.tsv") of the versions it deleted last are checked on docker.io, twice as many as the run deletes" + ok "ghcr.io/$IMAGE has $(count < "$WORK_DIR/ghcr-active.tsv") versions, $(awk -F '\t' '$3 == ""' "$WORK_DIR/ghcr-active.tsv" | count) of them untagged" } read_dockerhub_tags() { - local token page last="" - token="$(registry_token docker.io pull)" - while :; do - page="$(curl -sSf --retry 3 -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/tags/list?n=1000${last:+&last=$last}" | jq -r '.tags // [] | .[]')" \ - || die "could not list the tags of docker.io/$IMAGE" - [ -n "$page" ] || break - echo "$page" >> "$WORK_DIR/dockerhub-tags.txt" - last="$(tail -n 1 <<< "$page")" + local hub url page + hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME" + url="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags?page_size=100" + : > "$WORK_DIR/hub-tags.tsv" + : > "$WORK_DIR/hub-images.tsv" + : > "$WORK_DIR/dockerhub-images.tsv" + while [ -n "$url" ]; do + page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" || die "could not list the tags of docker.io/$IMAGE from the Docker Hub API" + jq -r '.results[] | [.name, (.digest // ""), (.media_type // ""), (.tag_last_pushed // .last_updated // ""), (.full_size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-tags.tsv" + jq -r '.results[] | .name as $tag | .images[]? | select(.digest) | [$tag, .digest, ([.os, .architecture] | join("/")), (.last_pushed // ""), (.size // 0)] | @tsv' <<< "$page" >> "$WORK_DIR/hub-images.tsv" + url="$(jq -r '.next // empty' <<< "$page")" done - request_manifests docker.io "$WORK_DIR/dockerhub-tags.txt" -f --head -o /dev/null -w '%header{docker-content-digest}\t%header{content-type}\t{}\n' \ - > "$WORK_DIR/dockerhub.tsv" || die "could not read the digests of the tags of docker.io/$IMAGE" - ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(cut -f 1 "$WORK_DIR/dockerhub.tsv" | sort -u | count) manifests" + awk -F '\t' '$2 == "" || $3 == "" { print $1 }' "$WORK_DIR/hub-tags.tsv" > "$WORK_DIR/old-tags.txt" + request_manifests docker.io "$WORK_DIR/old-tags.txt" -f --head -o /dev/null -w '{}\t%header{docker-content-digest}\t%header{content-type}\n' > "$WORK_DIR/old-tags.tsv" \ + || die "could not read the digests of the old tags of docker.io/$IMAGE, the Docker Hub API lists them without one" + awk -F '\t' -v OFS='\t' -v images="$WORK_DIR/dockerhub-images.tsv" ' + FILENAME == ARGV[1] { old_digest[$1] = $2; old_type[$1] = $3; next } + FILENAME == ARGV[2] { if ($2 == "") $2 = old_digest[$1]; if ($3 == "") $3 = old_type[$1]; root[$1] = $2; tags[$2] = ($2 in tags) ? tags[$2] "," $1 : $1; type[$2] = $3; if ($4 > pushed[$2]) pushed[$2] = $4; next } + $2 == root[$1] { platform[$2] = $3; size[$2] = $5; next } + !((root[$1], $2) in pair) { pair[root[$1], $2]; print root[$1], $2 > images } + !($2 in tags) && !seen[$2]++ { print $2, "", "", $4, $5, $3 } + END { for (d in tags) print d, type[d], tags[d], pushed[d], (type[d] ~ /index|list/ ? 0 : size[d] + 0), platform[d] }' \ + "$WORK_DIR/old-tags.tsv" "$WORK_DIR/hub-tags.tsv" "$WORK_DIR/hub-images.tsv" > "$WORK_DIR/dockerhub-manifests.tsv" + awk -F '\t' -v OFS='\t' '$3 != "" { n = split($3, tags, ","); for (i = 1; i <= n; i++) print $1, $2, tags[i] }' "$WORK_DIR/dockerhub-manifests.tsv" > "$WORK_DIR/dockerhub.tsv" + grep -q $'\tlatest$' "$WORK_DIR/dockerhub.tsv" || die "docker.io/$IMAGE lists no latest tag, expected the Docker Hub API to return every tag" + awk -F '\t' '$1 !~ /^sha256:/ || $4 == "" { exit 1 }' "$WORK_DIR/dockerhub-manifests.tsv" || die "docker.io/$IMAGE lists manifests without a digest or push date, expected both from the Docker Hub API" + ok "docker.io/$IMAGE has $(count < "$WORK_DIR/dockerhub.tsv") tags on $(awk -F '\t' '$3 != ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) manifests, which reference $(awk -F '\t' '$3 == ""' "$WORK_DIR/dockerhub-manifests.tsv" | count) more" +} + +read_dockerhub_manifests_deleted_on_ghcr() { + awk -F '\t' 'FILENAME == ARGV[1] { listed[$1]; next } !($1 in listed) && !seen[$1]++ { print $1 }' "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/ghcr-deleted.tsv" > "$WORK_DIR/deleted-on-ghcr.txt" + request_manifests docker.io "$WORK_DIR/deleted-on-ghcr.txt" --head -o /dev/null -w '%{http_code}\t{}\t%header{content-type}\n' > "$WORK_DIR/deleted-on-ghcr.tsv" \ + || die "could not check which of the versions that ghcr.io deleted last docker.io still has" + awk '$1 != 200 && $1 != 404 { exit 1 }' "$WORK_DIR/deleted-on-ghcr.tsv" || die "docker.io answered neither 200 nor 404 for a version that ghcr.io deleted last" + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { created[$1] = $4; next } $1 == 200 { print $2, $3, "", created[$2], "", "" }' "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/deleted-on-ghcr.tsv" >> "$WORK_DIR/dockerhub-manifests.tsv" + ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/deleted-on-ghcr.tsv" | count) of the $(count < "$WORK_DIR/ghcr-deleted.tsv") versions that ghcr.io deleted last and no tag references" +} + +# The manifests endpoint is undocumented, Docker Hub uses it for image management, so it can change without notice. It only adds the untagged manifests that no tag references +read_dockerhub_untagged_manifests() { + local hub base url page key + hub="$(hub_token)" || die "could not sign in to the Docker Hub API as $DOCKERHUB_USERNAME" + base="https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/manifests?page_size=100" + url="$base" + : > "$WORK_DIR/listing.tsv" + while [ -n "$url" ]; do + if ! page="$(curl -sSf --retry 3 -H "Authorization: Bearer $hub" "$url")" \ + || ! jq -r '.manifests[] | [.manifest_digest, .media_type, (.tags // [] | join(",")), .last_pushed, (if (.media_type // "" | test("index|list")) then .size else .total_size end), ([.os, .arch] | map(select(. != null and . != "")) | join("/"))] | @tsv' <<< "$page" >> "$WORK_DIR/listing.tsv"; then + warn "the undocumented manifests listing of the Docker Hub API failed, the untagged manifests that no tag references wait for a later run" + return + fi + key="$(jq -r 'select(.manifests | length > 0) | .last_evaluated_key // empty' <<< "$page")" + url="${key:+$base&last_evaluated_key=$(jq -rn --arg key "$key" '$key | @uri')}" + done + awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { known[$1]; next } !($1 in known) && $3 == "" { print $1, $2, "", $4, $5, $6 }' "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/listing.tsv" > "$WORK_DIR/untagged.tsv" + cat "$WORK_DIR/untagged.tsv" >> "$WORK_DIR/dockerhub-manifests.tsv" + ok "the undocumented manifests listing adds $(count < "$WORK_DIR/untagged.tsv") untagged manifests that no tag references" } select_kept_tags() { { cut -f 3 "$WORK_DIR/ghcr-active.tsv" | tr ',' '\n' && cut -f 3 "$WORK_DIR/dockerhub.tsv"; } | sed '/^$/d' | sort -u > "$WORK_DIR/tags.txt" git ls-remote --tags "https://github.com/$IMAGE.git" | sed -n 's|.*refs/tags/||p' | sed 's/\^{}$//; p; s/^v//' > "$WORK_DIR/keep.txt" \ || die "could not list the git tags of github.com/$IMAGE" - gh pr list --repo "$IMAGE" --state open --limit 1000 --json number --jq '.[] | "pr-\(.number)"' >> "$WORK_DIR/keep.txt" \ + gh pr list --repo "$IMAGE" --state open --limit 1000 --json number --jq '.[] | "pr-\(.number)"' > "$WORK_DIR/pull-requests.txt" \ || die "gh could not list the open pull requests of $IMAGE" - tr ' ' '\n' <<< "$KEEP_TAG_EXCEPTIONS" >> "$WORK_DIR/keep.txt" + awk -v variants="$VARIANTS" '{ print; n = split(variants, variant, ","); for (i = 1; i <= n; i++) print $0 "-" variant[i] }' "$WORK_DIR/pull-requests.txt" >> "$WORK_DIR/keep.txt" + tr ',' '\n' <<< "$KEEP_TAG_EXCEPTIONS" >> "$WORK_DIR/keep.txt" { grep -E "$KEEP_TAG_PATTERN" "$WORK_DIR/tags.txt" || true grep -xF -f "$WORK_DIR/keep.txt" "$WORK_DIR/tags.txt" || true } | sort -u > "$WORK_DIR/kept-tags.txt" grep -qx latest "$WORK_DIR/kept-tags.txt" || die "found no latest tag, expected it among the tags to keep" - ok "keeping $(count < "$WORK_DIR/kept-tags.txt") tags: every tag KEEP_TAG_PATTERN matches or KEEP_TAG_EXCEPTIONS lists, every git tag of github.com/$IMAGE and the tags of open pull requests" + ok "keeping $(count < "$WORK_DIR/kept-tags.txt") tags: the ones KEEP_TAG_PATTERN matches or KEEP_TAG_EXCEPTIONS lists, the git tags of github.com/$IMAGE and the tags of open pull requests" } read_manifests() { - local digest tag mkdir "$WORK_DIR/manifests" request_manifests ghcr.io "$WORK_DIR/on-ghcr.txt" -f -o "$WORK_DIR/manifests/{}" -w '{}\t%{size_download}\n' > "$WORK_DIR/bytes.tsv" \ || die "could not read the manifests of ghcr.io/$IMAGE" - awk -F '\t' -v OFS='\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && !seen[$1]++ { print $1, $2, $3 }' \ - "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-roots.tsv" - awk -F '\t' '$2 ~ /oci\.image\.index/ { print $1 }' "$WORK_DIR/dockerhub-roots.tsv" > "$WORK_DIR/dockerhub-indexes.txt" - request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the indexes kept tags point at on docker.io" - while IFS=$'\t' read -r digest tag; do - curl -sSf --retry 3 "https://hub.docker.com/v2/namespaces/${IMAGE%%/*}/repositories/${IMAGE#*/}/tags/$tag" | jq -r --arg root "$digest" '.images[] | [$root, .digest] | @tsv' \ - || die "could not read the manifests $tag points at on docker.io" - done < <(awk -F '\t' -v OFS='\t' '$2 !~ /oci\.image\.index/ { print $1, $3 }' "$WORK_DIR/dockerhub-roots.tsv") > "$WORK_DIR/children.tsv" + awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { ghcr[$1]; next } ($3 in kept) && !($1 in ghcr) && $2 ~ /oci\.image\.index/ && !seen[$1]++ { print $1 }' \ + "$WORK_DIR/kept-tags.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/dockerhub-indexes.txt" + request_manifests docker.io "$WORK_DIR/dockerhub-indexes.txt" -f -o "$WORK_DIR/manifests/{}" || die "could not read the kept indexes that only docker.io has" manifest_fields '.manifests[]? | [manifest, .digest, (.platform | [.os, .architecture, .variant] | map(values) | join("/")), .annotations["vnd.docker.reference.digest"]]' \ - >> "$WORK_DIR/children.tsv" + > "$WORK_DIR/children.tsv" manifest_fields '(.config // empty), .layers[]? | [manifest, .digest, .size]' > "$WORK_DIR/blobs.tsv" manifest_fields '[manifest, if .manifests then "index" elif any(.layers[]?; .mediaType == "application/vnd.in-toto+json") then "attestation" else "image" end]' \ > "$WORK_DIR/kinds.tsv" - ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests of ghcr.io/$IMAGE and the $(count < "$WORK_DIR/dockerhub-roots.tsv") that kept tags point at on docker.io alone" + ok "read the $(count < "$WORK_DIR/bytes.tsv") manifests on ghcr.io and the $(count < "$WORK_DIR/dockerhub-indexes.txt") kept indexes that only docker.io has, each with a GET that counts as a pull" } select_kept_manifests() { local kinds - kept_manifests "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/kept-ghcr.io.txt" - kept_manifests "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/kept-docker.io.txt" + manifests_of_kept_tags "$WORK_DIR/ghcr-active.tsv" > "$WORK_DIR/kept-ghcr.io.txt" + manifests_of_kept_tags "$WORK_DIR/dockerhub.tsv" > "$WORK_DIR/kept-docker.io.txt" sort -u "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kept-docker.io.txt" > "$WORK_DIR/kept.txt" awk 'FILENAME == ARGV[1] { ghcr[$1]; next } !($1 in ghcr) { exit 1 }' "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/kept-ghcr.io.txt" \ - || die "manifests the kept tags point at are missing from the versions gh listed on ghcr.io, expected every one of them" - kinds="$(awk -F '\t' 'FILENAME == ARGV[1] { kept[$1]; next } ($1 in kept) { count[$2]++ } - END { printf "%d indexes, %d images and %d attestations", count["index"], count["image"], count["attestation"] }' "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kinds.tsv")" - ok "keeping $(count < "$WORK_DIR/kept-ghcr.io.txt") manifests on ghcr.io, $kinds, and $(count < "$WORK_DIR/kept-docker.io.txt") on docker.io: the kept tags and every manifest they point at" -} - -find_dockerhub_only_manifests() { - awk -F '\t' 'FILENAME == ARGV[1] || FILENAME == ARGV[2] { skip[$1]; next } !($1 in skip) && !seen[$1]++ { print $1 }' \ - "$WORK_DIR/kept.txt" "$WORK_DIR/on-ghcr.txt" "$WORK_DIR/dockerhub.tsv" "$WORK_DIR/ghcr-deleted.tsv" > "$WORK_DIR/candidates.txt" - request_manifests docker.io "$WORK_DIR/candidates.txt" --head -o /dev/null -w '%{http_code}\t{}\t%header{content-type}\n' \ - > "$WORK_DIR/dockerhub-only.tsv" || die "could not check which manifests docker.io has" - awk '$1 != 200 && $1 != 404 { exit 1 }' "$WORK_DIR/dockerhub-only.tsv" \ - || die "could not check docker.io for the manifests ghcr.io does not have, expected 200 or 404" - ok "docker.io still has $(awk '$1 == 200' "$WORK_DIR/dockerhub-only.tsv" | count) of the $(count < "$WORK_DIR/candidates.txt") unused manifests ghcr.io does not have" + || die "ghcr.io lists no version for some kept manifests, expected one for each" + kinds="$(awk -F '\t' 'function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } + FILENAME == ARGV[1] { kept[$1]; next } ($1 in kept) { count[$2]++ } + END { print amount(count["index"], "index", "indexes") ", " amount(count["image"], "image", "images") ", " amount(count["attestation"], "attestation", "attestations") }' "$WORK_DIR/kept-ghcr.io.txt" "$WORK_DIR/kinds.tsv")" + ok "keeping $(count < "$WORK_DIR/kept-ghcr.io.txt") manifests on ghcr.io ($kinds) and $(count < "$WORK_DIR/kept-docker.io.txt") on docker.io" } write_list() { - awk -F '\t' -v OFS='\t' 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } + local cutoff + cutoff="$(jq -nr --argjson days "$MIN_AGE_DAYS" 'now - $days * 86400 | strftime("%Y-%m-%dT%H:%M:%SZ")')" + ok "keeping the $(awk -F '\t' -v cutoff="$cutoff" '$4 >= cutoff && !seen[$1]++' "$WORK_DIR/ghcr-active.tsv" | count) versions on ghcr.io and the $(awk -F '\t' -v cutoff="$cutoff" '$4 >= cutoff' "$WORK_DIR/dockerhub-manifests.tsv" | count) manifests on docker.io pushed since $cutoff, a running build may still need them" + awk -F '\t' -v OFS='\t' -v cutoff="$cutoff" 'function platform_of(digest) { return (reference[digest] in platform) ? platform[reference[digest]] : platform[digest] } FILENAME == ARGV[1] { kept[$1]; next } FILENAME == ARGV[2] { kind[$1] = $2; next } FILENAME == ARGV[3] { size[$1] = $2; next } FILENAME == ARGV[4] { size[$1] += $3; next } FILENAME == ARGV[5] { platform[$2] = $3; reference[$2] = $4; next } - FILENAME == ARGV[6] { tags[$1] = tags[$1] (tags[$1] == "" ? "" : ",") $3; next } - FILENAME == ARGV[7] { version[$1] = $2; created[$1] = $4; next } - FILENAME == ARGV[8] { if ($1 == 200) print "no", "yes", ($3 ~ /index|list/ ? "index" : "image"), platform_of($2), tags[$2], created[$2], "", version[$2], $2; next } - !($1 in kept) { print "yes", "undefined", kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ - "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub.tsv" \ - "$WORK_DIR/ghcr-deleted.tsv" "$WORK_DIR/dockerhub-only.tsv" "$WORK_DIR/ghcr-active.tsv" | sort -t $'\t' -k3,3r -k1,1 -k6,6 > "$WORK_DIR/rows.tsv" + FILENAME == ARGV[6] { + on_docker_hub[$1] + if (!($1 in kept) && !($1 in kind) && $4 != "" && $4 < cutoff && !seen[$1]++) + print "no", "yes", ($2 ~ /index|list/ ? "index" : ($6 == "unknown/unknown" ? "attestation" : "image")), ($6 == "unknown/unknown" ? "" : $6), $3, $4, $5, "", $1 + next + } + !($1 in kept) && $4 < cutoff && !seen[$1]++ { print "yes", (($1 in on_docker_hub) ? "yes" : "no"), kind[$1], platform_of($1), $3, $4, size[$1], $2, $1 }' \ + "$WORK_DIR/kept.txt" "$WORK_DIR/kinds.tsv" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/ghcr-active.tsv" \ + | sort -t $'\t' -k3,3r -k6,6 > "$WORK_DIR/sorted.tsv" + awk -F '\t' 'function emit(digest, list, n, i) { printed[digest]; print line[digest]; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) if ((list[i] in line) && !(list[i] in printed) && --parents[list[i]] == 0) emit(list[i]) } + FILENAME == ARGV[1] || FILENAME == ARGV[2] { if (!(($1, $2) in pair)) { pair[$1, $2]; children[$1] = children[$1] " " $2 } next } + { order[++rows] = $9; line[$9] = $0; type[$9] = $3 } + END { + for (i = 1; i <= rows; i++) if (order[i] in children) { n = split(children[order[i]], list, " "); for (j = 1; j <= n; j++) if (list[j] in line) parents[list[j]]++ } + for (i = 1; i <= rows; i++) if (type[order[i]] == "index" && !(order[i] in children)) emit(order[i]) + for (i = 1; i <= rows; i++) if (type[order[i]] == "image" && !(order[i] in parents) && !(order[i] in printed)) emit(order[i]) + for (i = 1; i <= rows; i++) if (!(order[i] in printed)) emit(order[i]) + }' "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-images.tsv" "$WORK_DIR/sorted.tsv" > "$WORK_DIR/rows.tsv" write_table "$WORK_DIR/rows.tsv" "$TO_DELETE_FILE" - ok "$(awk -F '\t' 'END { print $9 " to delete, " $3 ", they free " $7 }' "$TO_DELETE_FILE")" - ok "$(awk -F '\t' '$1 == "no"' "$WORK_DIR/rows.tsv" | count) of them on docker.io alone, ghcr.io deleted them already or never had them, their size stays empty as reading it counts as a pull" - info "$TO_DELETE_FILE lists every manifest to delete with its type, platform, tags, date and size, indexes first" + ok "$(awk -F '\t' 'END { print ($9 + 0 ? $9 " to delete (" $3 "), they free " $7 : "nothing to delete") }' "$TO_DELETE_FILE")" + info "$TO_DELETE_FILE lists every manifest to delete in the order it deletes them, with its type, platform, tags, date and size" } delete_manifests() { - local on_ghcr type tags version digest rest name deleted ghcr docker_hub status + local on_ghcr on_docker_hub type tags version digest rest name deleted ghcr docker_hub status gone=0 if [ -z "$LIMIT" ]; then - info "nothing is deleted, a run with --delete and a count deletes that many from the top of the list" + info "a dry run deletes nothing, --delete with a count deletes that many manifests from the top of the list" return fi - [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_TOKEN" ] || die "DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to delete manifests on docker.io" heading "Deleting" - write_table /dev/null "$DELETED_FILE" - while IFS='|' read -r on_ghcr type tags version digest rest; do + head -n 1 "$TO_DELETE_FILE" > "$DELETED_FILE" + : > "$WORK_DIR/deleted.tsv" + while IFS='|' read -r on_ghcr on_docker_hub type tags version digest rest; do + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || die "the list names '$digest', expected a sha256 digest" name="$digest${tags:+ ($tags)}" - deleted="" ghcr=no docker_hub=no + deleted="" ghcr=no docker_hub=no status="" if [ "$on_ghcr" = yes ]; then delete_ghcr_version "$version" "$type $name" || continue deleted=ghcr.io ghcr=yes fi - status="$(delete_dockerhub_manifest "$digest")" - if [[ "$status" == 2?? ]]; then - deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + if [ "$on_docker_hub" = yes ]; then + status="$(delete_dockerhub_manifest "$digest")" + if [[ "$status" == 2?? ]]; then + deleted="${deleted:+$deleted and }docker.io" docker_hub=yes + elif [ "$status" = 404 ]; then + info "docker.io had already deleted the $type $BOLD$name$RESET together with its index" + gone=$((gone + 1)) + fi fi if [ -n "$deleted" ]; then ok "deleted the $type $BOLD$name$RESET from $deleted" - printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" >> "$WORK_DIR/deleted.tsv" - write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + printf '%s\t%s\t%s\n' "$ghcr" "$docker_hub" "$rest" | tee -a "$WORK_DIR/deleted.tsv" >> "$DELETED_FILE" fi case "$status" in - 2?? | 404) ;; + "" | 2?? | 404) ;; 403 | 5??) warn "docker.io keeps the $type $name for now, it answered $status, the next run tries again" ;; *) die "could not delete the $type $name from docker.io, it answered $status" ;; esac - done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $3, $5, $8, $9, rest }') - ok "$(awk -F '\t' 'END { print "deleted " $9 ", " $3 ", they free " $7 }' "$DELETED_FILE")" - info "$DELETED_FILE lists what this run deleted and where, ghcr.io restores it for 30 days, docker.io deletes for good" + done < <(head -n "$LIMIT" "$WORK_DIR/rows.tsv" | awk -F '\t' -v OFS='|' '{ rest = $3; for (i = 4; i <= NF; i++) rest = rest "\t" $i; print $1, $2, $3, $5, $8, $9, rest }') + write_table "$WORK_DIR/deleted.tsv" "$DELETED_FILE" + ok "$(awk -F '\t' 'END { print ($9 + 0 ? "deleted " $9 " (" $3 "), they free " $7 : "deleted nothing") }' "$DELETED_FILE")" + [ "$gone" = 0 ] || info "docker.io had already deleted $gone of the listed manifests together with their index, it deletes the children of an index that nothing else references" + info "$DELETED_FILE lists what this run deleted and where, ghcr.io can restore them for 30 days, docker.io deletes them permanently" } verify_kept_manifests() { @@ -184,9 +243,9 @@ verify_kept_manifests() { for registry in ghcr.io docker.io; do missing="$(request_manifests "$registry" "$WORK_DIR/kept-$registry.txt" --head -o /dev/null -w '%{http_code}\t{}\n' | awk '$1 != 200 { print $2 }')" \ || die "could not check the kept manifests on $registry" - [ -z "$missing" ] || die "$(count <<< "$missing") manifests the kept tags point at are gone from $registry, expected all of them: $(head -n 3 <<< "$missing" | paste -sd ' ' -)" + [ -z "$missing" ] || die "$(count <<< "$missing") kept manifests are gone from $registry: $(head -n 3 <<< "$missing" | paste -sd ' ' -)" done - ok "every kept tag and every manifest it points at is still on ghcr.io and docker.io" + ok "every kept manifest is still on ghcr.io and docker.io" } list_ghcr_versions() { @@ -194,7 +253,7 @@ list_ghcr_versions() { shift gh api "$@" "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions?per_page=100&$query" \ --jq '.[] | [.name, (.id | tostring), (.metadata.container.tags | join(",")), .created_at] | @tsv' 2> /dev/null \ - || die "gh could not list the versions of ghcr.io/$IMAGE with $query, it needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" + || die "gh could not list the versions of ghcr.io/$IMAGE, its token needs read:packages and delete:packages: gh auth refresh -s read:packages,delete:packages" } request_manifests() { @@ -204,9 +263,7 @@ request_manifests() { split -l 1000 "$list" "$WORK_DIR/chunk-" for chunk in "$WORK_DIR"/chunk-*; do token="$(registry_token "$registry" pull)" - xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" \ - -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \ - "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 + xargs -P 16 -I {} curl -sS --retry 3 -H "Authorization: Bearer $token" -H "Accept: $MANIFEST_TYPES" "$@" "https://${registry/docker.io/registry-1.docker.io}/v2/$IMAGE/manifests/{}" < "$chunk" || return 1 rm "$chunk" done } @@ -215,11 +272,11 @@ manifest_fields() { find "$WORK_DIR/manifests" -type f -exec jq -r 'def manifest: input_filename | sub(".*/"; ""); '"$1"' | @tsv' {} + } -kept_manifests() { +manifests_of_kept_tags() { awk -F '\t' 'function keep(digest, list, n, i) { if (digest in kept) return; kept[digest]; print digest; n = split(children[digest], list, " "); for (i = 1; i <= n; i++) keep(list[i]) } - FILENAME == ARGV[1] { children[$1] = children[$1] " " $2; next } - FILENAME == ARGV[2] { tag[$1]; next } - { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/kept-tags.txt" "$1" + FILENAME == ARGV[1] || FILENAME == ARGV[2] { children[$1] = children[$1] " " $2; next } + FILENAME == ARGV[3] { tag[$1]; next } + { n = split($3, tags, ","); for (i = 1; i <= n; i++) if (tags[i] in tag) keep($1) }' "$WORK_DIR/children.tsv" "$WORK_DIR/dockerhub-images.tsv" "$WORK_DIR/kept-tags.txt" "$1" } write_table() { @@ -227,22 +284,26 @@ write_table() { function amount(count, one, many) { return (count + 0) " " (count == 1 ? one : many) } BEGIN { split("B kB MB GB TB", units, " "); print "ghcr", "docker_hub", "type", "platform", "tags", "created", "size", "ghcr_id", "digest" } FILENAME == ARGV[1] { removed[$9]; types[$3]++; manifests++; if ($7 != "") $7 = human($7); print; next } - FILENAME == ARGV[2] { before += $2; if (!($1 in removed)) after += $2; next } - { blob[$2] = $3; if (!($1 in removed)) used[$2] } + FILENAME == ARGV[2] { bytes[$1] = $2; before += $2; if (!($1 in removed)) after += $2; next } + FILENAME == ARGV[3] { if ($1 in bytes) on_hub[$1]; size = ($1 in bytes) ? bytes[$1] : $5; hub_before += size; if (!($1 in removed)) hub_after += size; next } + { blob[$2] = $3; if (!($1 in removed)) used[$2]; if ($1 in on_hub) { hub_blob[$2] = $3; if (!($1 in removed)) hub_used[$2] } } END { for (digest in blob) { before += blob[digest]; if (digest in used) after += blob[digest] } + for (digest in hub_blob) { hub_before += hub_blob[digest]; if (digest in hub_used) hub_after += hub_blob[digest] } print "total", "", amount(types["index"], "index", "indexes") ", " amount(types["image"], "image", "images") ", " amount(types["attestation"], "attestation", "attestations"), - "", "", "", human(before - after) ", " human(before) " → " human(after) " on ghcr.io", "", amount(manifests, "manifest", "manifests") - }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/blobs.tsv" > "$2" + "", "", "", human(before - after) " on ghcr.io (" human(before) " → " human(after) ") and ~" human(hub_before - hub_after) " on docker.io (~" human(hub_before) " → ~" human(hub_after) ")", + "", amount(manifests, "manifest", "manifests") + }' "$1" "$WORK_DIR/bytes.tsv" "$WORK_DIR/dockerhub-manifests.tsv" "$WORK_DIR/blobs.tsv" > "$2" } delete_ghcr_version() { local error + [[ "$1" =~ ^[0-9]+$ ]] || die "the version id of the $2 is '$1', expected a number from the GitHub API" until error="$(gh api -X DELETE "/orgs/${IMAGE%%/*}/packages/container/${IMAGE#*/}/versions/$1" 2>&1 > /dev/null)"; do case "$error" in *"rate limit"*) sleep 60 ;; *downloads*) - warn "ghcr.io and docker.io keep the $2, it has more than 5,000 downloads on ghcr.io" + warn "ghcr.io and docker.io keep the $2, ghcr.io deletes no version with more than 5,000 downloads" return 1 ;; *) die "could not delete the $2 from ghcr.io: $error" ;; @@ -251,22 +312,38 @@ delete_ghcr_version() { sleep 1 } +# docker.io answers 202 to the deletion of any digest, so a HEAD tells first whether the manifest is still there delete_dockerhub_manifest() { - local status - while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $(registry_token docker.io pull,push,delete)" \ - "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do + local token status + token="$(registry_token docker.io pull,push,delete)" + status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' --head -H "Authorization: Bearer $token" -H "Accept: $MANIFEST_TYPES" "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" + [ "$status" = 200 ] || { + echo "$status" + return + } + while status="$(curl -sS --retry 3 -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer $token" "https://registry-1.docker.io/v2/$IMAGE/manifests/$1")" && [ "$status" = 429 ]; do sleep 60 + token="$(registry_token docker.io pull,push,delete)" done echo "$status" } registry_token() { case "$1" in - ghcr.io) curl -sSf --retry 3 "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; - docker.io) curl -sSf --retry 3 ${DOCKERHUB_USERNAME:+-u "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN"} "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; + ghcr.io) basic_auth "x:$GHCR_TOKEN" | curl -sSf --retry 3 -K - "https://ghcr.io/token?scope=repository:$IMAGE:$2" ;; + docker.io) basic_auth "$DOCKERHUB_USERNAME:$DOCKERHUB_TOKEN" | curl -sSf --retry 3 -K - "https://auth.docker.io/token?service=registry.docker.io&scope=repository:$IMAGE:$2" ;; esac | jq -r .token } +basic_auth() { + printf 'header = "Authorization: Basic %s"\n' "$(printf '%s' "$1" | jq -Rrj @base64)" +} + +hub_token() { + jq -n --arg identifier "$DOCKERHUB_USERNAME" --arg secret "$DOCKERHUB_TOKEN" '{identifier: $identifier, secret: $secret}' \ + | curl -sSf --retry 3 -X POST -H 'Content-Type: application/json' --data-binary @- https://hub.docker.com/v2/auth/token | jq -er .access_token +} + count() { awk 'END { print NR }'; } main From 711921635de357594af42a8a104c1949ba7cc6dd Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:30 +0100 Subject: [PATCH 04/27] ci: disable the manifest cleanup after a failed run --- .github/workflows/clean-manifests.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/clean-manifests.yaml b/.github/workflows/clean-manifests.yaml index 8e2f823..1c42b5f 100644 --- a/.github/workflows/clean-manifests.yaml +++ b/.github/workflows/clean-manifests.yaml @@ -23,6 +23,7 @@ jobs: if: ${{ github.repository == 'peakimages/sqlite' }} permissions: contents: read + actions: write # disable this workflow after a failed run environment: name: cleanup deployment: false @@ -51,3 +52,9 @@ jobs: scripts/deleted-manifests.tsv if-no-files-found: ignore retention-days: 90 + + - name: Disable this workflow after a failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ github.token }} + run: gh workflow disable clean-manifests.yaml --repo "$GITHUB_REPOSITORY" From 86ac273427c1df7bf8012ea77a74ebe2c5b204d8 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:30 +0100 Subject: [PATCH 05/27] docs(release): merge next into main only on the day of the release --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 80e9246..8c47fce 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -17,6 +17,7 @@ Every release is tagged `v-v`, such as `v3.54.1-v | `v1.2.x` | backports and hotfixes, cut from the newest tag of v1.2 | stable `1.2.x` versions | - `main` takes fixes only. A new minor or major reaches it as a merge of `next` or a release branch +- Merge `next` or a release branch into `main` only right before you release it, on the same day. To freeze a minor for longer, publish an RC on `next` or cut `release/v1.4` - `next` and the release branches publish prereleases only, their stable versions are released on `main` - Name a branch after what it changes: `fix/`, `feat/`, and `fix/-v1.2.2` for a backport, after the version that releases it - Squash merge pull requests from contributors and pull requests with a single change, such as SQLite updates From a978c6f67ce736d2f0f68d169b2dea883889922e Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:30 +0100 Subject: [PATCH 06/27] feat(release): write the changelog from change files --- .gitattributes | 1 + RELEASE.md | 26 ++++++- scripts/change.sh | 168 ++++++++++++++++++++++++++++++++++++++++ scripts/release.sh | 88 +++++++++++++++------ scripts/undo-release.sh | 11 +-- 5 files changed, 263 insertions(+), 31 deletions(-) create mode 100755 scripts/change.sh diff --git a/.gitattributes b/.gitattributes index 8a01f67..e2545e1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,6 +3,7 @@ *.md diff=markdown CHANGELOG.md merge=union +/.changes export-ignore /.github export-ignore .editorconfig export-ignore .gitattributes export-ignore diff --git a/RELEASE.md b/RELEASE.md index 8c47fce..c859f6b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -36,11 +36,12 @@ Every release is tagged `v-v`, such as `v3.54.1-v > [!NOTE] > `next` publishes prereleases only. For its stable version, merge `next` into `main` with a merge commit and create the release on `main`. -1. Describe the changes under `## [Unreleased]` in `CHANGELOG.md`, they become the release notes -2. Check what the branch releases: +1. Describe the changes in change files with `scripts/change.sh`, they become the release notes, see [Change files](#change-files) +2. Check what the branch releases and the entries it lists: ```sh scripts/plan-version.sh + scripts/change.sh --notes ``` 3. Select the stage and the version, the script commits them and asks to push: @@ -58,6 +59,27 @@ Every release is tagged `v-v`, such as `v3.54.1-v > scripts/undo-release.sh > ``` +## Change files + +Every change to the image has a file in `.changes/`, `AGENTS.md` describes the types and the wording: + +```text +--- +type: fixed +commit: 1a2b3c4 +--- +Missing description and license on ghcr.io + +- The release index carries the labels of the image as annotations +``` + +- `scripts/change.sh` writes a file, it asks for what its flags leave out, and `--notes` prints the entries of the next release +- `scripts/release.sh` preselects the version the files need and writes them into `CHANGELOG.md`, the release workflow reads its notes from there +- A prerelease writes the files under `## [Unreleased]`, keeps them and adds its version to the files that have none +- A stable release writes the files under `## [v1.4.0] - ` and removes them +- `scripts/undo-release.sh` brings the files back as they were +- A new SQLite version needs no file, the release workflow writes its notes + ## Backport a fix `main` is on 1.3, and 1.1 and 1.2 still get fixes. A fix reaches each of them through a patch branch and a pull request of its own: diff --git a/scripts/change.sh b/scripts/change.sh new file mode 100755 index 0000000..7c1c1e9 --- /dev/null +++ b/scripts/change.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +CHANGE_DIRECTORY=.changes +TYPES='added,changed,deprecated,removed,fixed,security,maintenance' +SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Maintenance' + +BOLD=$'\033[1m' GREEN=$'\033[32m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' + +heading() { printf '\n %s%s%s\n' "$BOLD" "$*" "$RESET"; } +ok() { printf ' %s✔%s %s\n' "$GREEN" "$RESET" "$*"; } +die() { + printf ' %s✖%s %s\n' "$RED" "$RESET" "$*" >&2 + exit 1 +} +confirm() { + local answer + printf '\n %s [y/N] ' "$*" + read -r answer + [[ "$answer" =~ ^[Yy]$ ]] +} + +cd "$(dirname "$0")/.." +trap 'tput cnorm 2> /dev/null || true' EXIT + +main() { + read_arguments "$@" + if [ -n "$notes" ]; then + print_notes + return + fi + [ -n "$kind" ] && [ -n "$summary" ] || ask_questions + write_change +} + +read_arguments() { + kind='' summary='' commit='' breaking='' notes='' details=() + while [ $# -gt 0 ]; do + case "$1" in + --notes) notes=true ;; + --breaking) breaking=true ;; + --type | --summary | --detail | --commit) + [ $# -ge 2 ] || die "$1 has no value, expected one" + case "$1" in + --type) kind="$2" ;; + --summary) summary="$2" ;; + --detail) details+=("$2") ;; + --commit) commit="$2" ;; + esac + shift + ;; + *) die "'$1' is not an option, expected --type, --summary, --detail, --breaking, --commit or --notes" ;; + esac + shift + done +} + +ask_questions() { + local selected detail types + [ -t 0 ] || die "--type and --summary are required without a terminal" + if [ -z "$kind" ]; then + heading "Select the type" + IFS=, read -ra types <<< "$TYPES" + selected="$(menu 0 "${types[@]}")" + kind="${types[$selected]}" + fi + if [ -z "$summary" ]; then + heading "Summary, one sentence with the heading" + printf ' ' + read -r summary + fi + heading "Details, an empty line ends them" + while printf ' - ' && read -r detail && [ -n "$detail" ]; do + details+=("$detail") + done + if confirm "Breaking change?"; then + breaking=true + fi + printf '\n' +} + +write_change() { + local path detail + [[ ",$TYPES," == *",$kind,"* ]] || die "the type is '$kind', expected ${TYPES//,/, }" + [ -n "$summary" ] || die "the summary is empty, expected one sentence" + if [ -n "$commit" ]; then + git rev-parse --verify --quiet "$commit^{commit}" > /dev/null || die "$commit is not a commit of this repository" + commit="$(git rev-parse --short "$commit")" + fi + mkdir -p "$CHANGE_DIRECTORY" + path="$CHANGE_DIRECTORY/$(file_name).md" + { + printf -- '---\ntype: %s\n' "$kind" + [ -z "$breaking" ] || printf 'breaking: true\n' + [ -z "$commit" ] || printf 'commit: %s\n' "$commit" + printf -- '---\n%s\n' "$summary" + [ "${#details[@]}" = 0 ] || printf '\n' + for detail in ${details[@]+"${details[@]}"}; do + printf -- '- %s\n' "$detail" + done + } > "$path" + ok "$path describes the $BOLD$kind$RESET change" +} + +print_notes() { + local section entries sections + compgen -G "$CHANGE_DIRECTORY/*.md" > /dev/null || die "$CHANGE_DIRECTORY has no change files" + IFS=, read -ra sections <<< "$SECTIONS" + for section in "${sections[@]}"; do + entries="$(awk -v section="$section" ' + FNR == 1 { fences = 0; type = ""; breaking = 0 } + fences < 2 { + if ($0 == "---") fences++ + else if ($1 == "type:") type = $2 + else if ($0 == "breaking: true") breaking = 1 + next + } + NF && tolower(section) == (breaking ? "breaking changes" : type) { + prefix = /^- / ? " " : "- " + print prefix $0 + } + ' "$CHANGE_DIRECTORY"/*.md)" + [ -z "$entries" ] || printf '### %s\n\n%s\n\n' "$section" "$entries" + done +} + +file_name() { + local stem name number=2 + stem="$(printf '%s' "$summary" | LC_ALL=C tr -cs '[:alnum:]' '\n' | LC_ALL=C tr '[:upper:]' '[:lower:]' | awk 'NF && ++count <= 6' | paste -sd - -)" + name="$stem" + while [ -e "$CHANGE_DIRECTORY/$name.md" ]; do + name="$stem-$number" + number=$((number + 1)) + done + echo "$name" +} + +menu() { + local selected="$1" options key i + shift + options=("$@") + tput civis >&2 2> /dev/null || true + while true; do + for i in "${!options[@]}"; do + if [ "$i" = "$selected" ]; then + printf ' %s❯ %s%s\n' "$GREEN" "${options[$i]}" "$RESET" >&2 + else + printf ' %s %s%s\n' "$DIM" "${options[$i]}" "$RESET" >&2 + fi + done + IFS= read -rsn1 key + if [ "$key" = $'\x1b' ]; then + IFS= read -rsn2 key + if [ "$key" = '[A' ] && [ "$selected" -gt 0 ]; then + selected=$((selected - 1)) + elif [ "$key" = '[B' ] && [ "$selected" -lt $((${#options[@]} - 1)) ]; then + selected=$((selected + 1)) + fi + elif [ -z "$key" ]; then + break + fi + printf '\033[%dA\033[J' "${#options[@]}" >&2 + done + tput cnorm >&2 2> /dev/null || true + echo "$selected" +} + +main "$@" diff --git a/scripts/release.sh b/scripts/release.sh index f691c60..cde327b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,9 +3,10 @@ set -euo pipefail OPTION="$*" VERSION_FILE=release.json +CHANGE_DIRECTORY=.changes +SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Maintenance' ARTIFACT=image UPSTREAM=sqlite -BREAKING_CHANGE='^([a-z]+(\([^)]*\))?!|BREAKING[ -]CHANGE): ' WORK_DIR="$(mktemp -d)" BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' YELLOW=$'\033[33m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' @@ -38,6 +39,7 @@ main() { select_version heading "Releasing v$new_version" update_changelog + update_change_files update_version commit push @@ -64,9 +66,9 @@ check_repository() { read -r behind ahead <<< "$counts" [ "$behind$ahead" = 00 ] || die "$branch is $ahead commit(s) ahead of and $behind behind origin/$branch, expected it in sync" ok "$branch is clean and in sync with origin/$branch" - entries="$(awk '/^## \[Unreleased\]$/ { inside = 1; next } /^## / { inside = 0 } inside && /^- / { count++ } END { print count + 0 }' CHANGELOG.md)" - [ "$entries" != 0 ] || die "CHANGELOG.md has no entries under ## [Unreleased], describe the changes first" - ok "CHANGELOG.md lists $BOLD$entries$RESET change(s) under [Unreleased], the notes of this release" + entries="$(git ls-files "$CHANGE_DIRECTORY" | wc -l | tr -d ' ')" + [ "$entries" != 0 ] || die "$CHANGE_DIRECTORY has no change files, describe the changes first with scripts/change.sh" + ok "$CHANGE_DIRECTORY lists $BOLD$entries$RESET change(s), the notes of this release" } read_versions() { @@ -81,19 +83,17 @@ read_versions() { } read_changes() { - local tag range hash subject - tag="$(git describe --tags --abbrev=0 --match "v${UPSTREAM:+*-v}$stable" 2> /dev/null)" || true - range="${tag:+$tag..}HEAD" + local file required="patch" - if [ -n "$(git log -E --grep="$BREAKING_CHANGE" --format=%h "$range")" ]; then + if grep -qx 'breaking: true' "$CHANGE_DIRECTORY"/*.md; then required=major [ "${stable%%.*}" != 0 ] || required=minor - elif git log --format=%s "$range" | awk '/^feat(\([^)]*\))?!?: / && !/^feat\(release\): bump / { found = 1 } END { exit !found }'; then + elif grep -qxE 'type: (added|changed|deprecated|removed)' "$CHANGE_DIRECTORY"/*.md; then required=minor fi - info "the commits since ${tag:-the first commit} need a $BOLD$required$RESET release" - git log -E --grep="$BREAKING_CHANGE" --format='%h %s' "$range" | while read -r hash subject; do - warn "$DIM$hash$RESET $subject is a breaking change" + info "the change files need a $BOLD$required$RESET release" + for file in $(grep -lx 'breaking: true' "$CHANGE_DIRECTORY"/*.md || true); do + warn "$file is a breaking change" done } @@ -152,19 +152,59 @@ select_version() { } update_changelog() { - local today - if [ "$channel" = preview ]; then - info "CHANGELOG.md keeps the entries under [Unreleased] until the stable release dates them" - return - fi - today="$(date +%Y-%m-%d)" - awk -v heading="## [v$new_version] - $today" ' - /^#/ && NR > 1 && previous != "" { print "" } - { print; previous = $0 } - /^## \[Unreleased\]$/ { print ""; print heading; previous = heading } + local heading='' + [ "$channel" = preview ] || heading="## [v$new_version] - $(date +%Y-%m-%d)" + print_notes > "$WORK_DIR/notes.md" + awk -v heading="$heading" -v notes="$WORK_DIR/notes.md" ' + /^## / { inside = 0 } + /^## \[Unreleased\]$/ { + inside = 1 + print + print "" + if (heading != "") { print heading; print "" } + while ((getline line < notes) > 0) print line + next + } + !inside ' CHANGELOG.md > "$WORK_DIR/CHANGELOG.md" cp "$WORK_DIR/CHANGELOG.md" CHANGELOG.md - ok "CHANGELOG.md dates the unreleased entries as ${BOLD}v$new_version$RESET, $today" + ok "CHANGELOG.md lists the changes under $BOLD${heading:-## [Unreleased]}$RESET" +} + +print_notes() { + local section entries sections + IFS=, read -ra sections <<< "$SECTIONS" + for section in "${sections[@]}"; do + entries="$(awk -v section="$section" ' + FNR == 1 { fences = 0; type = ""; breaking = 0 } + fences < 2 { + if ($0 == "---") fences++ + else if ($1 == "type:") type = $2 + else if ($0 == "breaking: true") breaking = 1 + next + } + NF && tolower(section) == (breaking ? "breaking changes" : type) { + prefix = /^- / ? " " : "- " + print prefix $0 + } + ' "$CHANGE_DIRECTORY"/*.md)" + [ -z "$entries" ] || printf '### %s\n\n%s\n\n' "$section" "$entries" + done +} + +update_change_files() { + local file + if [ "$channel" = stable ]; then + rm "$CHANGE_DIRECTORY"/*.md + ok "$CHANGE_DIRECTORY is empty, CHANGELOG.md lists the changes now" + return + fi + for file in "$CHANGE_DIRECTORY"/*.md; do + ! grep -q '^version: ' "$file" || continue + awk -v version="$new_version" '/^---$/ && ++fences == 2 { print "version: " version } { print }' "$file" > "$WORK_DIR/change.md" + cp "$WORK_DIR/change.md" "$file" + done + ok "$CHANGE_DIRECTORY names ${BOLD}v$new_version$RESET in the change files that had no version" } update_version() { @@ -174,7 +214,7 @@ update_version() { } commit() { - git add "$VERSION_FILE" CHANGELOG.md + git add --all -- "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY" git commit --quiet -S --message "feat(release): bump $ARTIFACT version to v$new_version" ok "signed commit $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" } diff --git a/scripts/undo-release.sh b/scripts/undo-release.sh index 843e107..8444a4e 100755 --- a/scripts/undo-release.sh +++ b/scripts/undo-release.sh @@ -3,6 +3,7 @@ set -euo pipefail OPTION="$*" VERSION_FILE=release.json +CHANGE_DIRECTORY=.changes BOLD=$'\033[1m' BLUE=$'\033[34m' GREEN=$'\033[32m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' @@ -26,7 +27,7 @@ main() { [ -z "$OPTION" ] || die "'$OPTION' is not an option, expected none" check_repository heading "Undoing $(git log -1 --format='%h %s')" - info "resets the commit and restores $VERSION_FILE and CHANGELOG.md" + info "resets the commit and restores $VERSION_FILE, CHANGELOG.md and $CHANGE_DIRECTORY" confirm "Continue?" || die "aborted, nothing changed" undo_commit } @@ -35,18 +36,18 @@ check_repository() { local branch git rev-parse --is-inside-work-tree > /dev/null 2>&1 || die "$PWD is not a git repository" branch="$(git rev-parse --abbrev-ref HEAD)" - git diff --quiet HEAD~1 HEAD -- . ":(exclude)$VERSION_FILE" ":(exclude)CHANGELOG.md" || die "the last commit changes more than $VERSION_FILE and CHANGELOG.md, expected a release commit" + git diff --quiet HEAD~1 HEAD -- . ":(exclude)$VERSION_FILE" ":(exclude)CHANGELOG.md" ":(exclude)$CHANGE_DIRECTORY" || die "the last commit changes more than $VERSION_FILE, CHANGELOG.md and $CHANGE_DIRECTORY, expected a release commit" [ "$(versions_at HEAD~1)" != "$(versions_at HEAD)" ] || die "the last commit keeps the versions in $VERSION_FILE at $(versions_at HEAD), expected a release commit" - [ -z "$(git status --porcelain -- "$VERSION_FILE" CHANGELOG.md)" ] || die "$VERSION_FILE or CHANGELOG.md has uncommitted changes, commit or stash them first" + [ -z "$(git status --porcelain -- "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY")" ] || die "$VERSION_FILE, CHANGELOG.md or $CHANGE_DIRECTORY has uncommitted changes, commit or stash them first" git fetch --quiet origin || die "could not fetch origin" [ "$(git rev-list --count "origin/$branch..HEAD")" -ge 1 ] || die "the release commit is on origin/$branch already, nothing to undo locally" } undo_commit() { git reset --quiet --soft HEAD~1 - git restore --staged --worktree "$VERSION_FILE" CHANGELOG.md + git restore --staged --worktree "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY" ok "reset to $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" - ok "$VERSION_FILE is back to $BOLD$(versions_at HEAD)$RESET, the entries are back under [Unreleased]" + ok "$VERSION_FILE is back to $BOLD$(versions_at HEAD)$RESET, the change files are back" } versions_at() { From 3fdb6773954713636607a284c06c3114e0d1c239 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:30 +0100 Subject: [PATCH 07/27] fix(release): make an unsigned release commit when git cannot sign --- scripts/release.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index cde327b..0a3642f 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -215,8 +215,13 @@ update_version() { commit() { git add --all -- "$VERSION_FILE" CHANGELOG.md "$CHANGE_DIRECTORY" - git commit --quiet -S --message "feat(release): bump $ARTIFACT version to v$new_version" - ok "signed commit $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" + local sign=-S kind=signed + if ! git commit-tree -S -m 'release signing check' 'HEAD^{tree}' > /dev/null 2>&1 < /dev/null; then + sign=--no-gpg-sign kind=unsigned + warn "git cannot sign a commit (no user.signingkey, or no agent that holds the key), the release commit is unsigned" + fi + git commit --quiet "$sign" --message "feat(release): bump $ARTIFACT version to v$new_version" + ok "$kind commit $DIM$(git rev-parse --short HEAD)$RESET $(git log -1 --format=%s)" } push() { From 2d3468d26c308602a34ddfec4a09d6e9ce220e2d Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:31 +0100 Subject: [PATCH 08/27] refactor(release): name the upstream version upstream --- scripts/plan-version.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/plan-version.sh b/scripts/plan-version.sh index 5f7ab7f..c5287dd 100755 --- a/scripts/plan-version.sh +++ b/scripts/plan-version.sh @@ -130,21 +130,21 @@ check_version() { } select_releases() { - local name sqlite tag previous older moves listed added="" + local name upstream tag previous older moves listed added="" releases= [ "$publish" = true ] || return 0 - while read -r name sqlite; do - tag="v${sqlite:+$sqlite-v}$version" + while read -r name upstream; do + tag="v${upstream:+$upstream-v}$version" if grep -qxF "$tag" <<< "$tags"; then info "$tag is published" continue fi previous= while read -r older; do - ! newer "$sqlite" "$older" || previous="$older" + ! newer "$upstream" "$older" || previous="$older" done < <(with_version "$version" <<< "$tags" | sed -nE 's/^v(.*)-v.*/\1/p') added+="$tag"$'\n' - releases+="$(jq -nc --arg name "$name" --arg version "$sqlite" --arg image "$version" --arg tag "$tag" --arg previous "$previous" \ + releases+="$(jq -nc --arg name "$name" --arg version "$upstream" --arg image "$version" --arg tag "$tag" --arg previous "$previous" \ '{$name, $version, $image, $tag, $previous}')"$'\n' done < <(upstream_lines) moves="$(moved_tags "$tags" "$tags"$'\n'"$added")" @@ -263,7 +263,7 @@ moved_tags() { } moving_tags() { - local tags releases latest preview base image sqlite prefix seen=" " + local tags releases latest preview base image upstream prefix seen=" " tags="$(cat)" releases="$(sed -nE 's/^v((.*)-v)?([^-]*)$/\3 \2/p' <<< "$tags" | sort -k1,1rV -k2,2rV)" latest="$(awk 'NR == 1 && NF { print "v" ($2 == "" ? "" : $2 "-v") $1 }' <<< "$releases")" @@ -277,10 +277,10 @@ moving_tags() { done < <(tag_versions <<< "$tags" | sed -n 's/-.*//p' | sort -uV) [ -z "$latest" ] || echo "latest $latest" [ -z "$preview" ] || echo "preview $preview" - while read -r image sqlite; do - prefix="$sqlite" + while read -r image upstream; do + prefix="$upstream" while [ -n "$prefix" ] && [[ "$seen" != *" $prefix "* ]]; do - echo "$prefix v$sqlite-v$image" + echo "$prefix v$upstream-v$image" seen+="$prefix " [[ "$prefix" == *.* ]] || break prefix="${prefix%.*}" From 363b91e2a1f24588ef7941e90e6509c4efe16bde Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:31 +0100 Subject: [PATCH 09/27] docs(release): simplify the push warning and PR versions --- RELEASE.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index c859f6b..619e6d2 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -3,7 +3,7 @@ Every release is tagged `v-v`, such as `v3.54.1-v1.4.0-beta.1`. > [!IMPORTANT] -> A push that changes `release.json` or `src/upstream.json` publishes every release of it whose tag does not exist yet. Check what a push publishes with `scripts/plan-version.sh` first. +> A push that changes `release.json` or `src/upstream.json` releases the versions they set, unless they are released already. Run `scripts/plan-version.sh` first to see what a push releases. ## Rules @@ -153,8 +153,11 @@ Missing description and license on ghcr.io ## Pull requests -The pull request workflow builds every pull request and publishes the image of the line `latest` of one from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`, versioned `-pr.`: pull request 1234 into `next` is `1.4.0-beta.1-pr.1234` after `1.4.0-beta.1` and `1.5.0-alpha.0-pr.1234` before the first alpha of 1.5.0, into `main` `1.3.4-pr.1234`. PHP's `version_compare` sorts it after its base and before the next prerelease or release. +The pull request workflow builds every pull request and publishes the image of the line `latest` of a pull request from a branch of this repository as `ghcr.io/peakimages/sqlite:pr-`. Its version is `-pr.`: +- Into `next`: `1.4.0-beta.1-pr.1234` after `1.4.0-beta.1`, or `1.5.0-alpha.0-pr.1234` before the first alpha of 1.5.0 +- Into `main`: `1.3.4-pr.1234` +- PHP's `version_compare` sorts it after its base and before the next prerelease or release - Pull requests from forks and Dependabot are built, not published - A push to a pull request from `main`, `next`, a release branch or a patch branch that changes `release.json` or `src/upstream.json` is not built, the release workflow builds it - ghcr.io only, no git tag and no GitHub release, the clean manifests workflow keeps the image while the pull request is open From 2b184bddcf23bccad78bbe2a426a82d08f9197ab Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:31 +0100 Subject: [PATCH 10/27] ci: bump GitHub Actions to the latest releases --- .github/workflows/actions.lock | 10 +++++----- .github/workflows/pull-request.yaml | 2 +- .github/workflows/release.yaml | 8 ++++---- .github/workflows/tests.yaml | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index b7ef44f..225e010 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -19,11 +19,11 @@ workflows: '.github/workflows/pull-request.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' - - 'docker/login-action@v4.7.0' + - 'docker/login-action@v4.8.0' '.github/workflows/release.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' - - 'docker/login-action@v4.7.0' + - 'docker/login-action@v4.8.0' '.github/workflows/tests.yaml': - 'actions/checkout@v7.0.1' - 'actions/download-artifact@v8.0.2' @@ -49,9 +49,9 @@ dependencies: commit: 'sha1-c3c9e263c25d99ce0380d002d59b67737d91b0dc' owner_id: 5429470 repo_id: 241092383 - 'docker/login-action@v4.7.0': - ref: 'v4.7.0' - commit: 'sha1-81c6f93d4bc991e510b342a00b419b3bb4efc5c6' + 'docker/login-action@v4.8.0': + ref: 'v4.8.0' + commit: 'sha1-8aa5831fb8d48d8f76594c7ddde61ae58019f550' owner_id: 5429470 repo_id: 287743349 'docker/setup-buildx-action@v4.4.1': diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index d488618..3ce4d3a 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -67,7 +67,7 @@ jobs: path: ${{ runner.temp }}/images - name: Log in to ghcr.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f50019a..df1ebda 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,14 +88,14 @@ jobs: path: ${{ runner.temp }}/images - name: Log in to ghcr.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to docker.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -164,14 +164,14 @@ jobs: echo "tags=$tags" | tee -a "$GITHUB_OUTPUT" - name: Log in to ghcr.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to docker.io - uses: docker/login-action@v4.7.0 + uses: docker/login-action@v4.8.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5322ecc..a1142c1 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -35,9 +35,9 @@ jobs: - name: Verify the action lockfile env: GH_TOKEN: ${{ github.token }} - # https://github.com/github/gh-actions-lock/releases/tag/v0.4.0 + # https://github.com/github/gh-actions-lock/releases/tag/v0.4.1 run: | - gh extension install github/gh-actions-lock --pin v0.4.0 + gh extension install github/gh-actions-lock --pin v0.4.1 gh actions-lock --verify test: From 901a7ec18ca3f7072b4d9eb3eae5f432b0715ee5 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:31 +0100 Subject: [PATCH 11/27] chore(lint): bump jactionlint from v2.4.0 to v2.5.0 --- scripts/lint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lint.sh b/scripts/lint.sh index 0624685..b696e30 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -8,8 +8,8 @@ HADOLINT=hadolint/hadolint:v2.15.1 SHELLCHECK=koalaman/shellcheck:v0.11.0 # https://github.com/mvdan/sh/releases/tag/v3.14.1 SHFMT=mvdan/shfmt:v3.14.1 -# https://github.com/jdx/jactionlint/releases/tag/v2.4.0 -JACTIONLINT=ghcr.io/jdx/jactionlint:2.4.0 +# https://github.com/jdx/jactionlint/releases/tag/v2.5.0 +JACTIONLINT=ghcr.io/jdx/jactionlint:2.5.0 # https://github.com/editorconfig-checker/editorconfig-checker/releases/tag/v4.0.2 EDITORCONFIG_CHECKER=mstruebing/editorconfig-checker:4.0.2 From a447a3c44c43f162aeeb1d5a03404dccb07d8da1 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:31 +0100 Subject: [PATCH 12/27] feat(image): move the home of nonroot to /home/nonroot --- .changes/home-of-nonroot-from-var-lib.md | 6 ++++++ src/rootfs.sh | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 .changes/home-of-nonroot-from-var-lib.md diff --git a/.changes/home-of-nonroot-from-var-lib.md b/.changes/home-of-nonroot-from-var-lib.md new file mode 100644 index 0000000..7444068 --- /dev/null +++ b/.changes/home-of-nonroot-from-var-lib.md @@ -0,0 +1,6 @@ +--- +type: changed +--- +Home of `nonroot` from `/var/lib/sqlite` to `/home/nonroot`, so `.sqlite_history` and whatever else a program writes under `HOME` stay out of the data directory + +- The working directory stays `/var/lib/sqlite`, where the databases are diff --git a/src/rootfs.sh b/src/rootfs.sh index 5a61243..808e307 100644 --- a/src/rootfs.sh +++ b/src/rootfs.sh @@ -30,12 +30,13 @@ install_binaries() { create_user() { install -d /rootfs/etc - echo "nonroot:x:65532:65532::/var/lib/sqlite:/sbin/nologin" > /rootfs/etc/passwd + echo "nonroot:x:65532:65532:nonroot:/home/nonroot:/sbin/nologin" > /rootfs/etc/passwd echo "nonroot:x:65532:" > /rootfs/etc/group } create_directories() { install -d -m 0777 -o 65532 -g 65532 /rootfs/var/lib/sqlite + install -d -m 0755 -o 65532 -g 65532 /rootfs/home/nonroot install -d -m 1777 /rootfs/tmp } From 93c975f0ad94bda1a6edd8021c1a61a722b56385 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:32 +0100 Subject: [PATCH 13/27] fix(image): list a directory once in the SBOM document --- .changes/sbom-document-that-listed-every-file.md | 6 ++++++ src/sbom.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changes/sbom-document-that-listed-every-file.md diff --git a/.changes/sbom-document-that-listed-every-file.md b/.changes/sbom-document-that-listed-every-file.md new file mode 100644 index 0000000..a88521f --- /dev/null +++ b/.changes/sbom-document-that-listed-every-file.md @@ -0,0 +1,6 @@ +--- +type: fixed +--- +SBOM document that listed every file of a directory as an entry of its own, which made it very large for a package with many files such as time zone data + +- A directory is one entry diff --git a/src/sbom.sh b/src/sbom.sh index 318262f..58a47d3 100644 --- a/src/sbom.sh +++ b/src/sbom.sh @@ -68,7 +68,7 @@ write_document() { verify_files() { jq -r '.artifacts[].metadata.files[]?.path' "$OUTPUT_FILE" | while read -r path; do - [ -f "/rootfs$path" ] || die "$path is not a file in /rootfs" + [ -e "/rootfs$path" ] || die "$path is not in /rootfs" done } From 59c264cfc36620670aa792cfbfec22ab364c4db9 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:32 +0100 Subject: [PATCH 14/27] refactor(image): read the root of the SBOM document from ROOT_DIR --- src/sbom.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sbom.sh b/src/sbom.sh index 58a47d3..480d864 100644 --- a/src/sbom.sh +++ b/src/sbom.sh @@ -4,7 +4,8 @@ set -eu USAGE="usage: sbom.sh IMAGE COMPONENT..., a component is apk|NAME|VENDOR:PRODUCT|PATH..., deb|NAME|LICENSE|VENDOR:PRODUCT|PATH... or generic|NAME|VERSION|LICENSE|URL|VENDOR:PRODUCT" IMAGE="${1:-}" REPOSITORY="${IMAGE%:*}" -OUTPUT_FILE="/rootfs/var/lib/db/sbom/${REPOSITORY##*/}-${IMAGE##*:}.syft.json" +ROOT_DIR="${ROOT_DIR:-/rootfs}" +OUTPUT_FILE="$ROOT_DIR/var/lib/db/sbom/${REPOSITORY##*/}-${IMAGE##*:}.syft.json" die() { echo "sbom: $*" >&2 @@ -68,7 +69,7 @@ write_document() { verify_files() { jq -r '.artifacts[].metadata.files[]?.path' "$OUTPUT_FILE" | while read -r path; do - [ -e "/rootfs$path" ] || die "$path is not in /rootfs" + [ -e "$ROOT_DIR$path" ] || die "$path is not in $ROOT_DIR" done } From 904b621bffe819eddd8f274eac3e1851e2506d5b Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:32 +0100 Subject: [PATCH 15/27] fix(image): make the SBOM document valid against its schema --- .changes/sbom-document-in-var-lib-db.md | 6 ++++++ src/Dockerfile | 3 +++ src/sbom.sh | 16 +++++++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 .changes/sbom-document-in-var-lib-db.md diff --git a/.changes/sbom-document-in-var-lib-db.md b/.changes/sbom-document-in-var-lib-db.md new file mode 100644 index 0000000..d7ca954 --- /dev/null +++ b/.changes/sbom-document-in-var-lib-db.md @@ -0,0 +1,6 @@ +--- +type: fixed +--- +SBOM document in `/var/lib/db/sbom` without the fields the syft schema requires + +- `src/Dockerfile` pins the version of the syft schema diff --git a/src/Dockerfile b/src/Dockerfile index 8878e64..d6e2066 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -10,6 +10,8 @@ ARG BASE_IMAGE=alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf5196 ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static ARG BUSYBOX_VERSION=1.37.0-r31 +# https://github.com/anchore/syft/tree/v1.57.1/schema/json +ARG SYFT_SCHEMA_VERSION=16.1.13 # Set by the build workflow ARG IMAGE_VERSION @@ -23,6 +25,7 @@ ARG SQLITE_VERSION ARG IMAGE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION +ARG SYFT_SCHEMA_VERSION # curl, jq and openssl are only used during the build stage, so they are not pinned to a specific version # hadolint ignore=DL3018 diff --git a/src/sbom.sh b/src/sbom.sh index 480d864..3377870 100644 --- a/src/sbom.sh +++ b/src/sbom.sh @@ -5,6 +5,7 @@ USAGE="usage: sbom.sh IMAGE COMPONENT..., a component is apk|NAME|VENDOR:PRODUCT IMAGE="${1:-}" REPOSITORY="${IMAGE%:*}" ROOT_DIR="${ROOT_DIR:-/rootfs}" +SCHEMA_VERSION="${SYFT_SCHEMA_VERSION:-}" OUTPUT_FILE="$ROOT_DIR/var/lib/db/sbom/${REPOSITORY##*/}-${IMAGE##*:}.syft.json" die() { @@ -14,6 +15,7 @@ die() { main() { [ "$#" -gt 1 ] || die "$USAGE" + [ -n "$SCHEMA_VERSION" ] || die "SYFT_SCHEMA_VERSION is not set, src/Dockerfile pins the version of the syft schema" shift write_document "$@" verify_files @@ -22,7 +24,7 @@ main() { write_document() { mkdir -p "${OUTPUT_FILE%/*}" - cat /lib/apk/db/installed /var/lib/dpkg/status 2> /dev/null | jq -Rsc --rawfile os_release /etc/os-release --arg usage "$USAGE" ' + cat /lib/apk/db/installed /var/lib/dpkg/status 2> /dev/null | jq -Rsc --rawfile os_release /etc/os-release --arg usage "$USAGE" --arg image "$IMAGE" --arg repository "$REPOSITORY" --arg tag "${IMAGE##*:}" --arg schema "$SCHEMA_VERSION" ' ([$os_release | split("\n")[] | capture("^(?[A-Z_]+)=\"?(?[^\"]*)")] | from_entries) as $os | [ split("\n\n")[] | [split("\n")[] | capture("^(?[^ :]+): ?(?.*)$")] | from_entries @@ -34,10 +36,14 @@ write_document() { def package($type; $name; $version; $license; $cpe; $purl): ($cpe | select(test("^[a-z0-9._-]+:[a-z0-9._-]+$")) // error("sbom: \($cpe) is not a CPE, expected VENDOR:PRODUCT")) as $cpe | { + id: "\($type)-\($name)-\($version)", name: $name, version: $version, type: $type, - licenses: [{ value: $license, spdxExpression: $license, type: "declared" }], + foundBy: "sbom.sh", + locations: [], + licenses: [{ value: $license, spdxExpression: $license, type: "declared", urls: [], locations: [] }], + language: "", cpes: [{ cpe: "cpe:2.3:a:\($cpe):\($version | gsub("(?[^A-Za-z0-9._-])"; "\\\(.c)")):*:*:*:*:*:*:*", source: "declared" }], purl: $purl }; @@ -63,7 +69,11 @@ write_document() { elif $type == "generic" and length == 6 then generic($name; .[2]; .[3]; .[4]; .[5]) else error("sbom: \(join("|")) is not a component, \($usage)") end ], - schema: { version: "16.1.10", url: "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-16.1.10.json" } + artifactRelationships: [], + source: { id: "", name: $repository, version: $tag, type: "image", metadata: { userInput: $image } }, + distro: {}, + descriptor: { name: "sbom.sh", version: "" }, + schema: { version: $schema, url: "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-\($schema).json" } }' --args "$@" > "$OUTPUT_FILE" } From 55684bafb38905cb95688c43fa3700e35ca9f5ce Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:58:32 +0100 Subject: [PATCH 16/27] test: wait for locks the healthcheck holds briefly --- scripts/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index f20fdc8..cce256f 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -53,7 +53,7 @@ check_default_database() { local settings docker run -d --name "$CONTAINER-default" --health-interval 2s --health-start-period 1s "$IMAGE" > /dev/null wait_healthy "$CONTAINER-default" - settings="$(docker exec "$CONTAINER-default" sqlite3 database.sqlite 'PRAGMA page_size; PRAGMA auto_vacuum; PRAGMA journal_mode;' | tr '\n' ' ')" + settings="$(docker exec "$CONTAINER-default" sqlite3 -cmd '.timeout 5000' database.sqlite 'PRAGMA page_size; PRAGMA auto_vacuum; PRAGMA journal_mode;' | tr '\n' ' ')" [ "$settings" = '4096 0 wal ' ] || die "database.sqlite was created with '$settings', expected page size 4096, no auto-vacuum, WAL" ok "database.sqlite is created by default with page size 4096, no auto-vacuum and WAL" } @@ -171,7 +171,7 @@ recover_database() { backup_database() { local result - docker exec "$CONTAINER" sh -c "sqlite3 app.sqlite \"VACUUM INTO 'backup.sqlite'\" && cat backup.sqlite && rm backup.sqlite" > "$WORK_DIR/backup.sqlite" + docker exec "$CONTAINER" sh -c "sqlite3 -cmd '.timeout 5000' app.sqlite \"VACUUM INTO 'backup.sqlite'\" && cat backup.sqlite && rm backup.sqlite" > "$WORK_DIR/backup.sqlite" result="$(docker run --rm -i "$IMAGE" sh -c 'cat > backup.sqlite && sqlite3 backup.sqlite "PRAGMA integrity_check; SELECT count(*) FROM t;"' < "$WORK_DIR/backup.sqlite" | tr '\n' ' ')" [ "$result" = 'ok 3 ' ] || die "the backup is not intact and complete, got '$result'" ok "a backup with VACUUM INTO streams out through docker exec, intact and complete" @@ -180,7 +180,7 @@ backup_database() { restore_database() { local result sql app.sqlite 'DELETE FROM t;' - docker exec -i "$CONTAINER" sh -c 'cat > restore.sqlite && sqlite3 app.sqlite ".restore restore.sqlite" && rm restore.sqlite' < "$WORK_DIR/backup.sqlite" + docker exec -i "$CONTAINER" sh -c 'cat > restore.sqlite && sqlite3 -cmd ".timeout 5000" app.sqlite ".restore restore.sqlite" && rm restore.sqlite' < "$WORK_DIR/backup.sqlite" result="$(sql app.sqlite 'PRAGMA integrity_check; PRAGMA journal_mode; SELECT count(*) FROM t;' | tr '\n' ' ')" [ "$result" = 'ok wal 3 ' ] || die "the restored database is not intact, got '$result'" [ "$(sql app.sqlite "SELECT body FROM docs WHERE docs MATCH 'sqlite';")" = 'sqlite rocks' ] || die "FTS5 after restore" @@ -206,7 +206,7 @@ wait_healthy() { } sql() { - docker exec "$CONTAINER" sqlite3 "$@" + docker exec "$CONTAINER" sqlite3 -cmd '.timeout 5000' "$@" } cleanup() { From 53633125559def02b736a80794aa41b61d3d20c4 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Wed, 11 Nov 2026 02:24:30 +0100 Subject: [PATCH 17/27] chore: write the brand as PeakImages - URLs, image names and label keys keep the lowercase handle --- .changes/vendor-and-authors-labels-from-peakimages.md | 4 ++++ src/Dockerfile | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .changes/vendor-and-authors-labels-from-peakimages.md diff --git a/.changes/vendor-and-authors-labels-from-peakimages.md b/.changes/vendor-and-authors-labels-from-peakimages.md new file mode 100644 index 0000000..9cc9119 --- /dev/null +++ b/.changes/vendor-and-authors-labels-from-peakimages.md @@ -0,0 +1,4 @@ +--- +type: changed +--- +Vendor and authors labels from `peakimages` to `PeakImages`, the name as it is written everywhere but in a URL or an image name diff --git a/src/Dockerfile b/src/Dockerfile index d6e2066..9cfd41c 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -108,8 +108,8 @@ LABEL org.opencontainers.image.title="SQLite" \ org.opencontainers.image.created="${CREATED}" \ org.opencontainers.image.revision="${GIT_SHA}" \ org.opencontainers.image.licenses="blessing AND MIT AND GPL-2.0-only AND AGPL-3.0-or-later" \ - org.opencontainers.image.authors="peakimages" \ - org.opencontainers.image.vendor="peakimages" \ + org.opencontainers.image.authors="PeakImages" \ + org.opencontainers.image.vendor="PeakImages" \ org.opencontainers.image.url="https://hub.docker.com/r/peakimages/sqlite" \ org.opencontainers.image.source="https://github.com/peakimages/sqlite" \ org.opencontainers.image.documentation="https://github.com/peakimages/sqlite#readme" \ From 95863174c69726c72804e36626f09e863a77bca9 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 10 Nov 2026 22:30:32 +0100 Subject: [PATCH 18/27] fix(image): let scanners find the right packages and the OS - the SBOM attestation was correct, but a scanner never reads it, it makes its own list from the image, and that list was wrong in syft, Grype and Trivy - syft listed busybox 1.37.0 alone without --select-catalogers +sbom-cataloger, Grype and Trivy could not read the image correctly at all - removes the syft document in /var/lib/db/sbom that BuildKit made the attestation from --- ...e-metadata-instead-of-the-sbom-document.md | 10 ++ .changes/sbom-document-in-var-lib-db.md | 6 - .../sbom-document-that-listed-every-file.md | 6 - README.md | 24 ++++ src/Dockerfile | 14 +- src/sbom.sh | 131 +++++++++--------- 6 files changed, 104 insertions(+), 87 deletions(-) create mode 100644 .changes/package-metadata-instead-of-the-sbom-document.md delete mode 100644 .changes/sbom-document-in-var-lib-db.md delete mode 100644 .changes/sbom-document-that-listed-every-file.md diff --git a/.changes/package-metadata-instead-of-the-sbom-document.md b/.changes/package-metadata-instead-of-the-sbom-document.md new file mode 100644 index 0000000..e068d6e --- /dev/null +++ b/.changes/package-metadata-instead-of-the-sbom-document.md @@ -0,0 +1,10 @@ +--- +type: fixed +--- +Scanners listing the wrong packages and the wrong OS, `syft` found only `busybox 1.37.0`, `grype` reported two BusyBox CVEs Alpine has fixed and none of SQLite, and Trivy found nothing + +- `sqlite3` carries a package note and the image the apk package database records of tini and BusyBox, `syft` and `grype` list all three, Trivy all but SQLite +- A scanner reports the image as Alpine, the release tini and BusyBox are taken from, instead of BusyBox or no OS, the same way Docker Hardened Images does +- The SBOM attestation lists the same three packages as before, it was already correct, but a scanner does not read it and makes its own list from the image +- The syft document in `/var/lib/db/sbom` that BuildKit made the SBOM attestation from is gone, BuildKit runs syft, which reads the package note and the apk package database natively +- `syft` no longer needs the `--select-catalogers +sbom-cataloger` flag to read the custom SBOM in the syft JSON file, as its default list is now correct too diff --git a/.changes/sbom-document-in-var-lib-db.md b/.changes/sbom-document-in-var-lib-db.md deleted file mode 100644 index d7ca954..0000000 --- a/.changes/sbom-document-in-var-lib-db.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -type: fixed ---- -SBOM document in `/var/lib/db/sbom` without the fields the syft schema requires - -- `src/Dockerfile` pins the version of the syft schema diff --git a/.changes/sbom-document-that-listed-every-file.md b/.changes/sbom-document-that-listed-every-file.md deleted file mode 100644 index a88521f..0000000 --- a/.changes/sbom-document-that-listed-every-file.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -type: fixed ---- -SBOM document that listed every file of a directory as an entry of its own, which made it very large for a package with many files such as time zone data - -- A directory is one entry diff --git a/README.md b/README.md index fa53581..e53bfd5 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,30 @@ The defaults are set in the image, override them with `-e` or the `environment` | `SQLITE_PAGE_SIZE` | `4096` | Page size of a new database, a power of two from `512` to `65536`. **Fixed once the database has content** | | `SQLITE_TMPDIR` | unset | Directory for the temporary files of sorts and vacuums. Default `/tmp`, or the data directory when `/tmp` is not writable | +## Attestations + +Every image carries an SBOM and a provenance attestation. + +The SBOM lists the packages of the image: + +```sh +docker buildx imagetools inspect peakimages/sqlite:latest --format '{{ json .SBOM }}' +``` + +The provenance records how the image was built: + +```sh +docker buildx imagetools inspect peakimages/sqlite:latest --format '{{ json .Provenance }}' +``` + +A scanner lists the same packages, Trivy all but SQLite: + +```sh +syft peakimages/sqlite:latest +grype peakimages/sqlite:latest +trivy image peakimages/sqlite:latest +``` + ## Releases Every SQLite release becomes a new version of the image. A check for new SQLite versions runs twice a day. diff --git a/src/Dockerfile b/src/Dockerfile index 9cfd41c..9119cd2 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -10,8 +10,6 @@ ARG BASE_IMAGE=alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf5196 ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static ARG BUSYBOX_VERSION=1.37.0-r31 -# https://github.com/anchore/syft/tree/v1.57.1/schema/json -ARG SYFT_SCHEMA_VERSION=16.1.13 # Set by the build workflow ARG IMAGE_VERSION @@ -22,10 +20,8 @@ ARG GIT_SHA FROM ${BASE_IMAGE} AS build ARG SQLITE_VERSION -ARG IMAGE_VERSION ARG TINI_VERSION ARG BUSYBOX_VERSION -ARG SYFT_SCHEMA_VERSION # curl, jq and openssl are only used during the build stage, so they are not pinned to a specific version # hadolint ignore=DL3018 @@ -84,12 +80,12 @@ COPY --chmod=0755 rootfs/ /rootfs/ COPY --chmod=0644 rootfs.sh /src/ RUN sh /src/rootfs.sh -# A scratch image has no package database for the SBOM scanner to read, this syft JSON document takes its place +# A scratch image has no package database for the SBOM scanner to read, sbom.sh declares its packages COPY --chmod=0644 sbom.sh /src/ -RUN sh /src/sbom.sh "peakimages/sqlite:${SQLITE_VERSION}-v${IMAGE_VERSION:?}" \ - "generic|sqlite|${SQLITE_VERSION}|blessing|$(jq -r .url /src/sqlite.json)|sqlite:sqlite" \ - "apk|tini-static|tini_project:tini|/sbin/tini" \ - "apk|busybox-static|busybox:busybox|/bin/busybox" +RUN sh /src/sbom.sh peakimages/sqlite "PeakImages SQLite" \ + "generic|sqlite|${SQLITE_VERSION}|blessing|sqlite:sqlite|/usr/local/bin/sqlite3" \ + "apk|tini-static|/sbin/tini" \ + "apk|busybox-static|/bin/busybox" FROM scratch AS final diff --git a/src/sbom.sh b/src/sbom.sh index 3377870..d87937d 100644 --- a/src/sbom.sh +++ b/src/sbom.sh @@ -1,12 +1,12 @@ #!/bin/sh set -eu -USAGE="usage: sbom.sh IMAGE COMPONENT..., a component is apk|NAME|VENDOR:PRODUCT|PATH..., deb|NAME|LICENSE|VENDOR:PRODUCT|PATH... or generic|NAME|VERSION|LICENSE|URL|VENDOR:PRODUCT" +USAGE="usage: sbom.sh IMAGE FULL_NAME COMPONENT..., a component is apk|NAME|PATH,..., deb|NAME|PATH,... or generic|NAME|VERSION|LICENSE|VENDOR:PRODUCT|PATH" IMAGE="${1:-}" -REPOSITORY="${IMAGE%:*}" +FULL_NAME="${2:-}" ROOT_DIR="${ROOT_DIR:-/rootfs}" -SCHEMA_VERSION="${SYFT_SCHEMA_VERSION:-}" -OUTPUT_FILE="$ROOT_DIR/var/lib/db/sbom/${REPOSITORY##*/}-${IMAGE##*:}.syft.json" +APK_DATABASE="$ROOT_DIR/lib/apk/db/installed" +DPKG_DATABASE="$ROOT_DIR/var/lib/dpkg/status.d" die() { echo "sbom: $*" >&2 @@ -14,73 +14,72 @@ die() { } main() { - [ "$#" -gt 1 ] || die "$USAGE" - [ -n "$SCHEMA_VERSION" ] || die "SYFT_SCHEMA_VERSION is not set, src/Dockerfile pins the version of the syft schema" - shift - write_document "$@" - verify_files - echo "sbom: $OUTPUT_FILE lists $(jq -r '[.artifacts[] | "\(.name) \(.version)"] | join(", ")' "$OUTPUT_FILE")" + [ "$#" -gt 2 ] || die "$USAGE" + shift 2 + for component in "$@"; do + case "$component" in + apk\|*\|*) write_apk_record "${component#apk|}" ;; + deb\|*\|*) write_deb_record "${component#deb|}" ;; + generic\|*) write_note "${component#generic|}" ;; + *) die "'$component' is not a component, $USAGE" ;; + esac + done + write_os_release + echo "sbom: $ROOT_DIR declares $(printf '%s\n' "$@" | cut -d '|' -f 2 | paste -sd , - | sed 's/,/, /g')" } -write_document() { - mkdir -p "${OUTPUT_FILE%/*}" - cat /lib/apk/db/installed /var/lib/dpkg/status 2> /dev/null | jq -Rsc --rawfile os_release /etc/os-release --arg usage "$USAGE" --arg image "$IMAGE" --arg repository "$REPOSITORY" --arg tag "${IMAGE##*:}" --arg schema "$SCHEMA_VERSION" ' - ([$os_release | split("\n")[] | capture("^(?[A-Z_]+)=\"?(?[^\"]*)")] | from_entries) as $os - | [ - split("\n\n")[] | [split("\n")[] | capture("^(?[^ :]+): ?(?.*)$")] | from_entries - | select(.Status | . == null or endswith(" installed")) - ] as $installed - | def installed($key; $name): first($installed[] | select(.[$key] == $name)) // error("sbom: \($name) is not installed"); - def purl($path; $version; $qualifiers): "pkg:\($path)@\($version)?" + ([$qualifiers | to_entries[] | select(.value | length > 0) | "\(.key)=\(.value | @uri)"] | join("&")); - def os_purl($type; $name; $version; $arch; $upstream): purl("\($type)/\($os.ID)/\($name)"; $version; { arch: $arch, distro: "\($os.ID)-\($os.VERSION_ID)", upstream: $upstream }); - def package($type; $name; $version; $license; $cpe; $purl): - ($cpe | select(test("^[a-z0-9._-]+:[a-z0-9._-]+$")) // error("sbom: \($cpe) is not a CPE, expected VENDOR:PRODUCT")) as $cpe - | { - id: "\($type)-\($name)-\($version)", - name: $name, - version: $version, - type: $type, - foundBy: "sbom.sh", - locations: [], - licenses: [{ value: $license, spdxExpression: $license, type: "declared", urls: [], locations: [] }], - language: "", - cpes: [{ cpe: "cpe:2.3:a:\($cpe):\($version | gsub("(?[^A-Za-z0-9._-])"; "\\\(.c)")):*:*:*:*:*:*:*", source: "declared" }], - purl: $purl - }; - def apk($name; $cpe; $paths): - installed("P"; $name) - | package("apk"; .P; .V; .L; $cpe; os_purl("apk"; .P; .V; .A; if .o != .P then .o else "" end)) + { - metadataType: "apk-db-entry", - metadata: { package: .P, originPackage: .o, version: .V, architecture: .A, files: [$paths[] | { path: . }] } - }; - def deb($name; $license; $cpe; $paths): - installed("Package"; $name) - | package("deb"; .Package; .Version; $license; $cpe; os_purl("deb"; .Package; .Version; .Architecture; .Source // "" | sub(" \\((?.*)\\)$"; "@\(.version)"))) + { - metadataType: "dpkg-db-entry", - metadata: { package: .Package, source: (.Source // "" | sub(" .*"; "")), version: .Version, architecture: .Architecture, files: [$paths[] | { path: . }] } - }; - def generic($name; $version; $license; $url; $cpe): - package("binary"; $name; $version; $license; $cpe; purl("generic/\($name)"; $version; { download_url: $url })); - { - artifacts: [ - $ARGS.positional[] | split("|") | . as [$type, $name] - | if $type == "apk" and length > 3 then apk($name; .[2]; .[3:]) - elif $type == "deb" and length > 4 then deb($name; .[2]; .[3]; .[4:]) - elif $type == "generic" and length == 6 then generic($name; .[2]; .[3]; .[4]; .[5]) - else error("sbom: \(join("|")) is not a component, \($usage)") end - ], - artifactRelationships: [], - source: { id: "", name: $repository, version: $tag, type: "image", metadata: { userInput: $image } }, - distro: {}, - descriptor: { name: "sbom.sh", version: "" }, - schema: { version: $schema, url: "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-\($schema).json" } - }' --args "$@" > "$OUTPUT_FILE" +write_apk_record() { + name="${1%%|*}" + mkdir -p "${APK_DATABASE%/*}" + { + awk -v RS= -v name="$name" 'index("\n" $0 "\n", "\nP:" name "\n") { sub(/\nF:.*/, ""); print; found = 1 } END { exit !found }' /lib/apk/db/installed \ + || die "$name is not installed" + for path in $(echo "${1#*|}" | tr ',' ' '); do + [ -e "$ROOT_DIR$path" ] || die "$path is not in $ROOT_DIR" + if [ -d "$ROOT_DIR$path" ]; then + echo "F:${path#/}" + else + printf 'F:%s\nR:%s\n' "$(dirname "${path#/}")" "${path##*/}" + fi + done + echo + } >> "$APK_DATABASE" } -verify_files() { - jq -r '.artifacts[].metadata.files[]?.path' "$OUTPUT_FILE" | while read -r path; do +write_deb_record() { + name="${1%%|*}" + mkdir -p "$DPKG_DATABASE" + awk -v RS= -v name="$name" 'index("\n" $0 "\n", "\nPackage: " name "\n") { sub(/\nConffiles:(\n [^\n]*)*/, ""); print; found = 1 } END { exit !found }' /var/lib/dpkg/status > "$DPKG_DATABASE/$name" \ + || die "$name is not installed" + for path in $(echo "${1#*|}" | tr ',' ' '); do [ -e "$ROOT_DIR$path" ] || die "$path is not in $ROOT_DIR" - done + (cd "$ROOT_DIR" && find "${path#/}" -type f -exec md5sum {} +) + done > "$DPKG_DATABASE/$name.md5sums" + install -D -m 0644 "/usr/share/doc/$name/copyright" "$ROOT_DIR/usr/share/doc/$name/copyright" + install -D -m 0644 /etc/debian_version "$ROOT_DIR/etc/debian_version" +} + +write_note() { + path="${1##*|}" + [ -f "$ROOT_DIR$path" ] || die "$path is not in $ROOT_DIR" + note="$(echo "$1" | jq -Rce 'split("|") | select(length == 5 and (.[3] | test("^[a-z0-9._-]+:[a-z0-9._-]+$"))) | { type: "generic", name: .[0], version: .[1], license: .[2], appCpe: "cpe:2.3:a:\(.[3]):\(.[1]):*:*:*:*:*:*:*" }')" \ + || die "'generic|$1' is not a component, $USAGE" + as -o /src/note.o /dev/null + ld -shared --package-metadata="$note" -o /src/note.so /src/note.o + objcopy -O binary --only-section=.note.package /src/note.so /src/note + objcopy --add-section .note.package=/src/note "$ROOT_DIR$path" + objcopy --set-section-alignment .note.package=4 "$ROOT_DIR$path" +} + +write_os_release() { + [ -e "$APK_DATABASE" ] || [ -d "$DPKG_DATABASE" ] || return 0 + distribution="$(awk -F= '{ gsub(/"/, "", $2) } $1 == "NAME" { name = $2 } $1 == "VERSION_ID" { version = $2 } END { print name, version }' /etc/os-release)" + { + printf 'NAME="%s"\n' "$FULL_NAME" + grep -Ev '^(PRETTY_NAME|NAME|[A-Z_]+_URL)=' /etc/os-release + printf 'PRETTY_NAME="%s (packages from %s)"\n' "$FULL_NAME" "$distribution" + printf 'HOME_URL="https://github.com/%s"\nBUG_REPORT_URL="https://github.com/%s/issues"\n' "$IMAGE" "$IMAGE" + } > "$ROOT_DIR/etc/os-release" } main "$@" From c034bf44d7fd89033e70f4716c387af1889ca9c0 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:49:44 +0100 Subject: [PATCH 19/27] docs: add a trademark notice to README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e53bfd5..06739e7 100644 --- a/README.md +++ b/README.md @@ -151,3 +151,7 @@ trivy image peakimages/sqlite:latest Every SQLite release becomes a new version of the image. A check for new SQLite versions runs twice a day. The image version follows semantic versioning independently of SQLite. Changes to the image are listed in [CHANGELOG.md](CHANGELOG.md). + +## Trademarks + +This image is built and published by PeakImages. All third-party product names, logos and trademarks are the property of their respective owners and are used solely for identification. PeakImages claims no interest in those marks, and no affiliation, sponsorship or endorsement is implied. From 1af0a8bc3e93ac8652bf922e5a7f63031d07040e Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:28:18 +0100 Subject: [PATCH 20/27] refactor(release): name upstream lines with a leading v --- RELEASE.md | 4 ++-- scripts/check-upstream.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 619e6d2..5264aaf 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -113,11 +113,11 @@ Missing description and license on ghcr.io 1. Pin the line on a branch from `main`: ```sh - git switch -c feat/sqlite-3.53 origin/main && scripts/check-upstream.sh 3.53 + git switch -c feat/sqlite-3.53 origin/main && scripts/check-upstream.sh v3.53 ``` 2. Squash merge a pull request as `feat(release): add the SQLite 3.53 line`, the push publishes `v3.53.4-v1.3.0` -3. To stop the line, remove `3.53` from `src/upstream.json`, its tags stay where they are +3. To stop the line, remove `v3.53` from `src/upstream.json`, its tags stay where they are ## Release a major version diff --git a/scripts/check-upstream.sh b/scripts/check-upstream.sh index 61944c2..b26c73f 100755 --- a/scripts/check-upstream.sh +++ b/scripts/check-upstream.sh @@ -19,7 +19,7 @@ cd "$(dirname "$0")/.." trap 'rm -rf "$WORK_DIR"' EXIT main() { - [[ "$OPTION" =~ ^(latest|[0-9]+(\.[0-9]+)*)?$ ]] || die "'$OPTION' is not an option, expected a line such as 3.53, latest or none" + [[ "$OPTION" =~ ^(latest|v[0-9]+(\.[0-9]+)?)?$ ]] || die "'$OPTION' is not an option, expected a line such as v3.53, latest or none" read_newest_release compare_with_pinned download_amalgamation @@ -34,7 +34,7 @@ read_newest_release() { local release year number fetch "$WORK_DIR/chronology.html" https://sqlite.org/chronology.html release="$(sed -nE "s|.*>([0-9]{4})-[0-9]{2}-[0-9]{2}\$|year \1|p; s|.*data-sortkey='([0-9]+)'>($VERSION_PATTERN).*|version \1 \2|p" "$WORK_DIR/chronology.html" \ - | awk -v line="$LINE." '$1 == "year" { year = $2 } $1 == "version" && (line == "latest." || index($3 ".", line) == 1) { print year, $2, $3 }' \ + | awk -v line="${LINE#v}." '$1 == "year" { year = $2 } $1 == "version" && (line == "latest." || index($3 ".", line) == 1) { print year, $2, $3 }' \ | sort -k3,3V | tail -n 1)" [ -n "$release" ] || die "sqlite.org/chronology.html lists no SQLite $LINE release" read -r year number new_version <<< "$release" From 112250be08200371d398247778ffbc90059c501a Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:45:47 +0100 Subject: [PATCH 21/27] fix(release): add the refactored type to change files - the changelog of v0.2.0 and the version pull request template have a Refactored section, which no change file could write --- scripts/change.sh | 4 ++-- scripts/release.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/change.sh b/scripts/change.sh index 7c1c1e9..d8ac894 100755 --- a/scripts/change.sh +++ b/scripts/change.sh @@ -2,8 +2,8 @@ set -euo pipefail CHANGE_DIRECTORY=.changes -TYPES='added,changed,deprecated,removed,fixed,security,maintenance' -SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Maintenance' +TYPES='added,changed,deprecated,removed,fixed,security,refactored,maintenance' +SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Refactored,Maintenance' BOLD=$'\033[1m' GREEN=$'\033[32m' RED=$'\033[31m' DIM=$'\033[2m' RESET=$'\033[0m' diff --git a/scripts/release.sh b/scripts/release.sh index 0a3642f..39f34f8 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -4,7 +4,7 @@ set -euo pipefail OPTION="$*" VERSION_FILE=release.json CHANGE_DIRECTORY=.changes -SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Maintenance' +SECTIONS='Breaking Changes,Security,Added,Changed,Deprecated,Removed,Fixed,Refactored,Maintenance' ARTIFACT=image UPSTREAM=sqlite WORK_DIR="$(mktemp -d)" From b348d39f9d8f689531f04c6b6ddd7a0df33ec073 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:45:47 +0100 Subject: [PATCH 22/27] build(image): bump Alpine from v3.24.3 to v3.24.4 --- .changes/alpine-in-the-build-stage-from.md | 4 ++++ src/Dockerfile | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .changes/alpine-in-the-build-stage-from.md diff --git a/.changes/alpine-in-the-build-stage-from.md b/.changes/alpine-in-the-build-stage-from.md new file mode 100644 index 0000000..6c204b7 --- /dev/null +++ b/.changes/alpine-in-the-build-stage-from.md @@ -0,0 +1,4 @@ +--- +type: changed +--- +Alpine in the build stage from 3.24.3 to 3.24.4 diff --git a/src/Dockerfile b/src/Dockerfile index 9119cd2..e903d8f 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -4,8 +4,8 @@ ARG SQLITE_VERSION=3.54.1 # Build dependencies -# https://hub.docker.com/_/alpine/tags?name=3.24.3 -ARG BASE_IMAGE=alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a +# https://hub.docker.com/_/alpine/tags?name=3.24.4 +ARG BASE_IMAGE=alpine:3.24.4@sha256:f59abe656b692d901e184be3445e084786602108599d2a51d229c0f5a48c71bf # https://pkgs.alpinelinux.org/package/v3.24/community/x86_64/tini-static ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static From 601e228692bcde8612e37f0ab7cc9a529ea888bf Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:28:23 +0100 Subject: [PATCH 23/27] fix(image): check the health of databases without a lock --- .../health-check-failing-with-database-is.md | 7 +++++++ scripts/test.sh | 20 +++++++++++++++++++ src/rootfs/usr/local/bin/healthcheck.sh | 10 +++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 .changes/health-check-failing-with-database-is.md diff --git a/.changes/health-check-failing-with-database-is.md b/.changes/health-check-failing-with-database-is.md new file mode 100644 index 0000000..24053a2 --- /dev/null +++ b/.changes/health-check-failing-with-database-is.md @@ -0,0 +1,7 @@ +--- +type: fixed +--- +Health check failing with `database is locked` while an application writes, and making the writes of an application without a busy timeout fail + +- The health check reads every database with `immutable=1`, which takes no lock +- When that read fails, it opens the database as before, with a busy timeout of 3 seconds diff --git a/scripts/test.sh b/scripts/test.sh index cce256f..def0a96 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -27,6 +27,7 @@ main() { start_container check_databases check_healthcheck + check_locked_database check_restart check_other_user check_creation_settings @@ -85,6 +86,25 @@ check_healthcheck() { ok "healthcheck.sh passes for app.sqlite,jobs.sqlite and fails as soon as one file is missing" } +check_locked_database() { + local holder + touch "$WORK_DIR/hold" + ( + echo 'PRAGMA locking_mode = EXCLUSIVE; BEGIN EXCLUSIVE;' + while [ -e "$WORK_DIR/hold" ]; do sleep 1; done + ) | docker exec -i "$CONTAINER" sqlite3 jobs.sqlite > /dev/null & + holder=$! + for _ in $(seq 30); do + docker exec "$CONTAINER" sqlite3 -readonly jobs.sqlite 'PRAGMA schema_version;' > /dev/null 2>&1 || break + sleep 1 + done + docker exec "$CONTAINER" healthcheck.sh jobs.sqlite || die "healthcheck.sh failed for jobs.sqlite while another connection holds it locked, expected it to take no lock" + docker exec "$CONTAINER" sqlite3 -readonly jobs.sqlite 'PRAGMA schema_version;' > /dev/null 2>&1 && die "jobs.sqlite is not locked, expected a connection in exclusive locking mode to hold it" + rm "$WORK_DIR/hold" + wait "$holder" || die "the connection that held jobs.sqlite locked failed" + ok "healthcheck.sh passes for jobs.sqlite while another connection holds it in exclusive locking mode" +} + check_restart() { sql app.sqlite 'PRAGMA journal_mode=DELETE;' > /dev/null docker restart -t 5 "$CONTAINER" > /dev/null diff --git a/src/rootfs/usr/local/bin/healthcheck.sh b/src/rootfs/usr/local/bin/healthcheck.sh index 8134d39..c5aa5fd 100755 --- a/src/rootfs/usr/local/bin/healthcheck.sh +++ b/src/rootfs/usr/local/bin/healthcheck.sh @@ -6,8 +6,16 @@ DATA_DIR=/var/lib/sqlite main() { [ "$#" -gt 0 ] || set -- "${SQLITE_DATABASES:-database.sqlite}" for name in $(printf '%s' "$*" | tr ',' ' '); do - sqlite3 -readonly "$DATA_DIR/$name" 'PRAGMA schema_version;' > /dev/null + read_without_lock "$DATA_DIR/$name" || read_with_lock "$DATA_DIR/$name" done } +read_without_lock() { + sqlite3 -readonly "file:$1?immutable=1" 'PRAGMA schema_version;' > /dev/null 2>&1 +} + +read_with_lock() { + sqlite3 -readonly -cmd '.timeout 3000' "$1" 'PRAGMA schema_version;' > /dev/null +} + main "$@" From 4f45cef54dc01ceb5687122f337eda0a284b6901 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:28:23 +0100 Subject: [PATCH 24/27] docs: describe the home of nonroot as a security change - on v0.2.0 a .sqliterc that any user wrote into the data directory ran in every health check, as nonroot --- .changes/home-of-nonroot-from-var-lib.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.changes/home-of-nonroot-from-var-lib.md b/.changes/home-of-nonroot-from-var-lib.md index 7444068..6b9541d 100644 --- a/.changes/home-of-nonroot-from-var-lib.md +++ b/.changes/home-of-nonroot-from-var-lib.md @@ -1,6 +1,7 @@ --- -type: changed +type: security --- -Home of `nonroot` from `/var/lib/sqlite` to `/home/nonroot`, so `.sqlite_history` and whatever else a program writes under `HOME` stay out of the data directory +Moved the home of `nonroot` from `/var/lib/sqlite` to `/home/nonroot`, `sqlite3` runs the `.sqliterc` of the home on every start, and every user can write `/var/lib/sqlite` -- The working directory stays `/var/lib/sqlite`, where the databases are +- `sqlite3` reads its startup file from `/home/nonroot/.sqliterc`, which only `nonroot` can write +- The working directory stays `/var/lib/sqlite` From 41139dd55b9bbf2d107370e939398bf0854ca354 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:30:32 +0100 Subject: [PATCH 25/27] test: check what the image does, not how it is built --- scripts/test.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index def0a96..e378d45 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -44,9 +44,9 @@ main() { check_version() { local printed version source_id printed="$(docker run --rm "$IMAGE" --version)" - version="$(docker image inspect --format '{{ index .Config.Labels "dev.peakimages.sqlite.version" }}' "$IMAGE")" + version="${printed%% *}" source_id="$(jq -r --arg version "$version" '.sqlite[] | select(.version == $version) | .source_id' src/upstream.json)" - [[ "$printed" == "$version $source_id "* ]] || die "sqlite3 --version prints '$printed', expected '$version $source_id'" + [[ -n "$source_id" && "$printed" == "$version $source_id "* ]] || die "sqlite3 --version prints '$printed', expected a version with the source id src/upstream.json pins for it" ok "sqlite3 --version prints $version with the source id src/upstream.json pins for it" } @@ -54,7 +54,7 @@ check_default_database() { local settings docker run -d --name "$CONTAINER-default" --health-interval 2s --health-start-period 1s "$IMAGE" > /dev/null wait_healthy "$CONTAINER-default" - settings="$(docker exec "$CONTAINER-default" sqlite3 -cmd '.timeout 5000' database.sqlite 'PRAGMA page_size; PRAGMA auto_vacuum; PRAGMA journal_mode;' | tr '\n' ' ')" + settings="$(docker exec "$CONTAINER-default" sqlite3 database.sqlite 'PRAGMA page_size; PRAGMA auto_vacuum; PRAGMA journal_mode;' | tr '\n' ' ')" [ "$settings" = '4096 0 wal ' ] || die "database.sqlite was created with '$settings', expected page size 4096, no auto-vacuum, WAL" ok "database.sqlite is created by default with page size 4096, no auto-vacuum and WAL" } @@ -66,9 +66,7 @@ start_container() { -e SQLITE_DATABASES="app.sqlite,jobs.sqlite" \ --health-interval 2s --health-start-period 1s "$IMAGE" > /dev/null wait_healthy "$CONTAINER" - # no exact match (-x), under QEMU ps shows "{sleep} /usr/bin/qemu-riscv64 /bin/sleep infinity" - docker exec "$CONTAINER" ps -o args | grep -q 'sleep infinity$' || die "the main process is not sleep infinity" - ok "healthy with a read-only root filesystem and all capabilities dropped, main process sleep infinity" + ok "healthy with a read-only root filesystem and all capabilities dropped" } check_databases() { @@ -191,7 +189,7 @@ recover_database() { backup_database() { local result - docker exec "$CONTAINER" sh -c "sqlite3 -cmd '.timeout 5000' app.sqlite \"VACUUM INTO 'backup.sqlite'\" && cat backup.sqlite && rm backup.sqlite" > "$WORK_DIR/backup.sqlite" + docker exec "$CONTAINER" sh -c "sqlite3 app.sqlite \"VACUUM INTO 'backup.sqlite'\" && cat backup.sqlite && rm backup.sqlite" > "$WORK_DIR/backup.sqlite" result="$(docker run --rm -i "$IMAGE" sh -c 'cat > backup.sqlite && sqlite3 backup.sqlite "PRAGMA integrity_check; SELECT count(*) FROM t;"' < "$WORK_DIR/backup.sqlite" | tr '\n' ' ')" [ "$result" = 'ok 3 ' ] || die "the backup is not intact and complete, got '$result'" ok "a backup with VACUUM INTO streams out through docker exec, intact and complete" @@ -200,7 +198,7 @@ backup_database() { restore_database() { local result sql app.sqlite 'DELETE FROM t;' - docker exec -i "$CONTAINER" sh -c 'cat > restore.sqlite && sqlite3 -cmd ".timeout 5000" app.sqlite ".restore restore.sqlite" && rm restore.sqlite' < "$WORK_DIR/backup.sqlite" + docker exec -i "$CONTAINER" sh -c 'cat > restore.sqlite && sqlite3 app.sqlite ".restore restore.sqlite" && rm restore.sqlite' < "$WORK_DIR/backup.sqlite" result="$(sql app.sqlite 'PRAGMA integrity_check; PRAGMA journal_mode; SELECT count(*) FROM t;' | tr '\n' ' ')" [ "$result" = 'ok wal 3 ' ] || die "the restored database is not intact, got '$result'" [ "$(sql app.sqlite "SELECT body FROM docs WHERE docs MATCH 'sqlite';")" = 'sqlite rocks' ] || die "FTS5 after restore" @@ -226,7 +224,7 @@ wait_healthy() { } sql() { - docker exec "$CONTAINER" sqlite3 -cmd '.timeout 5000' "$@" + docker exec "$CONTAINER" sqlite3 "$@" } cleanup() { From 7e28ce5199005053c67819c23d4a61b1fc80a427 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:30:32 +0100 Subject: [PATCH 26/27] feat(image): streamline image labels --- .changes/image-labels-to-one-set-that.md | 9 +++++++++ src/Dockerfile | 25 +++++++++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 .changes/image-labels-to-one-set-that.md diff --git a/.changes/image-labels-to-one-set-that.md b/.changes/image-labels-to-one-set-that.md new file mode 100644 index 0000000..42524f2 --- /dev/null +++ b/.changes/image-labels-to-one-set-that.md @@ -0,0 +1,9 @@ +--- +type: changed +--- +Image labels to one set that every image uses + +- `dev.peakimages.image` gains `distro`, `variant`, `flavor`, `shell` and `package-manager` +- `dev.peakimages.image.upstream.version` replaces `dev.peakimages.sqlite.version` +- `dev.peakimages.sqlite.extensions`, `dev.peakimages.tini.version` and `dev.peakimages.busybox.version` are removed as these labels do not scale to bigger images +- `org.opencontainers.image.licenses` names the license of the image and of SQLite, and `base.name` is empty with an empty `base.digest` for an image built from `scratch` diff --git a/src/Dockerfile b/src/Dockerfile index e903d8f..6bad7ea 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -6,6 +6,7 @@ ARG SQLITE_VERSION=3.54.1 # Build dependencies # https://hub.docker.com/_/alpine/tags?name=3.24.4 ARG BASE_IMAGE=alpine:3.24.4@sha256:f59abe656b692d901e184be3445e084786602108599d2a51d229c0f5a48c71bf +ARG DISTRO=alpine-3.24 # https://pkgs.alpinelinux.org/package/v3.24/community/x86_64/tini-static ARG TINI_VERSION=0.19.0-r3 # https://pkgs.alpinelinux.org/package/v3.24/main/x86_64/busybox-static @@ -89,10 +90,9 @@ RUN sh /src/sbom.sh peakimages/sqlite "PeakImages SQLite" \ FROM scratch AS final +ARG DISTRO ARG SQLITE_VERSION ARG IMAGE_VERSION -ARG TINI_VERSION -ARG BUSYBOX_VERSION ARG CREATED ARG GIT_SHA @@ -101,20 +101,23 @@ COPY --from=build /rootfs/ / LABEL org.opencontainers.image.title="SQLite" \ org.opencontainers.image.description="Minimal, production-ready, hardened and secure SQLite Docker image." \ org.opencontainers.image.version="${SQLITE_VERSION}-v${IMAGE_VERSION}" \ - org.opencontainers.image.created="${CREATED}" \ org.opencontainers.image.revision="${GIT_SHA}" \ - org.opencontainers.image.licenses="blessing AND MIT AND GPL-2.0-only AND AGPL-3.0-or-later" \ + org.opencontainers.image.created="${CREATED}" \ + org.opencontainers.image.licenses="AGPL-3.0-or-later AND blessing" \ org.opencontainers.image.authors="PeakImages" \ org.opencontainers.image.vendor="PeakImages" \ org.opencontainers.image.url="https://hub.docker.com/r/peakimages/sqlite" \ - org.opencontainers.image.source="https://github.com/peakimages/sqlite" \ org.opencontainers.image.documentation="https://github.com/peakimages/sqlite#readme" \ - org.opencontainers.image.base.name="scratch" \ - dev.peakimages.sqlite.version="${SQLITE_VERSION}" \ - dev.peakimages.sqlite.extensions="fts3,fts4,fts5,rtree,geopoly,math,dbstat,dbpage,stmt,soundex" \ - dev.peakimages.tini.version="${TINI_VERSION}" \ - dev.peakimages.busybox.version="${BUSYBOX_VERSION}" \ - dev.peakimages.image.version="${IMAGE_VERSION}" + org.opencontainers.image.source="https://github.com/peakimages/sqlite" \ + org.opencontainers.image.base.name="" \ + org.opencontainers.image.base.digest="" \ + dev.peakimages.image.version="${IMAGE_VERSION}" \ + dev.peakimages.image.upstream.version="${SQLITE_VERSION}" \ + dev.peakimages.image.distro="${DISTRO}" \ + dev.peakimages.image.variant="runtime" \ + dev.peakimages.image.flavor="" \ + dev.peakimages.image.shell="busybox" \ + dev.peakimages.image.package-manager="" ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ SQLITE_DATABASES=database.sqlite \ From 9960454c263a2de96c39b815d046a34abc612e53 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:30:32 +0100 Subject: [PATCH 27/27] style(image): remove comments that repeat the build steps --- src/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Dockerfile b/src/Dockerfile index 6bad7ea..0712350 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -3,7 +3,7 @@ # Set by scripts/check-upstream.sh ARG SQLITE_VERSION=3.54.1 -# Build dependencies +# Dependencies # https://hub.docker.com/_/alpine/tags?name=3.24.4 ARG BASE_IMAGE=alpine:3.24.4@sha256:f59abe656b692d901e184be3445e084786602108599d2a51d229c0f5a48c71bf ARG DISTRO=alpine-3.24 @@ -76,12 +76,10 @@ RUN gcc -static-pie -fPIE -Os \ shell.c sqlite3.c -lm -o sqlite3 \ && strip --strip-all sqlite3 -# Assemble the root filesystem of the final image in /rootfs COPY --chmod=0755 rootfs/ /rootfs/ COPY --chmod=0644 rootfs.sh /src/ RUN sh /src/rootfs.sh -# A scratch image has no package database for the SBOM scanner to read, sbom.sh declares its packages COPY --chmod=0644 sbom.sh /src/ RUN sh /src/sbom.sh peakimages/sqlite "PeakImages SQLite" \ "generic|sqlite|${SQLITE_VERSION}|blessing|sqlite:sqlite|/usr/local/bin/sqlite3" \