diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e0a2b55..c96c94f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -214,8 +214,10 @@ jobs: dist/SHA256SUMS if-no-files-found: error + # bash, for pipefail: a differing file has to fail the job, not just `tee`. - name: Check it against the release if: inputs.compare-release != '' + shell: bash run: | asset="android-sysroot-${{ inputs.compare-release }}-$ABI.tar.gz" scripts/compare-sysroot "dist/android-sysroot-$TAG-$ABI.tar.gz" "reference/$asset" | diff --git a/docs/ci-build.md b/docs/ci-build.md index 0bed995..c1c7e6c 100644 --- a/docs/ci-build.md +++ b/docs/ci-build.md @@ -109,10 +109,16 @@ be copied to the path, which is what the workflow does. ` is accepted by gitlab.gnome.org, gitlab.freedesktop.org and github, and pixiewood takes a per-dependency `revision`. `scripts/pin-sources.mjs` writes it for the four declared dependencies (glib, fontconfig, gtk, libadwaita); their transitive wraps are -`wrap-redirect`s into those trees, so pinning four pins 22 of 24 sources. The two that stay -floating are **pango** (`revision = main` in `gtk/subprojects/pango.wrap`) and **freetype2** -(`revision = master` in `fontconfig/subprojects/freetype2.wrap`) — neither is reachable -through pixiewood's `revision` attribute. +`wrap-redirect`s into those trees, so pinning four pins 22 of 24 sources. The two that +float are **pango** (`revision = main` in `gtk/subprojects/pango.wrap`) and **freetype2** +(`revision = master` in `fontconfig/subprojects/freetype2.wrap`; its checkout and manifest +entry are named `freetype`) — neither is reachable through pixiewood's `revision` +attribute. Meson clones a wrap once, during `pixiewood prepare`'s `meson setup`, and never +looks at its `revision` again, so editing the wrap afterwards does nothing. `pin-sources.mjs +--checkouts subprojects` therefore runs between `prepare` and `build` and, in each of the +two realized checkouts, fetches the manifest's commit (`git fetch --depth 1 origin `) +and checks it out. It fails if the manifest records no commit or the checkout is missing. +Meson regenerates on the changed sources, as it does for the libjpeg-turbo date. ## A cold build is not an incremental one @@ -156,12 +162,13 @@ arm64-v8a gives the same three lines for `arm64-v8a.json` instead. All 21 typeli `typelibs` and `ndk` fields, are equal. `scripts/compare-sysroot ` is that check; it exits non-zero on any -differing content. +differing content, and the workflow step runs under `bash` with `pipefail`, so a differing +file turns the job red (after the artifact is uploaded). ## What is left -- **pango and freetype2 float.** Two consecutive builds can still differ, and nothing in - pixiewood's recipe format can pin them. +- **Without `--pin`, pango and freetype2 float.** Two unpinned builds can differ; a pinned + one fixes them to the manifest's commits. - **The gir ordering race is worked around, not fixed.** The fix belongs in gtk's `meson.build`, upstream. - **`gobject-introspection` in the release manifest** is a stale entry, not a real input. diff --git a/scripts/ci-build b/scripts/ci-build index 2eb2e27..855c43e 100755 --- a/scripts/ci-build +++ b/scripts/ci-build @@ -30,8 +30,9 @@ usage() { Usage: ci-build [--abi x86_64|arm64-v8a] [--tag ] [--out ] [--pin ] [--build-date YYYYMMDD] - --pin pin every declared dependency to the commit a release's manifest - records, so the result can be compared against that release + --pin pin every dependency, pango and freetype2 included, to the commit a + release's manifest records, so the result can be compared against + that release --build-date libjpeg-turbo's meson.build bakes `datetime.now()` into JVERSION and the BUILD macro, which puts the build's calendar day into libjpeg.so and libturbojpeg.so. Fixing it makes those two @@ -68,6 +69,8 @@ for variable in ANDROID_SDK_ROOT ANDROID_NDK_ROOT ANDROID_SYSTEM_ROOT PIXIEWOOD; done repo=$(cd "$(dirname "$0")/.." && pwd) +# The script changes directory before it needs the manifest again. +[ -z "$pin" ] || pin=$(realpath "$pin") out=$(mkdir -p "$out" && cd "$out" && pwd) work=${CI_BUILD_WORK:-$(mktemp -d)/gi-probe} rm -rf "$work" @@ -100,6 +103,10 @@ if [ -n "$build_date" ]; then done fi +# pango and freetype2 come from nested wraps that track a branch, and meson never +# revisits a wrap once it has cloned it, so they are fixed in the realized tree. +[ -z "$pin" ] || node "$repo/scripts/pin-sources.mjs" --checkouts subprojects --manifest "$pin" + # gtk's Gdk-4.0.gir target does not declare the pango GIRs it includes, so a # cold, fully parallel build can scan Gdk before PangoCairo-1.0.gir exists. An # incremental rebuild never hits it, which is why the hand build never saw it. diff --git a/scripts/pin-sources.mjs b/scripts/pin-sources.mjs index 5867bc2..ce7c4d7 100644 --- a/scripts/pin-sources.mjs +++ b/scripts/pin-sources.mjs @@ -8,61 +8,115 @@ // dependency, and every other subproject is a `wrap-redirect` into one of those // trees — so pinning the four the recipe declares pins the whole source set. // +// Two sources are not reachable that way: pango and freetype2 are pulled in by +// nested wraps that say `revision = main` / `master`. They are fixed after +// `pixiewood prepare` has cloned them (see pinCheckouts), by checking out the +// manifest's commit in the realized tree. +// // The commits come from a release's own manifest, `share/android-sysroot/.json`. -import { readFileSync, writeFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** Realized checkouts of the sources whose nested wraps track a branch, by directory name. */ +export const FLOATING = ['pango', 'freetype']; + +/** + * Check out, in each floating source under `subprojects`, the commit the manifest's + * `sources` records. Meson clones a wrap once and never revisits its `revision`, so + * this has to happen on the realized tree, between `pixiewood prepare` and `build`. + * A bare sha is fetchable with depth 1, as it is for pixiewood's own pinning. + */ +export function pinCheckouts(subprojects, sources, names = FLOATING) { + const done = []; + for (const name of names) { + const commit = sources[name]?.git; + if (!commit) throw new Error(`the manifest records no git commit for ${name}`); + const dir = join(subprojects, name); + if (!existsSync(join(dir, '.git'))) throw new Error(`${dir} is not a git checkout`); + const git = (...args) => execFileSync('git', ['-C', dir, ...args], { stdio: ['ignore', 'pipe', 'inherit'] }); + git('fetch', '-q', '--depth', '1', 'origin', commit); + git('checkout', '-q', '--detach', commit); + const head = git('rev-parse', 'HEAD').toString().trim(); + if (head !== commit) throw new Error(`${name} is at ${head}, not ${commit}`); + done.push(`${name}@${commit.slice(0, 10)}`); + } + return done; +} function usage(code) { console.log(`Usage: pin-sources.mjs --xml --manifest Rewrites in place: every declared dependency the manifest knows a git commit -for gets revision="".`); - process.exit(code); -} +for gets revision="". + + pin-sources.mjs --checkouts --manifest -const opts = {}; -for (let i = 2; i < process.argv.length; i += 2) { - const flag = process.argv[i]; - if (flag === '--help' || flag === '-h') usage(0); - const value = process.argv[i + 1]; - if (value === undefined) usage(2); - if (flag === '--xml') opts.xml = value; - else if (flag === '--manifest') opts.manifest = value; - else usage(2); +Checks out the manifest's commit in the realized checkouts of the sources no +revision attribute reaches (${FLOATING.join(', ')}). Fails if one is missing.`); + process.exit(code); } -if (!opts.xml || !opts.manifest) usage(2); -const sources = JSON.parse(readFileSync(opts.manifest, 'utf8')).sources ?? {}; -let xml = readFileSync(opts.xml, 'utf8'); +if (process.argv[1] !== fileURLToPath(import.meta.url)) { + // imported by a test +} else { + const opts = {}; + for (let i = 2; i < process.argv.length; i += 2) { + const flag = process.argv[i]; + if (flag === '--help' || flag === '-h') usage(0); + const value = process.argv[i + 1]; + if (value === undefined) usage(2); + if (flag === '--xml') opts.xml = value; + else if (flag === '--manifest') opts.manifest = value; + else if (flag === '--checkouts') opts.checkouts = value; + else usage(2); + } + if (!opts.manifest || !opts.xml === !opts.checkouts) usage(2); -const dependencies = xml.match(/([\s\S]*?)<\/dependencies>/); -if (!dependencies) { - console.error(`pin-sources: no in ${opts.xml}`); - process.exit(1); -} + const sources = JSON.parse(readFileSync(opts.manifest, 'utf8')).sources ?? {}; -/** Children of a dependency, not dependencies themselves. */ -const NOT_A_DEPENDENCY = new Set(['patch', '/patch']); + if (opts.checkouts) { + try { + console.log(`checked out: ${pinCheckouts(opts.checkouts, sources).join(' ')}`); + } catch (error) { + console.error(`pin-sources: ${error.message}`); + process.exit(1); + } + process.exit(0); + } + let xml = readFileSync(opts.xml, 'utf8'); -const pinned = []; -const unpinned = []; -const block = dependencies[1].replace(/<([a-zA-Z/][\w-]*)([^>]*)>/g, (match, name, attributes) => { - if (NOT_A_DEPENDENCY.has(name) || name.startsWith('/')) return match; - if (attributes.includes('revision=')) return match; - const commit = sources[name]?.git; - if (!commit) { - unpinned.push(name); - return match; + const dependencies = xml.match(/([\s\S]*?)<\/dependencies>/); + if (!dependencies) { + console.error(`pin-sources: no in ${opts.xml}`); + process.exit(1); } - pinned.push(`${name}@${commit.slice(0, 10)}`); - // Both `` and `` with children, hence the trailing slash dance. - const selfClosing = attributes.endsWith('/'); - const rest = selfClosing ? attributes.slice(0, -1) : attributes; - return `<${name}${rest} revision="${commit}"${selfClosing ? '/' : ''}>`; -}); -xml = xml.replace(dependencies[1], block); -writeFileSync(opts.xml, xml); + /** Children of a dependency, not dependencies themselves. */ + const NOT_A_DEPENDENCY = new Set(['patch', '/patch']); -console.log(`pinned: ${pinned.join(' ') || 'nothing'}`); -if (unpinned.length) console.log(`no commit in the manifest: ${unpinned.join(' ')}`); + const pinned = []; + const unpinned = []; + const block = dependencies[1].replace(/<([a-zA-Z/][\w-]*)([^>]*)>/g, (match, name, attributes) => { + if (NOT_A_DEPENDENCY.has(name) || name.startsWith('/')) return match; + if (attributes.includes('revision=')) return match; + const commit = sources[name]?.git; + if (!commit) { + unpinned.push(name); + return match; + } + pinned.push(`${name}@${commit.slice(0, 10)}`); + // Both `` and `` with children, hence the trailing slash dance. + const selfClosing = attributes.endsWith('/'); + const rest = selfClosing ? attributes.slice(0, -1) : attributes; + return `<${name}${rest} revision="${commit}"${selfClosing ? '/' : ''}>`; + }); + + xml = xml.replace(dependencies[1], block); + writeFileSync(opts.xml, xml); + + console.log(`pinned: ${pinned.join(' ') || 'nothing'}`); + if (unpinned.length) console.log(`no commit in the manifest: ${unpinned.join(' ')}`); +} diff --git a/scripts/pin-sources.test.mjs b/scripts/pin-sources.test.mjs new file mode 100644 index 0000000..ea8fddd --- /dev/null +++ b/scripts/pin-sources.test.mjs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { pinCheckouts } from './pin-sources.mjs'; + +const git = (cwd, ...args) => execFileSync('git', ['-C', cwd, ...args], { encoding: 'utf8' }).trim(); + +/** An upstream with two commits, and a depth-1 clone of its tip, as a wrap-git leaves it. */ +function floating(root, name) { + const upstream = join(root, 'upstream', name); + mkdirSync(upstream, { recursive: true }); + git(upstream, 'init', '-q', '-b', 'main'); + git(upstream, 'config', 'user.email', 't@example.org'); + git(upstream, 'config', 'user.name', 't'); + writeFileSync(join(upstream, 'f'), '1'); + git(upstream, 'add', 'f'); + git(upstream, 'commit', '-qm', 'one'); + const first = git(upstream, 'rev-parse', 'HEAD'); + writeFileSync(join(upstream, 'f'), '2'); + git(upstream, 'commit', '-qam', 'two'); + mkdirSync(join(root, 'subprojects'), { recursive: true }); + execFileSync('git', ['clone', '-q', '--depth', '1', `file://${upstream}`, join(root, 'subprojects', name)]); + return first; +} + +test('the realized checkouts move to the commits the manifest records', () => { + const root = mkdtempSync(join(tmpdir(), 'pin-sources-')); + const sources = { pango: { git: floating(root, 'pango') }, freetype: { git: floating(root, 'freetype') } }; + const done = pinCheckouts(join(root, 'subprojects'), sources); + assert.equal(done.length, 2); + for (const name of ['pango', 'freetype']) { + assert.equal(git(join(root, 'subprojects', name), 'rev-parse', 'HEAD'), sources[name].git); + } +}); + +test('a missing commit or checkout fails loudly', () => { + const root = mkdtempSync(join(tmpdir(), 'pin-sources-')); + const first = floating(root, 'pango'); + assert.throws(() => pinCheckouts(join(root, 'subprojects'), { pango: { git: first } }), /no git commit for freetype/); + assert.throws(() => pinCheckouts(join(root, 'subprojects'), { pango: {}, freetype: { git: first } }), /no git commit for pango/); + assert.throws( + () => pinCheckouts(join(root, 'subprojects'), { pango: { git: first }, freetype: { git: first } }), + /freetype is not a git checkout/, + ); +});