Pin pango and freetype2, fail on a differing comparison

.github/workflows/build.yml

@@ -214,8 +214,10 @@ jobs:
             dist/SHA256SUMS
           if-no-files-found: error
 
+      # bash, for pipefail: a differing file has to fail the job, not just `tee`.
       - name: Check it against the release
         if: inputs.compare-release != ''
+        shell: bash
         run: |
           asset="android-sysroot-${{ inputs.compare-release }}-$ABI.tar.gz"
           scripts/compare-sysroot "dist/android-sysroot-$TAG-$ABI.tar.gz" "reference/$asset" |

docs/ci-build.md

@@ -109,10 +109,16 @@ be copied to the path, which is what the workflow does.
 <sha>` is accepted by gitlab.gnome.org, gitlab.freedesktop.org and github, and pixiewood
 takes a per-dependency `revision`. `scripts/pin-sources.mjs` writes it for the four declared
 dependencies (glib, fontconfig, gtk, libadwaita); their transitive wraps are
-`wrap-redirect`s into those trees, so pinning four pins 22 of 24 sources. The two that stay
-floating are **pango** (`revision = main` in `gtk/subprojects/pango.wrap`) and **freetype2**
-(`revision = master` in `fontconfig/subprojects/freetype2.wrap`) — neither is reachable
-through pixiewood's `revision` attribute.
+`wrap-redirect`s into those trees, so pinning four pins 22 of 24 sources. The two that
+float are **pango** (`revision = main` in `gtk/subprojects/pango.wrap`) and **freetype2**
+(`revision = master` in `fontconfig/subprojects/freetype2.wrap`; its checkout and manifest
+entry are named `freetype`) — neither is reachable through pixiewood's `revision`
+attribute. Meson clones a wrap once, during `pixiewood prepare`'s `meson setup`, and never
+looks at its `revision` again, so editing the wrap afterwards does nothing. `pin-sources.mjs
+--checkouts subprojects` therefore runs between `prepare` and `build` and, in each of the
+two realized checkouts, fetches the manifest's commit (`git fetch --depth 1 origin <sha>`)
+and checks it out. It fails if the manifest records no commit or the checkout is missing.
+Meson regenerates on the changed sources, as it does for the libjpeg-turbo date.
 
 ## A cold build is not an incremental one
 
@@ -156,12 +162,13 @@ arm64-v8a gives the same three lines for `arm64-v8a.json` instead. All 21 typeli
   `typelibs` and `ndk` fields, are equal.
 
 `scripts/compare-sysroot <built> <reference>` is that check; it exits non-zero on any
-differing content.
+differing content, and the workflow step runs under `bash` with `pipefail`, so a differing
+file turns the job red (after the artifact is uploaded).
 
 ## What is left
 
-- **pango and freetype2 float.** Two consecutive builds can still differ, and nothing in
-  pixiewood's recipe format can pin them.
+- **Without `--pin`, pango and freetype2 float.** Two unpinned builds can differ; a pinned
+  one fixes them to the manifest's commits.
 - **The gir ordering race is worked around, not fixed.** The fix belongs in gtk's
   `meson.build`, upstream.
 - **`gobject-introspection` in the release manifest** is a stale entry, not a real input.

scripts/ci-build

@@ -30,8 +30,9 @@ usage() {
 Usage: ci-build [--abi x86_64|arm64-v8a] [--tag <tag>] [--out <dir>] [--pin <abi.json>]
                 [--build-date YYYYMMDD]
 
-  --pin         pin every declared dependency to the commit a release's manifest
-                records, so the result can be compared against that release
+  --pin         pin every dependency, pango and freetype2 included, to the commit a
+                release's manifest records, so the result can be compared against
+                that release
   --build-date  libjpeg-turbo's meson.build bakes `datetime.now()` into JVERSION
                 and the BUILD macro, which puts the build's calendar day into
                 libjpeg.so and libturbojpeg.so. Fixing it makes those two
@@ -68,6 +69,8 @@ for variable in ANDROID_SDK_ROOT ANDROID_NDK_ROOT ANDROID_SYSTEM_ROOT PIXIEWOOD;
 done
 
 repo=$(cd "$(dirname "$0")/.." && pwd)
+# The script changes directory before it needs the manifest again.
+[ -z "$pin" ] || pin=$(realpath "$pin")
 out=$(mkdir -p "$out" && cd "$out" && pwd)
 work=${CI_BUILD_WORK:-$(mktemp -d)/gi-probe}
 rm -rf "$work"
@@ -100,6 +103,10 @@ if [ -n "$build_date" ]; then
     done
 fi
 
+# pango and freetype2 come from nested wraps that track a branch, and meson never
+# revisits a wrap once it has cloned it, so they are fixed in the realized tree.
+[ -z "$pin" ] || node "$repo/scripts/pin-sources.mjs" --checkouts subprojects --manifest "$pin"
+
 # gtk's Gdk-4.0.gir target does not declare the pango GIRs it includes, so a
 # cold, fully parallel build can scan Gdk before PangoCairo-1.0.gir exists. An
 # incremental rebuild never hits it, which is why the hand build never saw it.

scripts/pin-sources.mjs

@@ -8,61 +8,115 @@
 // dependency, and every other subproject is a `wrap-redirect` into one of those
 // trees — so pinning the four the recipe declares pins the whole source set.
 //
+// Two sources are not reachable that way: pango and freetype2 are pulled in by
+// nested wraps that say `revision = main` / `master`. They are fixed after
+// `pixiewood prepare` has cloned them (see pinCheckouts), by checking out the
+// manifest's commit in the realized tree.
+//
 // The commits come from a release's own manifest, `share/android-sysroot/<abi>.json`.
 
-import { readFileSync, writeFileSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+/** Realized checkouts of the sources whose nested wraps track a branch, by directory name. */
+export const FLOATING = ['pango', 'freetype'];
+
+/**
+ * Check out, in each floating source under `subprojects`, the commit the manifest's
+ * `sources` records. Meson clones a wrap once and never revisits its `revision`, so
+ * this has to happen on the realized tree, between `pixiewood prepare` and `build`.
+ * A bare sha is fetchable with depth 1, as it is for pixiewood's own pinning.
+ */
+export function pinCheckouts(subprojects, sources, names = FLOATING) {
+    const done = [];
+    for (const name of names) {
+        const commit = sources[name]?.git;
+        if (!commit) throw new Error(`the manifest records no git commit for ${name}`);
+        const dir = join(subprojects, name);
+        if (!existsSync(join(dir, '.git'))) throw new Error(`${dir} is not a git checkout`);
+        const git = (...args) => execFileSync('git', ['-C', dir, ...args], { stdio: ['ignore', 'pipe', 'inherit'] });
+        git('fetch', '-q', '--depth', '1', 'origin', commit);
+        git('checkout', '-q', '--detach', commit);
+        const head = git('rev-parse', 'HEAD').toString().trim();
+        if (head !== commit) throw new Error(`${name} is at ${head}, not ${commit}`);
+        done.push(`${name}@${commit.slice(0, 10)}`);
+    }
+    return done;
+}
 
 function usage(code) {
     console.log(`Usage: pin-sources.mjs --xml <pixiewood.xml> --manifest <abi.json>
 
 Rewrites <xml> in place: every declared dependency the manifest knows a git commit
-for gets revision="<commit>".`);
-    process.exit(code);
-}
+for gets revision="<commit>".
+
+       pin-sources.mjs --checkouts <subprojects dir> --manifest <abi.json>
 
-const opts = {};
-for (let i = 2; i < process.argv.length; i += 2) {
-    const flag = process.argv[i];
-    if (flag === '--help' || flag === '-h') usage(0);
-    const value = process.argv[i + 1];
-    if (value === undefined) usage(2);
-    if (flag === '--xml') opts.xml = value;
-    else if (flag === '--manifest') opts.manifest = value;
-    else usage(2);
+Checks out the manifest's commit in the realized checkouts of the sources no
+revision attribute reaches (${FLOATING.join(', ')}). Fails if one is missing.`);
+    process.exit(code);
 }
-if (!opts.xml || !opts.manifest) usage(2);
 
-const sources = JSON.parse(readFileSync(opts.manifest, 'utf8')).sources ?? {};
-let xml = readFileSync(opts.xml, 'utf8');
+if (process.argv[1] !== fileURLToPath(import.meta.url)) {
+    // imported by a test
+} else {
+    const opts = {};
+    for (let i = 2; i < process.argv.length; i += 2) {
+        const flag = process.argv[i];
+        if (flag === '--help' || flag === '-h') usage(0);
+        const value = process.argv[i + 1];
+        if (value === undefined) usage(2);
+        if (flag === '--xml') opts.xml = value;
+        else if (flag === '--manifest') opts.manifest = value;
+        else if (flag === '--checkouts') opts.checkouts = value;
+        else usage(2);
+    }
+    if (!opts.manifest || !opts.xml === !opts.checkouts) usage(2);
 
-const dependencies = xml.match(/<dependencies>([\s\S]*?)<\/dependencies>/);
-if (!dependencies) {
-    console.error(`pin-sources: no <dependencies> in ${opts.xml}`);
-    process.exit(1);
-}
+    const sources = JSON.parse(readFileSync(opts.manifest, 'utf8')).sources ?? {};
 
-/** Children of a dependency, not dependencies themselves. */
-const NOT_A_DEPENDENCY = new Set(['patch', '/patch']);
+    if (opts.checkouts) {
+        try {
+            console.log(`checked out: ${pinCheckouts(opts.checkouts, sources).join(' ')}`);
+        } catch (error) {
+            console.error(`pin-sources: ${error.message}`);
+            process.exit(1);
+        }
+        process.exit(0);
+    }
+    let xml = readFileSync(opts.xml, 'utf8');
 
-const pinned = [];
-const unpinned = [];
-const block = dependencies[1].replace(/<([a-zA-Z/][\w-]*)([^>]*)>/g, (match, name, attributes) => {
-    if (NOT_A_DEPENDENCY.has(name) || name.startsWith('/')) return match;
-    if (attributes.includes('revision=')) return match;
-    const commit = sources[name]?.git;
-    if (!commit) {
-        unpinned.push(name);
-        return match;
+    const dependencies = xml.match(/<dependencies>([\s\S]*?)<\/dependencies>/);
+    if (!dependencies) {
+        console.error(`pin-sources: no <dependencies> in ${opts.xml}`);
+        process.exit(1);
     }
-    pinned.push(`${name}@${commit.slice(0, 10)}`);
-    // Both `<dep/>` and `<dep>` with children, hence the trailing slash dance.
-    const selfClosing = attributes.endsWith('/');
-    const rest = selfClosing ? attributes.slice(0, -1) : attributes;
-    return `<${name}${rest} revision="${commit}"${selfClosing ? '/' : ''}>`;
-});
 
-xml = xml.replace(dependencies[1], block);
-writeFileSync(opts.xml, xml);
+    /** Children of a dependency, not dependencies themselves. */
+    const NOT_A_DEPENDENCY = new Set(['patch', '/patch']);
 
-console.log(`pinned: ${pinned.join(' ') || 'nothing'}`);
-if (unpinned.length) console.log(`no commit in the manifest: ${unpinned.join(' ')}`);
+    const pinned = [];
+    const unpinned = [];
+    const block = dependencies[1].replace(/<([a-zA-Z/][\w-]*)([^>]*)>/g, (match, name, attributes) => {
+        if (NOT_A_DEPENDENCY.has(name) || name.startsWith('/')) return match;
+        if (attributes.includes('revision=')) return match;
+        const commit = sources[name]?.git;
+        if (!commit) {
+            unpinned.push(name);
+            return match;
+        }
+        pinned.push(`${name}@${commit.slice(0, 10)}`);
+        // Both `<dep/>` and `<dep>` with children, hence the trailing slash dance.
+        const selfClosing = attributes.endsWith('/');
+        const rest = selfClosing ? attributes.slice(0, -1) : attributes;
+        return `<${name}${rest} revision="${commit}"${selfClosing ? '/' : ''}>`;
+    });
+
+    xml = xml.replace(dependencies[1], block);
+    writeFileSync(opts.xml, xml);
+
+    console.log(`pinned: ${pinned.join(' ') || 'nothing'}`);
+    if (unpinned.length) console.log(`no commit in the manifest: ${unpinned.join(' ')}`);
+}

scripts/pin-sources.test.mjs

@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: MIT
+import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { test } from 'node:test';
+import { pinCheckouts } from './pin-sources.mjs';
+
+const git = (cwd, ...args) => execFileSync('git', ['-C', cwd, ...args], { encoding: 'utf8' }).trim();
+
+/** An upstream with two commits, and a depth-1 clone of its tip, as a wrap-git leaves it. */
+function floating(root, name) {
+    const upstream = join(root, 'upstream', name);
+    mkdirSync(upstream, { recursive: true });
+    git(upstream, 'init', '-q', '-b', 'main');
+    git(upstream, 'config', 'user.email', 't@example.org');
+    git(upstream, 'config', 'user.name', 't');
+    writeFileSync(join(upstream, 'f'), '1');
+    git(upstream, 'add', 'f');
+    git(upstream, 'commit', '-qm', 'one');
+    const first = git(upstream, 'rev-parse', 'HEAD');
+    writeFileSync(join(upstream, 'f'), '2');
+    git(upstream, 'commit', '-qam', 'two');
+    mkdirSync(join(root, 'subprojects'), { recursive: true });
+    execFileSync('git', ['clone', '-q', '--depth', '1', `file://${upstream}`, join(root, 'subprojects', name)]);
+    return first;
+}
+
+test('the realized checkouts move to the commits the manifest records', () => {
+    const root = mkdtempSync(join(tmpdir(), 'pin-sources-'));
+    const sources = { pango: { git: floating(root, 'pango') }, freetype: { git: floating(root, 'freetype') } };
+    const done = pinCheckouts(join(root, 'subprojects'), sources);
+    assert.equal(done.length, 2);
+    for (const name of ['pango', 'freetype']) {
+        assert.equal(git(join(root, 'subprojects', name), 'rev-parse', 'HEAD'), sources[name].git);
+    }
+});
+
+test('a missing commit or checkout fails loudly', () => {
+    const root = mkdtempSync(join(tmpdir(), 'pin-sources-'));
+    const first = floating(root, 'pango');
+    assert.throws(() => pinCheckouts(join(root, 'subprojects'), { pango: { git: first } }), /no git commit for freetype/);
+    assert.throws(() => pinCheckouts(join(root, 'subprojects'), { pango: {}, freetype: { git: first } }), /no git commit for pango/);
+    assert.throws(
+        () => pinCheckouts(join(root, 'subprojects'), { pango: { git: first }, freetype: { git: first } }),
+        /freetype is not a git checkout/,
+    );
+});