From 876db229b5c919261936092d039d0bb611786dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Fri, 20 Nov 2026 21:23:45 -0500 Subject: [PATCH 1/6] Webhook: a job without a job label gets no runner, and swaps are logged, not repaired A queued job in shape but without a ci-job- label is ignored (ignored: labels) with a log line naming why. On in_progress, a runner of this stack running another job than its own is logged as an anomaly; nothing is re-queued or re-tagged. The webhook role loses ec2:DescribeInstances, ec2:CreateTags, ecs:ListTasks, ecs:DescribeTasks and ecs:TagResource, and the function loses CLUSTER. Co-Authored-By: Claude Opus 5.5 --- functions/launcher.js | 6 +- functions/notifier.js | 6 +- functions/webhook.js | 119 ++++++---------------- template.yaml | 33 +------ test/conventions.test.js | 23 +---- test/notifier.test.js | 24 +---- test/tasks-ecs.test.js | 48 +-------- test/template.test.js | 16 +-- test/webhook.test.js | 207 ++++++++------------------------------- 9 files changed, 88 insertions(+), 394 deletions(-) diff --git a/functions/launcher.js b/functions/launcher.js index 3971c8c..48f0d8a 100644 --- a/functions/launcher.js +++ b/functions/launcher.js @@ -35,7 +35,7 @@ const RECHECK_SECONDS = 300; // a runner's idle exit, for the first relaunch, doubling for each after, up to // 60 minutes. A held concurrency group still frees into a runner within the // hour. The message carries the count and the time of the job's last launch; -// one without them (the webhook's, a swap's re-queue) has had none. +// one without them (the webhook's) has had none. const relaunchGapMs = (launches) => Math.min(15 * 2 ** (launches - 1), 60) * 60e3; const b64u = (s) => Buffer.from(s).toString('base64url'); @@ -129,8 +129,8 @@ function createHandler({ secret, fetch, ec2, ecs, sqs, now, env, log = console.l return out; } - // The tags a runner's instance or task carries: the job, for the swap - // repair and the notifier, and the shape, for the budgets. + // The tags a runner's instance or task carries: the job, for the notifier + // and the re-check (#51), and the shape, for the budgets. function jobTags(m, name, runnerId, labelsSep) { const { vcpus, arch } = shapeOf(m); return [ diff --git a/functions/notifier.js b/functions/notifier.js index 7f53404..e88d9c0 100644 --- a/functions/notifier.js +++ b/functions/notifier.js @@ -121,9 +121,9 @@ function createHandler({ secret, fetch, ec2, ecs, now, env, log = console.log }) return log(`already reported ${id}: ${repo} job ${jobId}`); } // Only a job this machine's CI runner is running can be lost with it. The - // tags name the job the machine was launched for, or took in a swap (the - // webhook re-tags it); a CI runner that took no job (another one served - // its launch job) is idle, and a completed job is no loss. + // tags name the job the machine was launched for, the only one its runner + // can take (ADR-0004); a CI runner that took no job is idle, and a + // completed job is no loss. if (ghJob.status !== 'in_progress' || ghJob.runner_name !== t.runner) { return log(`ignored ${id}: its runner is not running job ${jobId} (${ghJob.status}, runner ${ghJob.runner_name || 'none'})`); } diff --git a/functions/webhook.js b/functions/webhook.js index f961531..2614e72 100644 --- a/functions/webhook.js +++ b/functions/webhook.js @@ -2,17 +2,15 @@ // webhook: GitHub workflow_job receiver behind a Function URL. Verifies the // delivery, applies the label contract, the job label (ADR-0004) and fence 3, // and enqueues the job for the launcher. It launches nothing itself, but -// repairs swaps of jobs without the job label (ADR-0003) and -// tells the notifier when a job of this stack's runners succeeds. +// logs a runner that runs another job than its own, which the job label +// should make impossible, and tells the notifier when a job of this stack's +// runners succeeds. // Inlined into the template; keep it one dependency-free CommonJS file. const crypto = require('node:crypto'); const CAPACITY = ['ci-spot', 'ci-ondemand', 'ci-fargate-spot', 'ci-fargate']; const SIZE = { 'ci-cpu-2': 2, 'ci-cpu-4': 4, 'ci-cpu-8': 8, 'ci-cpu-16': 16 }; const ARCH = ['x64', 'arm64']; -// The shape of an instance or task launched before the shape tags, which -// swap repair re-queues with it. The launcher carries a copy. -const DEFAULT_SHAPE = { vcpus: 4, arch: 'x64' }; function reply(statusCode, text) { return { statusCode, headers: { 'content-type': 'text/plain; charset=utf-8' }, body: text + '\n' }; @@ -33,8 +31,8 @@ function signatureOk(key, raw, sig) { // The label contract: `self-hosted`, exactly one capacity label, exactly one // size label, exactly one architecture label, and optionally `linux`. None has -// a default, so a swap only happens between runners of the same shape. A job -// label, checked by jobLabelFault after, is no part of the shape. +// a default. The job label, checked by jobLabelFault after, is no part of the +// shape. // Returns the job's shape, or null outside it. function shapeOf(labels) { const ls = (labels || []).filter((l) => !isJobLabel(l)).map((l) => String(l).toLowerCase()); @@ -50,13 +48,14 @@ function shapeOf(labels) { // The job label (ADR-0004), `ci-job----`: unique to one job, so the runner launched for the job, which -// registers with the job's labels, can take no other. Optional for now: a job -// without it is served as before. Returns why a job's labels break it, as -// { label, why }, or null. +// registers with the job's labels, can take no other. Required: GitHub gives +// a job to any runner whose labels include all of the job's, so a job without +// one could take any runner of its shape (a swap). Returns why a job's job +// label breaks it, as { label, why }, or null; the handler checks first that +// it has one. const isJobLabel = (l) => String(l).toLowerCase().startsWith('ci-job-'); function jobLabelFault(labels, job) { const ls = (labels || []).filter(isJobLabel).map(String); - if (!ls.length) return null; if (ls.length > 1) return { label: ls.join(','), why: 'two job labels' }; const label = ls[0]; const m = label.toLowerCase().match(/^ci-job-(\d+)-(\d+)-(.*)$/); @@ -71,33 +70,7 @@ function jobLabelFault(labels, job) { return null; } -// The job fields the instance or task carries, as tags. -const JOB_TAGS = { repo: 'repo', job_id: 'job-id', run_id: 'run-id', head_sha: 'head-sha', job_name: 'job-name' }; -// ECS allows only these characters in a tag (EC2 allows any). A copy of the -// launcher's, so a re-tag reads as the launch did. -const ECS_TAG_CHARS = /[^\p{L}\p{Z}\p{N}_.:/=+\-@]/gu; -const ecsTag = ([key, v]) => ({ key, value: String(v).replace(ECS_TAG_CHARS, '_').slice(0, 256) }); - -// An EC2 (Key, Value) or ECS (key, value) tag list as an object, -// keyed without the ci-runner: prefix. -const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value])); - -function createHandler({ secret, queue, notifier, ec2, ecs, env, log = console.log }) { - // The runner's task in the cluster, not yet told to stop, with its tags. - async function findTask(name) { - let nextToken; - do { - const r = await ecs.listTasks({ cluster: env.CLUSTER, desiredStatus: 'RUNNING', nextToken }); - if ((r.taskArns || []).length) { - const d = await ecs.describeTasks({ cluster: env.CLUSTER, tasks: r.taskArns, include: ['TAGS'] }); - const t = (d.tasks || []).find((x) => (x.tags || []).some((g) => g.key === 'ci-runner:runner' && g.value === name)); - if (t) return t; - } - nextToken = r.nextToken; - } while (nextToken); - return undefined; - } - +function createHandler({ secret, queue, notifier, env, log = console.log }) { let key; // the webhook secret, cached once the App exists // The job id a runner of this stack was launched for, from its name @@ -107,47 +80,18 @@ function createHandler({ secret, queue, notifier, ec2, ecs, env, log = console.l return m && m[1] === env.STACK_NAME ? Number(m[2]) : null; } - // A swap: a runner of this stack, named `${stack}--`, started - // a job other than the one it was launched for. Re-queue that one, so the - // launcher's "still queued?" check launches it a runner or drops it, and - // re-tag the instance or task so the notifier names the job it runs. - async function repairSwap(p, repo, job) { - const name = String(job.runner_name || ''); - const launchJob = launchJobOf(name); - if (launchJob === null || launchJob === job.id) return reply(200, 'ignored: no swap'); - const r = await ec2.describeInstances({ - Filters: [ - { Name: 'tag:ci-runner:stack', Values: [env.STACK_NAME] }, - { Name: 'tag:ci-runner:runner', Values: [name] }, - { Name: 'instance-state-name', Values: ['pending', 'running'] }, - ], - }); - const i = (r.Reservations || []).flatMap((x) => x.Instances || [])[0]; - const task = !i && (await findTask(name)); - if (!i && !task) return reply(200, 'ignored: no instance or task'); - const t = tagsOf(i ? i.Tags : task.tags); - // Already re-tagged for this job: a duplicate delivery. - if (!t['job-id'] || Number(t['job-id']) === job.id) return reply(200, 'ignored: no swap'); - - const orig = { - repo: t.repo, job_id: Number(t['job-id']), run_id: Number(t['run-id']), head_sha: t['head-sha'], - job_name: t['job-name'], label: t.label, vcpus: Number(t.vcpus) || DEFAULT_SHAPE.vcpus, arch: t.arch || DEFAULT_SHAPE.arch, - // A task's labels are space-joined: ECS refuses a comma in a tag. - labels: t.labels ? t.labels.split(i ? ',' : ' ') : ['self-hosted', t.label], - installation_id: p.installation && p.installation.id, - }; - // Queue first: if the re-tag then fails, a redelivery still finds the - // old tags and repairs again, rather than skip a job never re-queued. - await queue.send(orig); - const ran = { repo: repo.full_name, job_id: job.id, run_id: job.run_id, head_sha: job.head_sha, job_name: job.name }; - const tags = Object.entries(JOB_TAGS).map(([f, k]) => [`ci-runner:${k}`, ran[f]]); - if (i) await ec2.createTags({ Resources: [i.InstanceId], Tags: tags.map(([Key, v]) => ({ Key, Value: String(v).slice(0, 256) })) }); - else await ecs.tagResource({ resourceArn: task.taskArn, tags: tags.map(ecsTag) }); + // A job on a runner of this stack launched for another job: a swap, which + // the job label should make impossible (ADR-0004). It is logged, not + // repaired; if the runner's own job is stranded, the launcher's re-check + // (#51) launches it another runner. + function checkRunner(repo, job) { + const launchedFor = launchJobOf(job.runner_name); + if (launchedFor === null || launchedFor === job.id) return reply(200, 'ignored: no swap'); log(JSON.stringify({ - swap: name, ...(i ? { instance: i.InstanceId } : { task: task.taskArn }), - took: { repo: ran.repo, job_id: ran.job_id }, requeued: { repo: orig.repo, job_id: orig.job_id }, + anomaly: 'runner of another job', runner: job.runner_name, launched_for: launchedFor, + took: { repo: repo.full_name, job_id: job.id }, })); - return reply(202, 'swap repaired'); + return reply(200, 'logged: runner of another job'); } // A job that finished with success on a runner of this stack. The notifier @@ -180,11 +124,17 @@ function createHandler({ secret, queue, notifier, ec2, ecs, env, log = console.l log(JSON.stringify({ refused: repo.full_name, visibility: repo.visibility, job_id: job.id })); return reply(403, 'refused: only private repos of ' + env.GITHUB_ORG); } - if (p.action === 'in_progress') return repairSwap(p, repo, job); + if (p.action === 'in_progress') return checkRunner(repo, job); if (p.action === 'completed') return reportSuccess(p, repo, job); if (p.action !== 'queued') return reply(200, 'ignored: action'); const shape = shapeOf(job.labels); if (!shape) return reply(200, 'ignored: labels'); + // A job with no job label breaks the label contract, as one with no size + // does, but its own repo's workflow is waiting on it: say so in the log. + if (!job.labels.some(isJobLabel)) { + log(JSON.stringify({ ignored: repo.full_name, job_id: job.id, why: 'no job label' })); + return reply(200, 'ignored: labels'); + } const fault = jobLabelFault(job.labels, job); if (fault) { log(JSON.stringify({ ignored: repo.full_name, job_id: job.id, ...fault })); @@ -212,28 +162,15 @@ module.exports.handler = async (event) => { const sm = require('@aws-sdk/client-secrets-manager'); const sqs = require('@aws-sdk/client-sqs'); const LAMBDA = require('@aws-sdk/client-lambda'); - const EC2 = require('@aws-sdk/client-ec2'); const smc = new sm.SecretsManagerClient({}); const sqsc = new sqs.SQSClient({}); const lambdac = new LAMBDA.LambdaClient({}); - const ECS = require('@aws-sdk/client-ecs'); - const ec2c = new EC2.EC2Client({}); - const ecsc = new ECS.ECSClient({}); live = createHandler({ secret: { get: async () => JSON.parse((await smc.send(new sm.GetSecretValueCommand({ SecretId: process.env.SECRET_ARN }))).SecretString) }, queue: { send: async (m) => { await sqsc.send(new sqs.SendMessageCommand({ QueueUrl: process.env.QUEUE_URL, MessageBody: JSON.stringify(m) })); } }, notifier: { send: async (m) => { await lambdac.send(new LAMBDA.InvokeCommand({ FunctionName: process.env.NOTIFIER_FUNCTION, InvocationType: 'Event', Payload: JSON.stringify(m) })); }, }, - ec2: { - describeInstances: (i) => ec2c.send(new EC2.DescribeInstancesCommand(i)), - createTags: (i) => ec2c.send(new EC2.CreateTagsCommand(i)), - }, - ecs: { - listTasks: (i) => ecsc.send(new ECS.ListTasksCommand(i)), - describeTasks: (i) => ecsc.send(new ECS.DescribeTasksCommand(i)), - tagResource: (i) => ecsc.send(new ECS.TagResourceCommand(i)), - }, env: process.env, }); } diff --git a/template.yaml b/template.yaml index 710e2bf..968ac9c 100644 --- a/template.yaml +++ b/template.yaml @@ -160,7 +160,7 @@ Resources: MessageRetentionPeriod: 1209600 SqsManagedSseEnabled: true - # --- webhook: verify, label contract, fence 3, enqueue, swap repair --------- + # --- webhook: verify, label contract, fence 3, enqueue ---------------------- WebhookLogGroup: Type: AWS::Logs::LogGroup Properties: @@ -196,36 +196,6 @@ Resources: - Effect: Allow Action: lambda:InvokeFunction Resource: !GetAtt NotifierFunction.Arn - # Swap repair: find a runner's instance, and re-tag it with the - # job it really runs. Only the job tags, only the stack's instances. - - Effect: Allow - Action: ec2:DescribeInstances - Resource: "*" - - Effect: Allow - Action: ec2:CreateTags - Resource: !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:instance/*" - Condition: - StringEquals: - aws:ResourceTag/ci-runner:stack: !Ref AWS::StackName - ForAllValues:StringEquals: - aws:TagKeys: [ci-runner:repo, ci-runner:job-id, ci-runner:run-id, ci-runner:head-sha, ci-runner:job-name] - # The same for a runner's task: only the cluster's tasks, only - # the job tags. - - Effect: Allow - Action: ecs:ListTasks - Resource: "*" - Condition: - ArnEquals: - ecs:cluster: !GetAtt RunnerCluster.Arn - - Effect: Allow - Action: ecs:DescribeTasks - Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*" - - Effect: Allow - Action: ecs:TagResource - Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*" - Condition: - ForAllValues:StringEquals: - aws:TagKeys: [ci-runner:repo, ci-runner:job-id, ci-runner:run-id, ci-runner:head-sha, ci-runner:job-name] WebhookFunction: Type: AWS::Lambda::Function @@ -246,7 +216,6 @@ Resources: NOTIFIER_FUNCTION: !Ref NotifierFunction GITHUB_ORG: !Ref GitHubOrg STACK_NAME: !Ref AWS::StackName - CLUSTER: !Ref RunnerCluster Code: ZipFile: | //@inline functions/webhook.js diff --git a/test/conventions.test.js b/test/conventions.test.js index 22bccff..13bb961 100644 --- a/test/conventions.test.js +++ b/test/conventions.test.js @@ -84,29 +84,10 @@ test('the launcher, notifier and sweeper carry identical copies of the GitHub he } }); -test('the launcher and the webhook sanitize an ECS tag the same way', () => { - // The webhook re-tags a task the launcher tagged; bundling one rule would - // cost more template bytes than the two-line copy. - for (const re of [/^const ECS_TAG_CHARS = .*$/m, /^const ecsTag = .*$/m]) { - const [launcher, webhook] = ['launcher', 'webhook'].map((f) => (read(`functions/${f}.js`).match(re) || [])[0]); - assert.ok(launcher, `the launcher has no ${re}`); - assert.equal(webhook, launcher, `the webhook's ${re} differs from the launcher's`); - } -}); - -test('the launcher, webhook and notifier read tags the same way', () => { +test('the launcher and the notifier read tags the same way', () => { // Each strips the ci-runner: prefix, so t.capacity means the same everywhere. const re = /^const tagsOf = .*$/m; - const [launcher, ...rest] = ['launcher', 'webhook', 'notifier'].map((f) => ({ f, code: (read(`functions/${f}.js`).match(re) || [])[0] })); + const [launcher, ...rest] = ['launcher', 'notifier'].map((f) => ({ f, code: (read(`functions/${f}.js`).match(re) || [])[0] })); assert.ok(launcher.code, 'the launcher has no tagsOf'); for (const o of rest) assert.equal(o.code, launcher.code, `${o.f}'s tagsOf differs from the launcher's`); }); - -test('the launcher and the webhook carry the same DEFAULT_SHAPE', () => { - // Swap repair re-queues a runner launched before the shape tags with the - // shape the launcher gave it, so the two defaults must agree. - const re = /^const DEFAULT_SHAPE = .*$/m; - const [launcher, webhook] = ['launcher', 'webhook'].map((f) => (read(`functions/${f}.js`).match(re) || [])[0]); - assert.ok(launcher, 'the launcher has no DEFAULT_SHAPE'); - assert.equal(webhook, launcher, "the webhook's DEFAULT_SHAPE differs from the launcher's"); -}); diff --git a/test/notifier.test.js b/test/notifier.test.js index f7d9c63..8b7a5d4 100644 --- a/test/notifier.test.js +++ b/test/notifier.test.js @@ -139,7 +139,7 @@ test('a reclaim report tells how to unstick a lost job GitHub keeps in progress' }); // A machine is reported only for the job its CI runner is running. The tags -// say which job it was launched for, or took in a swap (the webhook re-tags it). +// say which job it was launched for, the only one its runner can take (ADR-0004). for (const [why, job] of [ ['an idle CI runner, whose launch job another CI runner took', { ...RUNNING, runner_name: 'ci-runner-20261112115400-777' }], ['a CI runner whose job is still queued', { ...RUNNING, status: 'queued', runner_name: null }], @@ -153,17 +153,6 @@ for (const [why, job] of [ }); } -test('a swapped CI runner is reported for the job it took, not the one it was launched for', async () => { - // The webhook re-tagged the instance with job 555; its runner is still named for job 999. - const runner = 'ci-runner-20261112115400-999'; - const inst = instance({ ...TAGS, 'ci-runner:runner': runner }); - const { handler, gh } = setup({ inst, gh: fakeGitHub({ jobs: [{ ...RUNNING, runner_name: runner }] }) }); - await handler(EVENT); - const { body } = gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs'); - assert.equal(body.external_id, '555'); - assert.equal(body.name, 'ci-runner / tests'); -}); - // AWS gives a 2-minute warning, and a job can finish inside it. The webhook // tells the notifier when a job succeeds. If that word came before the report // existed, the notifier's own re-read of the job, right after posting, clears it. @@ -357,8 +346,7 @@ for (const [why, tasks] of [ }); } -// A task's runner is named for its launch job too, and the webhook re-tags a -// swapped task the same way, so the same rules hold. +// A task's runner is named for its launch job too, so the same rules hold. for (const [why, job] of [ ['an idle Fargate CI runner', { ...RUNNING, runner_name: 'ci-runner-20261112115400-777' }], @@ -373,14 +361,6 @@ for (const [why, job] of [ }); } -test('a swapped Fargate CI runner is reported for the job it took', async () => { - const runner = 'ci-runner-20261112115400-999'; - const tags = { ...TASK_TAGS, 'ci-runner:runner': runner }; - const { handler, gh } = setup({ inst: null, tasks: [task(tags)], gh: fakeGitHub({ jobs: [{ ...RUNNING, runner_name: runner }] }) }); - await handler(TASK_EVENT); - assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.external_id, '555'); -}); - test('a Fargate job that has finished with success by the time the report is posted turns its check run neutral', async () => { const gh = fakeGitHub({ jobs: [TASK_JOB, { ...TASK_JOB, status: 'completed', conclusion: 'success' }] }); await setup({ inst: null, tasks: [task(TASK_TAGS)], gh }).handler(TASK_EVENT); diff --git a/test/tasks-ecs.test.js b/test/tasks-ecs.test.js index 11a7395..5728bd5 100644 --- a/test/tasks-ecs.test.js +++ b/test/tasks-ecs.test.js @@ -1,9 +1,8 @@ 'use strict'; // A task's lifecycle against real ECS shapes, from aws/aws-emulator -// (./aws-emulator.js): the launcher runs a task, the webhook repairs a swap -// on it (TagResource validates the characters, as ECS does), the sweeper -// stops it, and the notifier reports a reclaim from the ECS Task State Change -// the emulator publishes to EventBridge, matched by template.yaml's own rule. +// (./aws-emulator.js): the launcher runs a task, the sweeper stops it, and +// the notifier reports a reclaim from the ECS Task State Change the emulator +// publishes to EventBridge, matched by template.yaml's own rule. // The emulator drops a task's tags once it is told to stop (AWS keeps them // readable through DescribeTasks, measured live); the tests rely on that only // where they say so. @@ -14,7 +13,6 @@ const fs = require('node:fs'); const path = require('node:path'); const { skip, startEmulator, network, taskDefinition } = require('./aws-emulator.js'); const launcher = require('../functions/launcher.js'); -const webhook = require('../functions/webhook.js'); const sweeper = require('../functions/sweeper.js'); const notifier = require('../functions/notifier.js'); @@ -39,7 +37,6 @@ const ecs = { return r; }, stopTask: op('StopTask'), - tagResource: op('TagResource'), }; // template.yaml's NotifierTaskRule pattern, with the cluster's ARN for its @@ -89,7 +86,7 @@ const secret = { get: async () => ({ app_id: 42, private_key: privateKey.export( // GitHub, faked: every job is queued and keeps its real name, the group // exists, JIT configs mint, and runners and check runs are recorded (and a // commit's check runs listed). -const JOB_NAMES = { 999: 'build (x64, a)', 555: 'test (x64, b)' }; +const JOB_NAMES = { 999: 'build (x64, a)' }; function fakeGitHub(runners = [], job = {}) { const calls = []; const fetch = async (url, init = {}) => { @@ -129,43 +126,6 @@ async function launch() { return { arn: l.launched, runner: l.runner }; } -const tagsOf = async (arn) => Object.fromEntries((await ecs.describeTasks({ cluster: 'ci-runner', tasks: [arn], include: ['TAGS'] })) - .tasks[0].tags.map((t) => [t.key, t.value])); - -test('a swap on a task: the job is re-queued from the task\'s tags, and ECS takes the re-tag', { skip }, async () => { - const { arn, runner } = await launch(); - const sent = []; - const logs = []; - const h = webhook.createHandler({ - secret, queue: { send: async (m) => sent.push(m) }, - ec2: { describeInstances: async () => ({ Reservations: [] }), createTags: assert.fail }, - ecs, env, log: (l) => logs.push(l), - }); - const raw = JSON.stringify({ - action: 'in_progress', - workflow_job: { id: 555, run_id: 444, head_sha: 'abc123', name: JOB_NAMES[555], labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-4', 'x64'], runner_name: runner }, - repository: { full_name: 'zeroae/app', visibility: 'private', owner: { login: 'zeroae' } }, - installation: { id: 181 }, - }); - const sig = 'sha256=' + crypto.createHmac('sha256', 'whsec').update(raw).digest('hex'); - const res = await h({ headers: { 'x-github-event': 'workflow_job', 'x-hub-signature-256': sig }, body: raw }); - assert.equal(res.statusCode, 202, logs.join('\n')); - - assert.equal(sent.length, 1); - assert.deepEqual( - { ...sent[0], job_name: undefined }, - { repo: 'zeroae/lib', job_id: 999, run_id: 888, head_sha: 'fff000', job_name: undefined, label: 'ci-fargate-spot', - vcpus: 4, arch: 'x64', labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-4', 'x64', 'linux'], installation_id: 181 }, - ); - const tags = await tagsOf(arn); - assert.deepEqual( - ['repo', 'job-id', 'run-id', 'head-sha', 'job-name'].map((k) => tags[`ci-runner:${k}`]), - ['zeroae/app', '555', '444', 'abc123', 'test _x64_ b_'], - ); - assert.equal(tags['ci-runner:runner'], runner, 'the other tags stay'); - await ecs.stopTask({ cluster: 'ci-runner', task: arn }); -}); - test('the sweeper keeps a live task\'s registration, stops the task after 2 hours, then removes the registration', { skip }, async () => { const { arn, runner } = await launch(); const later = Date.now() + 3 * 3600e3; diff --git a/test/template.test.js b/test/template.test.js index 1f894ad..66b92c7 100644 --- a/test/template.test.js +++ b/test/template.test.js @@ -183,15 +183,9 @@ test('the notifier may describe only the cluster\'s tasks, and nothing else on E ]); }); -test('the webhook may find and re-tag only the cluster\'s tasks, and only with the job tags', () => { - const ecs = statements(resource('WebhookRole')).filter((s) => /ecs:/.test(s)); - const tasks = 'Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*"'; - const at = (n) => '\n' + ' '.repeat(n); - assert.deepEqual(ecs, [ - `Allow${at(16)}Action: ecs:ListTasks${at(16)}Resource: "*"${at(16)}Condition:${at(18)}ArnEquals:${at(20)}ecs:cluster: !GetAtt RunnerCluster.Arn`, - `Allow${at(16)}Action: ecs:DescribeTasks${at(16)}${tasks}`, - `Allow${at(16)}Action: ecs:TagResource${at(16)}${tasks}${at(16)}Condition:${at(18)}ForAllValues:StringEquals:${at(20)}` - + 'aws:TagKeys: [ci-runner:repo, ci-runner:job-id, ci-runner:run-id, ci-runner:head-sha, ci-runner:job-name]\n', - ]); - assert.match(resource('WebhookFunction'), /CLUSTER: !Ref RunnerCluster\n/); +test('the webhook touches no instance and no task, and knows no cluster', () => { + // The job label ended swaps, and with them the swap repair, the only user + // of these (ADR-0004). + assert.deepEqual(statements(resource('WebhookRole')).filter((s) => /ec2:|ecs:/.test(s)), []); + assert.doesNotMatch(resource('WebhookFunction'), /CLUSTER:/); }); diff --git a/test/webhook.test.js b/test/webhook.test.js index 92c202b..11b245b 100644 --- a/test/webhook.test.js +++ b/test/webhook.test.js @@ -6,62 +6,33 @@ const { createHandler } = require('../functions/webhook.js'); const SECRET = 'whsec-test'; -// instances: what DescribeInstances knows, as { InstanceId, Tags: { key: value } }. -// The fake applies the tag filters the way EC2 does. -// tasks: the cluster's tasks, as { taskArn, desiredStatus, tags: { key: value } }. -function setup({ secretValue = { webhook_secret: SECRET }, instances = [], tasks = [] } = {}) { +function setup({ secretValue = { webhook_secret: SECRET } } = {}) { const sent = []; const notified = []; - const described = []; - const tagged = []; - const taskTagged = []; const logs = []; let reads = 0; const handler = createHandler({ secret: { get: async () => { reads++; return secretValue; } }, queue: { send: async (msg) => { sent.push(msg); } }, notifier: { send: async (msg) => { notified.push(msg); } }, - ec2: { - describeInstances: async (input) => { - described.push(input); - const match = (i) => input.Filters.every((f) => !f.Name.startsWith('tag:') || f.Values.includes(i.Tags[f.Name.slice(4)])); - const Instances = instances.filter(match).map((i) => ({ - InstanceId: i.InstanceId, State: { Name: 'running' }, - Tags: Object.entries(i.Tags).map(([Key, Value]) => ({ Key, Value })), - })); - return { Reservations: Instances.length ? [{ Instances }] : [] }; - }, - createTags: async (input) => { tagged.push(input); }, - }, - ecs: { - listTasks: async (input) => { - described.push(input); - assert.equal(input.cluster, 'ci-runner'); - return { taskArns: tasks.filter((t) => t.desiredStatus === (input.desiredStatus || 'RUNNING')).map((t) => t.taskArn) }; - }, - describeTasks: async (input) => { - assert.deepEqual([input.cluster, input.include], ['ci-runner', ['TAGS']]); - return { - tasks: tasks.filter((t) => input.tasks.includes(t.taskArn)).map((t) => ({ - taskArn: t.taskArn, desiredStatus: t.desiredStatus, lastStatus: 'RUNNING', - tags: Object.entries(t.tags).map(([key, value]) => ({ key, value })), - })), - }; - }, - tagResource: async (input) => { taskTagged.push(input); return {}; }, - }, - env: { GITHUB_ORG: 'zeroae', STACK_NAME: 'ci-runner', CLUSTER: 'ci-runner' }, + env: { GITHUB_ORG: 'zeroae', STACK_NAME: 'ci-runner' }, log: (line) => logs.push(JSON.parse(line)), }); - return { handler, sent, notified, described, tagged, taskTagged, logs, reads: () => reads }; + return { handler, sent, notified, logs, reads: () => reads }; } +// The job label of the payload's job: run 444, attempt 2, key tests, index 0. +const JOB_LABEL = 'ci-job-444-2-tests-0'; + // A workflow_job payload as GitHub sends it (only the fields that matter). -function job({ action = 'queued', labels = ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64'], visibility = 'private', owner = 'zeroae', runner, conclusion } = {}) { +// The job carries JOB_LABEL after `labels`, unless `labels` has a job label of +// its own or jobLabel is false. +function job({ action = 'queued', labels = ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64'], jobLabel = true, visibility = 'private', owner = 'zeroae', runner, conclusion } = {}) { + const all = jobLabel && !labels.some((l) => /^ci-job-/i.test(l)) ? [...labels, JOB_LABEL] : labels; return { action, workflow_job: { - id: 555, run_id: 444, run_attempt: 2, head_sha: 'abc123', name: 'tests', labels, + id: 555, run_id: 444, run_attempt: 2, head_sha: 'abc123', name: 'tests', labels: all, ...(runner && { runner_name: runner, runner_id: 77 }), ...(conclusion && { conclusion }), }, @@ -87,7 +58,7 @@ test('an accepted ci-spot job is enqueued with every field', async () => { assert.equal(res.statusCode, 202); assert.deepEqual(sent, [{ repo: 'zeroae/app', job_id: 555, run_id: 444, head_sha: 'abc123', job_name: 'tests', - label: 'ci-spot', vcpus: 4, arch: 'x64', labels: ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64'], installation_id: 181, + label: 'ci-spot', vcpus: 4, arch: 'x64', labels: ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64', JOB_LABEL], installation_id: 181, }]); }); @@ -99,7 +70,7 @@ test('every size label is enqueued with its vCPUs and the architecture', async ( assert.equal((await handler(delivery(job({ labels })))).statusCode, 202, labels.join(',')); assert.deepEqual( { label: sent[0].label, vcpus: sent[0].vcpus, arch: sent[0].arch, labels: sent[0].labels }, - { label: cap, vcpus, arch: 'x64', labels }, + { label: cap, vcpus, arch: 'x64', labels: [...labels, JOB_LABEL] }, ); } } @@ -242,7 +213,7 @@ test('other actions and other events are ignored', async () => { } }); -// A runner the launcher started for job 999 of run 888, and its instance. +// A runner the launcher started for job 999 of run 888. const RUNNER = 'ci-runner-20261113120000-999'; // The notifier is told when a job of this stack's runners finishes with @@ -268,80 +239,35 @@ for (const [why, opts] of [ assert.deepEqual(notified, []); }); } -const instance = (tags = {}) => ({ - InstanceId: 'i-0abc', - Tags: { - 'ci-runner:stack': 'ci-runner', 'ci-runner:runner': RUNNER, 'ci-runner:runner-id': '77', - 'ci-runner:label': 'ci-spot', 'ci-runner:labels': 'self-hosted,ci-spot,ci-cpu-8,arm64,linux', - 'ci-runner:capacity': 'ci-spot', 'ci-runner:vcpus': '8', 'ci-runner:arch': 'arm64', - 'ci-runner:repo': 'zeroae/lib', 'ci-runner:job-id': '999', 'ci-runner:run-id': '888', - 'ci-runner:head-sha': 'fff000', 'ci-runner:job-name': 'build', - ...tags, - }, -}); - -test('a swap: the runner\'s own job is re-queued from its tags, and the instance is re-tagged with the job it runs', async () => { - const { handler, sent, tagged, logs } = setup({ instances: [instance()] }); +// A runner of this stack is named `${stack}-<14-digit stamp>-`, for +// the job it was launched for. The job label means it takes no other job, so +// one that does is logged and left alone (ADR-0004). +test('a job in progress on a runner launched for another job is logged, and nothing is re-queued', async () => { + const { handler, sent, logs } = setup(); const res = await handler(delivery(job({ action: 'in_progress', runner: RUNNER }))); - assert.equal(res.statusCode, 202); - assert.deepEqual(sent, [{ - repo: 'zeroae/lib', job_id: 999, run_id: 888, head_sha: 'fff000', job_name: 'build', - label: 'ci-spot', vcpus: 8, arch: 'arm64', labels: ['self-hosted', 'ci-spot', 'ci-cpu-8', 'arm64', 'linux'], installation_id: 181, - }]); - assert.deepEqual(tagged, [{ - Resources: ['i-0abc'], - Tags: [ - { Key: 'ci-runner:repo', Value: 'zeroae/app' }, { Key: 'ci-runner:job-id', Value: '555' }, - { Key: 'ci-runner:run-id', Value: '444' }, { Key: 'ci-runner:head-sha', Value: 'abc123' }, - { Key: 'ci-runner:job-name', Value: 'tests' }, - ], - }]); - assert.deepEqual(logs, [{ - swap: RUNNER, instance: 'i-0abc', took: { repo: 'zeroae/app', job_id: 555 }, requeued: { repo: 'zeroae/lib', job_id: 999 }, - }]); + assert.deepEqual([res.statusCode, res.body], [200, 'logged: runner of another job\n']); + assert.deepEqual(sent, []); + assert.deepEqual(logs, [{ anomaly: 'runner of another job', runner: RUNNER, launched_for: 999, took: { repo: 'zeroae/app', job_id: 555 } }]); }); test('in_progress on a runner running its own job does nothing', async () => { - const own = 'ci-runner-20261113120000-555'; - const { handler, sent, described, tagged, logs } = setup({ instances: [instance({ 'ci-runner:runner': own, 'ci-runner:job-id': '555' })] }); - assert.equal((await handler(delivery(job({ action: 'in_progress', runner: own })))).statusCode, 200); - assert.deepEqual([sent, described, tagged, logs], [[], [], [], []]); + const { handler, sent, logs } = setup(); + const res = await handler(delivery(job({ action: 'in_progress', runner: 'ci-runner-20261113120000-555' }))); + assert.deepEqual([res.statusCode, res.body, sent, logs], [200, 'ignored: no swap\n', [], []]); }); test('in_progress on a runner that isn\'t the stack\'s is ignored', async () => { - const others = [ - 'GitHub Actions 2', 'other-stack-20261113120000-999', 'ci-runner-hand-20261113120000', - 'ci-runner-2026111312-999', undefined, - ]; - for (const runner of others) { - const { handler, sent, tagged } = setup({ instances: [instance({ 'ci-runner:runner': runner })] }); - assert.equal((await handler(delivery(job({ action: 'in_progress', runner })))).statusCode, 200, String(runner)); - assert.deepEqual([sent, tagged], [[], []], String(runner)); + for (const runner of ['GitHub Actions 2', 'other-stack-20261113120000-999', 'ci-runner-hand-20261113120000', 'ci-runner-2026111312-999', undefined]) { + const { handler, sent, logs } = setup(); + const res = await handler(delivery(job({ action: 'in_progress', runner }))); + assert.deepEqual([res.statusCode, res.body, sent, logs], [200, 'ignored: no swap\n', [], []], String(runner)); } - // The stack's name, but no instance of the stack carries it. - const { handler, sent, tagged } = setup({ instances: [instance({ 'ci-runner:stack': 'other' })] }); - assert.equal((await handler(delivery(job({ action: 'in_progress', runner: RUNNER })))).statusCode, 200); - assert.deepEqual([sent, tagged], [[], []]); }); -test('a repeated in_progress after the re-tag does nothing more', async () => { - const { handler, sent, tagged } = setup({ instances: [instance({ 'ci-runner:job-id': '555' })] }); - assert.equal((await handler(delivery(job({ action: 'in_progress', runner: RUNNER })))).statusCode, 200); - assert.deepEqual([sent, tagged], [[], []]); -}); - -test('a swap on an instance launched before the labels and shape tags re-queues a ci-cpu-4 x64 job with self-hosted and its capacity label', async () => { - const old = instance(); - for (const k of ['labels', 'capacity', 'vcpus', 'arch']) delete old.Tags[`ci-runner:${k}`]; - const { handler, sent } = setup({ instances: [old] }); - await handler(delivery(job({ action: 'in_progress', runner: RUNNER }))); - assert.deepEqual([sent[0].labels, sent[0].vcpus, sent[0].arch], [['self-hosted', 'ci-spot'], 4, 'x64']); -}); - -test('in_progress from outside the fence is refused before any lookup', async () => { - const { handler, sent, described } = setup({ instances: [instance()] }); +test('in_progress from outside the fence is refused', async () => { + const { handler, sent } = setup(); assert.equal((await handler(delivery(job({ action: 'in_progress', runner: RUNNER, visibility: 'public' })))).statusCode, 403); - assert.deepEqual([sent, described], [[], []]); + assert.deepEqual(sent, []); }); // The Fargate capacities (#29): Docker-free jobs, ADR-0001. @@ -354,7 +280,7 @@ test('both Fargate capacities are enqueued with every size and architecture', as assert.equal((await handler(delivery(job({ labels })))).statusCode, 202, labels.join(',')); assert.deepEqual( { label: sent[0].label, vcpus: sent[0].vcpus, arch: sent[0].arch, labels: sent[0].labels }, - { label: cap, vcpus, arch, labels }, + { label: cap, vcpus, arch, labels: [...labels, JOB_LABEL] }, ); } } @@ -381,66 +307,6 @@ test('a Fargate job follows the same contract: no default size or architecture, } }); -// A runner the launcher started on Fargate Spot for job 999, and its task. ECS -// refuses commas and brackets in a tag: the labels are space-joined and the -// job name's brackets and commas are `_`. -const TASK_ARN = 'arn:aws:ecs:us-east-1:123456789012:task/ci-runner/0f1e2d3c4b5a69788796a5b4c3d2e1f0'; -const fargateTask = (tags = {}, desiredStatus = 'RUNNING') => ({ - taskArn: TASK_ARN, desiredStatus, - tags: { - ...instance().Tags, 'ci-runner:label': 'ci-fargate-spot', 'ci-runner:capacity': 'ci-fargate-spot', - 'ci-runner:labels': 'self-hosted ci-fargate-spot ci-cpu-8 arm64 linux', 'ci-runner:job-name': 'build _x64_ a_', - ...tags, - }, -}); -const fargateJob = () => { - const p = job({ action: 'in_progress', runner: RUNNER, labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-8', 'arm64', 'linux'] }); - p.workflow_job.name = 'test (arm64, b)'; - return p; -}; - -test('a swap on a task: the runner\'s own job is re-queued from the task\'s tags, and the task is re-tagged with the job it runs', async () => { - const { handler, sent, tagged, taskTagged, logs } = setup({ tasks: [fargateTask()] }); - const res = await handler(delivery(fargateJob())); - assert.equal(res.statusCode, 202); - assert.deepEqual(sent, [{ - repo: 'zeroae/lib', job_id: 999, run_id: 888, head_sha: 'fff000', job_name: 'build _x64_ a_', - label: 'ci-fargate-spot', vcpus: 8, arch: 'arm64', labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-8', 'arm64', 'linux'], installation_id: 181, - }]); - assert.deepEqual(tagged, []); - assert.deepEqual(taskTagged, [{ - resourceArn: TASK_ARN, - tags: [ - { key: 'ci-runner:repo', value: 'zeroae/app' }, { key: 'ci-runner:job-id', value: '555' }, - { key: 'ci-runner:run-id', value: '444' }, { key: 'ci-runner:head-sha', value: 'abc123' }, - { key: 'ci-runner:job-name', value: 'test _arm64_ b_' }, - ], - }]); - assert.deepEqual(logs, [{ - swap: RUNNER, task: TASK_ARN, took: { repo: 'zeroae/app', job_id: 555 }, requeued: { repo: 'zeroae/lib', job_id: 999 }, - }]); -}); - -test('a repeated in_progress after a task\'s re-tag does nothing more', async () => { - const { handler, sent, taskTagged } = setup({ tasks: [fargateTask({ 'ci-runner:job-id': '555' })] }); - assert.equal((await handler(delivery(fargateJob()))).statusCode, 200); - assert.deepEqual([sent, taskTagged], [[], []]); -}); - -test('a task told to stop, or another runner\'s task, is no swap to repair', async () => { - for (const t of [fargateTask({}, 'STOPPED'), fargateTask({ 'ci-runner:runner': 'ci-runner-20261113120000-998' })]) { - const { handler, sent, taskTagged } = setup({ tasks: [t] }); - assert.equal((await handler(delivery(fargateJob()))).statusCode, 200); - assert.deepEqual([sent, taskTagged], [[], []]); - } -}); - -test('a swap on an instance looks up no task', async () => { - const { handler, described } = setup({ instances: [instance()], tasks: [fargateTask()] }); - assert.equal((await handler(delivery(job({ action: 'in_progress', runner: RUNNER })))).statusCode, 202); - assert.ok(described.every((d) => d.Filters), 'only DescribeInstances'); -}); - // The job label: ci-job----, here // for run 444, attempt 2 (the payload's). const SHAPE = ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64']; @@ -502,3 +368,10 @@ test('a job outside the label contract is ignored without a log line, whatever i const res = await handler(delivery(job({ labels: ['ubuntu-latest', 'ci-job-1-1-tests-0'] }))); assert.deepEqual([res.statusCode, res.body, sent.length, logs.length], [200, 'ignored: labels\n', 0, 0]); }); + +test('a queued job without a job label is ignored, and the log says why', async () => { + const { handler, sent, logs } = setup(); + const res = await handler(delivery(job({ labels: SHAPE, jobLabel: false }))); + assert.deepEqual([res.statusCode, res.body, sent.length], [200, 'ignored: labels\n', 0]); + assert.deepEqual(logs, [{ ignored: 'zeroae/app', job_id: 555, why: 'no job label' }]); +}); From dd9091066499199c69fdbe9ed2f517e3311b8e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Fri, 20 Nov 2026 21:24:56 -0500 Subject: [PATCH 2/6] Docs: the job label is required, and the swap repair is gone README's adoption examples carry the job label; the label contract, stack.md, traps, the runbook's log lines, the glossary's Swap entry and ADR 0004's rollout say a job without one gets no runner and a swap is logged, not repaired. Co-Authored-By: Claude Opus 5.5 --- GLOSSARY.md | 4 +- README.md | 28 +++--- ...a-job-label-pins-each-runner-to-its-job.md | 6 +- docs/runbook.md | 10 +- docs/stack.md | 93 +++++++------------ docs/traps.md | 4 +- 6 files changed, 64 insertions(+), 81 deletions(-) diff --git a/GLOSSARY.md b/GLOSSARY.md index bc87760..c41f8f8 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -26,14 +26,14 @@ The `runs-on` label that picks a CI runner's CPU architecture: `x64` or `arm64`. A CI runner's capacity, size and architecture together. Runners of the same shape are interchangeable. **Label contract**: -The labels a job must carry to be served: `self-hosted`, exactly one capacity label, exactly one size label and exactly one architecture label, plus `linux` if the job wants it, and a job label. Nothing else is allowed. It holds on every capacity. The job label is optional until the consumers carry it. +The labels a job must carry to be served: `self-hosted`, exactly one capacity label, exactly one size label and exactly one architecture label, plus `linux` if the job wants it, and a job label. Nothing else is allowed. It holds on every capacity. **Job label**: The `runs-on` label unique to one job: `ci-job----`. The CI runner launched for the job registers with it, so it can take no job with another job label. The job key is written by hand, since `github.job` is empty inside `runs-on`. _Avoid_: run label (that names a run, shared by its jobs) **Swap**: -A CI runner taking a queued job other than the one it was launched for. GitHub gives a job to any idle runner whose labels cover the job's, so swaps between runners of the same shape happen, but only of a job without a job label. +A CI runner taking a queued job other than the one it was launched for. Swaps can no longer happen: GitHub gives a job only to a runner whose labels include all of the job's, every job must carry its own job label, and each runner carries only its own job's. The webhook logs one as an anomaly if it ever sees it, and repairs nothing. **Fargate capacity**: Fargate Spot or Fargate On-Demand, the two capacities with no Docker daemon. A job that picks one promises not to use Docker. diff --git a/README.md b/README.md index 0f115a6..21b528c 100644 --- a/README.md +++ b/README.md @@ -62,18 +62,24 @@ it's in both. ```yaml jobs: tests: - runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-spot","ci-cpu-8","x64"]') || 'ubuntu-latest' }} + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON(format('["self-hosted","ci-spot","ci-cpu-8","x64","ci-job-{0}-{1}-tests-{2}"]', github.run_id, github.run_attempt, strategy.job-index)) || 'ubuntu-latest' }} build-arm: - runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-spot","ci-cpu-4","arm64"]') || 'ubuntu-latest' }} + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON(format('["self-hosted","ci-spot","ci-cpu-4","arm64","ci-job-{0}-{1}-build-arm-{2}"]', github.run_id, github.run_attempt, strategy.job-index)) || 'ubuntu-latest' }} apply: - runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-ondemand","ci-cpu-2","x64"]') || 'ubuntu-latest' }} + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON(format('["self-hosted","ci-ondemand","ci-cpu-2","x64","ci-job-{0}-{1}-apply-{2}"]', github.run_id, github.run_attempt, strategy.job-index)) || 'ubuntu-latest' }} lint: - runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-fargate-spot","ci-cpu-2","arm64"]') || 'ubuntu-latest' }} + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON(format('["self-hosted","ci-fargate-spot","ci-cpu-2","arm64","ci-job-{0}-{1}-lint-{2}"]', github.run_id, github.run_attempt, strategy.job-index)) || 'ubuntu-latest' }} docs: - runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-fargate","ci-cpu-4","x64"]') || 'ubuntu-latest' }} + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON(format('["self-hosted","ci-fargate","ci-cpu-4","x64","ci-job-{0}-{1}-docs-{2}"]', github.run_id, github.run_attempt, strategy.job-index)) || 'ubuntu-latest' }} ``` - Each `runs-on` names the full shape: capacity, size and architecture. + Each `runs-on` names the full shape: capacity, size and architecture, and + the job label, `ci-job----`, with + the job key typed by hand (`github.job` is empty inside `runs-on`). The + label pins the runner launched for the job to that job + ([ADR-0004](docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md)); + a reusable workflow needs an input in it too (see "The job label" in + [docs/stack.md](docs/stack.md)). A job on `ci-fargate-spot` or `ci-fargate` promises not to use Docker (no `services:`, `container:`, Docker actions or `docker` commands): a Fargate task has no Docker daemon @@ -95,11 +101,11 @@ it's in both. **The label contract:** `runs-on` must include `self-hosted`, exactly one of `ci-spot`, `ci-ondemand`, `ci-fargate-spot` or `ci-fargate`, exactly one size label (`ci-cpu-2`, `ci-cpu-4`, -`ci-cpu-8`, `ci-cpu-16`) and exactly one of `x64` or `arm64`. It may also -include `linux`. None of them has a default. A job missing a size or an -architecture label, or with any other label (`aarch64`, an unknown -`ci-cpu-*`, two size labels, both `x64` and `arm64`), is ignored, and it waits -for a runner that never comes. Before changing the contract, `make consumers` +`ci-cpu-8`, `ci-cpu-16`), exactly one of `x64` or `arm64`, and exactly one +job label of its own run and attempt. It may also include `linux`. None of +them has a default. A job missing a size, an architecture or a job label, or +with any other label (`aarch64`, an unknown `ci-cpu-*`, two size labels, both +`x64` and `arm64`), is ignored, and it waits for a runner that never comes. Before changing the contract, `make consumers` lists every job it affects. **What the job gets:** Ubuntu 24.04, the runner as user `runner` (passwordless diff --git a/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md b/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md index fc8a611..4379e61 100644 --- a/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md +++ b/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md @@ -4,17 +4,17 @@ Supersedes [ADR 0003](0003-tolerate-swaps-instead-of-run-labels.md). GitHub gives a queued job to any idle runner whose labels cover the job's. ADR 0003 let that happen, so a CI runner often ran a job other than the one it was launched for (a swap), and the system repaired the bookkeeping afterwards. Swaps turned out to be routine (the webhook logged three in five seconds on 2026-11-20), and every piece built on top had to allow for them: the swap repair, and #51's re-check, which finds a job's machines by the job-id tag because the runner's name can't be trusted. We now prevent swaps instead. -Every job carries one label unique to it, the **job label**: +Every job must carry one label unique to it, the **job label**: ``` ci-job---- ``` -The launcher registers each runner with its job's labels, so the runner launched for a job carries that job's label, and GitHub can give it no job with another job label. GitHub gives a job to a runner whose labels include all of the job's, so the runner can still take a job of its shape that has no job label: swaps end once every job carries one. The webhook accepts a job label only when it names the job's own run and attempt (`workflow_job.run_id` and `run_attempt`) and has something after them. A job with two, or with one for another run or attempt, is ignored. +The launcher registers each runner with its job's labels, so the runner launched for a job carries that job's label, and GitHub can give it no job with another job label. GitHub gives a job to a runner whose labels include all of the job's, so a runner could still take a job of its shape that has no job label: that is why every job must carry one. The webhook accepts a job label only when it names the job's own run and attempt (`workflow_job.run_id` and `run_attempt`) and has something after them. A job with two, or with one for another run or attempt, is ignored. The rest of the label is the consumer's to make unique within the run. GitHub can't do it for them: inside `runs-on`, `github.job` evaluates to an empty string (measured on 2026-11-20 in a scratch repo, by reading the labels of the queued jobs back from the API), so the job key is written by hand. `strategy.job-index` does work there, and is `0` in a job with no matrix. A reusable workflow sees only its own job keys and its own matrix, so two calls in one run would make the same label: the caller passes something unique into it, as an input. -Rollout is in two steps. First the label is accepted and checked but not required: a job without it is served as before, and swaps are still repaired, including a labelled job's runner taking a job without the label. Once the consumers carry it, the label is required and the swap repair is removed. +Rollout was in two steps. First the label was accepted and checked but not required: a job without it was served as before, and swaps were still repaired, including a labelled job's runner taking a job without the label. Once the consumers carried it, the label became required: a job of our shape without one is ignored, like any other job outside the label contract, and the webhook logs why. The swap repair is removed, and with it the webhook's EC2 and ECS permissions. A runner that runs a job other than its own, which should now be impossible, is logged as an anomaly and not repaired. #51's re-check stays: it covers runners that never take their job for other reasons (a concurrency group, a failed boot). ## Considered Options diff --git a/docs/runbook.md b/docs/runbook.md index f4d5517..8c3cd3c 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -352,10 +352,12 @@ $AWS logs tail /aws/lambda/ci-runner-notifier --since 1d **The webhook** logs one JSON line per accepted job, `{"enqueued": "", "job_id": ..., "label": ...}`, and `{"refused": "", "visibility": ...}` -for a job from any repo but a private `zeroae` one (fence 3), and `{"swap": -"", "instance": "i-...", "took": {...}, "requeued": {...}}` (with -`"task": ""` for a Fargate runner) for each swap it repairs: the runner launched for the `requeued` job started the `took` -job instead. Everything else it ignores without a log line. +for a job from any repo but a private `zeroae` one (fence 3), `{"ignored": +"", "job_id": ..., "why": ...}` for a job of our shape with no job label, +or a bad one (see "The job label" in `stack.md`), and `{"anomaly": "runner of +another job", "runner": ..., "launched_for": , "took": {...}}` if a +runner ever starts a job other than its own, which the job label should make +impossible (`docs/traps.md`). Everything else it ignores without a log line. **The launcher** logs one JSON line per job, carrying the job's fields plus one of: diff --git a/docs/stack.md b/docs/stack.md index 597ba5f..1c7ca8f 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -208,7 +208,8 @@ App's webhook points at), for each delivery: 2. Ignores (`200`) any event but `workflow_job`. 3. **Fence 3:** refuses (`403`, logged) unless `repository.visibility` is `private` and the owner is `GITHUB_ORG`. -4. On `in_progress`, repairs a swap (below) and answers. +4. On `in_progress`, logs a runner running another job than its own (below) + and answers. 5. On `completed`, tells the notifier when the job succeeded on a runner of this stack (below) and answers. 6. Ignores (`200`) actions other than `queued`, and jobs outside the label @@ -216,8 +217,10 @@ App's webhook points at), for each delivery: `ci-spot`/`ci-ondemand`/`ci-fargate-spot`/`ci-fargate`, exactly one size label (`ci-cpu-2`, `-4`, `-8`, `-16`), exactly one of `x64`/`arm64`, and optionally `linux`. None has a default: a missing size or architecture label, an unknown `ci-cpu-*`, or - both `x64` and `arm64`, are outside it. A job label (below) is allowed on - top, and checked once the rest passes: a job with two, or one that leaves + both `x64` and `arm64`, are outside it. A job label (below) is required on + top, and checked once the rest passes: a job with none is ignored (`200`, + `ignored: labels`) and logged as `{"ignored": , "job_id", "why": "no + job label"}`. A job with two, or one that leaves out its run and attempt, names another run or attempt, has nothing after them, or is over 255 characters, is ignored (`200`, `ignored: job label`) and logged as `{"ignored": , "job_id", "label", "why"}`, `why` being @@ -228,16 +231,15 @@ App's webhook points at), for each delivery: labels, installation_id}` (`label` is the capacity label, lower case; `vcpus` is 2, 4, 8 or 16; `arch` is `x64` or `arm64`) and answers `202`. -**The job label** (ADR-0004). A job may carry one label unique to it, +**The job label** (ADR-0004). A job must carry one label unique to it, `ci-job----`, compared case-insensitively. The webhook accepts it only when it starts with `ci-job---` of the event's `workflow_job` and has something after that. It goes into the message's `labels` with the rest, the launcher mints the runner's JIT config with those labels, and so the runner launched for the job can take no job with another job label. GitHub gives a -job to a runner whose labels include all of the job's, so that runner can -still take a job of its shape that has no job label. It is optional for now: -a job without it is served as before, and swaps are still repaired (below). +job to a runner whose labels include all of the job's, so a job without one +could take any runner of its shape: that is why the label is required. A workflow writes it in `runs-on`, with the job key typed by hand: inside `runs-on`, `github.job` evaluates to an empty string. `strategy.job-index` is @@ -297,47 +299,22 @@ ignored, `200`), the webhook invokes the notifier asynchronously (about 3 deliveries per job: `queued`, `in_progress`, `completed`), so nothing about the App changes. The webhook does no GitHub call itself. -**Swap repair** (ADR-0003, superseded by ADR-0004 once the job label is -required). GitHub gives a queued job to any idle runner whose labels cover it, -so a runner launched for job Y may take job X, unless X carries a job label. Left alone, -X's own message is later dropped as not queued, and Y never gets a runner. -On `workflow_job` `in_progress` the webhook reads `runner_name`: - -- a name that isn't `${stack}-<14-digit stamp>-` (a hand launch, a - GitHub-hosted runner, another stack), or whose job id is the event's, is - ignored (`200`) with no AWS call; -- otherwise it describes the stack's `pending`/`running` instance tagged - `ci-runner:runner=`, and if there is none, the cluster's task tagged - so among those whose desired status is `RUNNING` (`ListTasks`, then - `DescribeTasks` with their tags). No instance or task, or a - `ci-runner:job-id` tag already equal to the event's job (a repeated - delivery), is ignored too; -- otherwise it **re-queues** the machine's job, rebuilt from its tags - (`repo`, `job-id`, `run-id`, `head-sha`, `job-name`, `label`, `vcpus`, - `arch`, `labels`; the event's installation id), so the launcher's "still - queued?" check launches it a runner or drops it. A task's `labels` are - split on spaces, an instance's on commas. A task's `job-name` is the - sanitized one (see "The Fargate capacities"); only tags read it, and the - notifier fetches the real name. An instance launched before those tags - existed re-queues as `ci-cpu-4`/`x64` with `self-hosted` and its capacity - label; -- then **re-tags** the instance (`CreateTags`) or task (`TagResource`) with - the event's job (`ci-runner:repo`, `job-id`, `run-id`, `head-sha`, - `job-name`), so the notifier reports a reclaim on the commit of the job it - really ran. A task's values are sanitized as the launcher's are. It logs - `{"swap": , "instance" or "task", "took": {repo, job_id}, - "requeued": {repo, job_id}}`, answering `202`. - -It queues before it tags: if the tag fails, a redelivery still sees the old -tags and repairs again. Re-queueing a job that another runner already took is -harmless: the launcher drops it. - -Its role allows `GetSecretValue` on the secret, `SendMessage` on the queue, -`lambda:InvokeFunction` on the notifier, `ec2:DescribeInstances`, and `ec2:CreateTags` only on instances tagged -`ci-runner:stack=` and only for the five job tag keys. On ECS: -`ecs:ListTasks` with the condition `ecs:cluster` = the cluster, and -`ecs:DescribeTasks` and `ecs:TagResource` only on the cluster's tasks, the -latter only for the five job tag keys (`aws:TagKeys`). +**A runner running another job** (ADR-0004). A runner can take only a job +whose labels it carries, and each carries its own job's label, so a runner +of this stack never runs another job. On `workflow_job` `in_progress` the +webhook still reads `runner_name`: a name that isn't `${stack}-<14-digit +stamp>-` (a hand launch, a GitHub-hosted runner, another stack), or +whose job id is the event's, is ignored (`200`, `ignored: no swap`). +Otherwise it logs `{"anomaly": "runner of another job", "runner", +"launched_for": , "took": {repo, job_id}}` and answers `200`. It +re-queues and re-tags nothing: the job the runner was launched for is +relaunched by the launcher's next check (#51), and the machine's tags still +name that job, so a reclaim of it is not reported (the notifier finds that +job not running on it). Until ADR-0004 the webhook repaired swaps instead +(ADR-0003). + +Its role allows `GetSecretValue` on the secret, `SendMessage` on the queue and +`lambda:InvokeFunction` on the notifier, and nothing on EC2 or ECS. **The queue** `${stack}-jobs` has a 120 s visibility timeout. A message that fails 5 receives goes to `${stack}-jobs-dlq` (kept 14 days). The outputs are @@ -377,8 +354,8 @@ failures per message), for each job: families of the job's architecture. Tags: `ci-runner:runner`, `ci-runner:runner-id`, `ci-runner:label`, `ci-runner:repo`, `ci-runner:job-id`, `ci-runner:run-id`, `ci-runner:head-sha`, - `ci-runner:job-name`, `ci-runner:labels` (the job's labels, comma-joined, - for the swap repair), and the shape: `ci-runner:capacity` (the capacity + `ci-runner:job-name`, `ci-runner:labels` (the job's labels, comma-joined), + and the shape: `ci-runner:capacity` (the capacity label), `ci-runner:vcpus` and `ci-runner:arch`; plus the launch template's `ci-runner:stack`. The `launched` log line carries `label`, `vcpus` and `arch`. @@ -396,7 +373,7 @@ minutes after its last launch, a gap that doubles for each launch after that, up to 60 minutes (its log line says `waiting` until then). So a job whose runners never take it (the repo left the group, labels nothing matches) costs about one launch an hour, not one every check. The message carries `launches` and -`launched_at` for this; one without them (the webhook's, a swap's re-queue) has +`launched_at` for this; one without them (the webhook's) has had no launch and launches at once. The checks end when GitHub no longer has the job queued, which drops the message. This is why a job held behind a concurrency group the first job holds gets a runner once the group @@ -538,8 +515,8 @@ vCPU tag counts as 4. EC2 instances and tasks never count towards each other's budgets. **A task's lifecycle** is an instance's: the sweeper stops it 2 hours after -launch and keeps its runner's registration while it lives, the swap repair -re-tags it, and the notifier reports a Fargate Spot reclaim (below). +launch and keeps its runner's registration while it lives, and the notifier +reports a Fargate Spot reclaim (below). ## Reporting a Spot reclaim: the notifier @@ -569,16 +546,14 @@ For each event it: 2. Signs an App JWT, finds the repo's installation (`GET /repos/{repo}/installation`) and gets its token. 3. Reads the job from GitHub (`GET /repos/{repo}/actions/jobs/{job_id}`), the - job the tags name: the one the machine was launched for, or the one it took - in a swap (the webhook re-tags it). It **reports only a job the machine's CI + job the tags name: the one the machine was launched for, the only one its + runner can take (ADR-0004). It **reports only a job the machine's CI runner is running**: `status` `in_progress` and `runner_name` equal to the machine's `ci-runner:runner` tag. Otherwise it logs `ignored : its runner is not running job (...)` and posts nothing: the job is `queued`, or `completed`, or another CI runner runs it. The usual case is an idle CI - runner: another CI runner took the job it was launched for, it took none, - and so its tags are still the launch tags. A swapped job that the webhook - has not yet re-tagged, in the moments after it started, is missed the same - way. + runner: its job was cancelled, or another launch's runner for the same job + (a relaunch, #51) took it. 4. Takes the job's real name from that job, falling back to the tag: a tag is cut at 256 characters, and a task's has `_` for brackets and commas. The webhook's word (step 8) carries GitHub's name too, so the two meet. diff --git a/docs/traps.md b/docs/traps.md index 7e6d480..81c3451 100644 --- a/docs/traps.md +++ b/docs/traps.md @@ -9,10 +9,10 @@ Symptoms seen on the stack, and their causes. A quoted section name is in | `permission denied` on `/var/run/docker.sock` | the host's `docker` group isn't gid 123 | | `setup-python` fails, or picks up a broken Python | `AGENT_TOOLSDIRECTORY` isn't `/opt/hostedtoolcache` | | A deploy waits 60 minutes, then fails on the state lock | an `apply` ran on Spot and was reclaimed; it must be `ci-ondemand` | -| A job sits "Waiting for a runner" | its capacity's vCPU budget is full, the repo isn't in the group, the App isn't installed on it, or the repo is public. Also check the label contract: `self-hosted`, exactly one of `ci-spot`/`ci-ondemand`/`ci-fargate-spot`/`ci-fargate`, exactly one size label (`ci-cpu-2`/`-4`/`-8`/`-16`), exactly one of `x64`/`arm64`, and otherwise only `linux` and the job label. A job label (`ci-job-...`) must be the job's own run and attempt with a job key after them, and the only one: the webhook logs `ignored` with a `why` otherwise (see "The job label" in `stack.md`). The launcher's log tells the first case (`deferred`) apart from the rest (no line at all). A job held behind a concurrency group shows `served` while its runner is up; once that runner idles out, a later check (5 minutes apart, see "Launching for queued jobs" in `stack.md`) launches another, no sooner than 15 minutes after the last launch and backing off to an hour (`waiting` in the log). A job that queued before the repo was turned on stays waiting after the fix: cancel and re-run it | +| A job sits "Waiting for a runner" | its capacity's vCPU budget is full, the repo isn't in the group, the App isn't installed on it, or the repo is public. Also check the label contract: `self-hosted`, exactly one of `ci-spot`/`ci-ondemand`/`ci-fargate-spot`/`ci-fargate`, exactly one size label (`ci-cpu-2`/`-4`/`-8`/`-16`), exactly one of `x64`/`arm64`, exactly one job label, and otherwise only `linux`. The job label (`ci-job-...`) must be the job's own run and attempt with a job key after them: the webhook logs `ignored` with a `why` otherwise (`no job label` when it has none) (see "The job label" in `stack.md`). The launcher's log tells the first case (`deferred`) apart from the rest (no line at all). A job held behind a concurrency group shows `served` while its runner is up; once that runner idles out, a later check (5 minutes apart, see "Launching for queued jobs" in `stack.md`) launches another, no sooner than 15 minutes after the last launch and backing off to an hour (`waiting` in the log). A job that queued before the repo was turned on stays waiting after the fix: cancel and re-run it | | A job that ran on our runners before now sits "Waiting for a runner", and the webhook's log shows no `enqueued` line for it | missing size or architecture label: its `runs-on` names only its capacity (`self-hosted`, `ci-spot`). There are no defaults; add a size and an architecture label (`ci-cpu-4` and `x64` keep the old shape), then cancel and re-run the job | | An `arm64` runner's instance comes up and terminates without taking its job, and its console shows `no matching manifest for linux/arm64` | `ghcr.io/zeroae/ci-runner:latest` has no arm64 entry yet: the image workflow hasn't pushed a multi-arch `:latest` since it began building arm64. Run the image workflow on `main`, then re-run the job | -| A job's runner was launched for another job (`$RUNNER_NAME` ends in another job id), and that other job sits "Waiting for a runner" | a swap: the job the runner took has no job label (see "The job label" in `stack.md`). The webhook repairs it on the job's `in_progress` (its log says `swap`), and the launcher then launches a runner for the other job. No `swap` line: check the webhook's log for errors (its role needs `ec2:DescribeInstances` and `ec2:CreateTags`, and for a task `ecs:ListTasks`, `ecs:DescribeTasks` and `ecs:TagResource`); cancel and re-run the waiting job | +| A job's runner was launched for another job (`$RUNNER_NAME` ends in another job id) | a swap, which the job label should make impossible (ADR-0004): the webhook logs `{"anomaly": "runner of another job", ...}` and repairs nothing. Compare the two jobs' `labels` (`gh api repos//actions/jobs/`): a runner registers with its job's labels, so the job it took must carry all of them. The other job is relaunched by the launcher's next check (`waiting`, then `launched`); a reclaim of that machine is not reported, since the notifier finds the job it was launched for not running on it | | A Fargate Spot job was lost (its task's `stopCode` is `SpotInterruption`) and its commit has no `ci-runner / ` check run | the notifier logged `ignored: is no job task of this stack` for every change of the stop: it found no job tags on the task. ECS keeps a stopped task's tags readable through `DescribeTasks` (measured live), so check the task's tags (`describe-tasks --include TAGS`, within an hour of the stop; `list-tags-for-resource` refuses a stopped task) and the notifier's role. Re-run the job | | A job fails "The runner has received a shutdown signal", "lost communication" or "exit code 130", next to a failed `ci-runner / ` check | Spot reclaim: re-run the job | | A job lost to a Spot reclaim stays in progress after its machine is gone; `gh run cancel` does nothing, and `gh run rerun` says `This workflow is already running` | GitHub didn't fail the job its runner died under. Let the run's other jobs finish (a force-cancel ends them all), then `gh api -X POST repos/zeroae//actions/runs//force-cancel` and re-run the failed jobs (see "Read a lost job") | From f7de6f8ea21c9e15669f1305d16d19c8bc06ea12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Fri, 20 Nov 2026 21:25:35 -0500 Subject: [PATCH 3/6] Docs: stack.md no longer describes the webhook's ECS tag copy Co-Authored-By: Claude Opus 5.5 --- docs/stack.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/stack.md b/docs/stack.md index 1c7ca8f..e56a8c0 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -458,10 +458,7 @@ are what a hand-run task needs. - the instances' `ci-runner:*` tags, plus `ci-runner:stack`. ECS allows only letters, digits, spaces and `_ . : / = + - @` in a tag, so on a task `ci-runner:labels` is joined by spaces, not commas, and any other character - (a job name's brackets or commas) becomes `_`. The webhook's re-tag - sanitizes the same way, with a copy of the launcher's `ECS_TAG_CHARS` and - `ecsTag` (bundling one costs more bytes than the copy); - `test/conventions.test.js` fails if the copies differ. + (a job name's brackets or commas) becomes `_`. The launch is logged as `launched: `. A `RunTask` that returns no task fails the message with `no task launched: : `. From 8ffa0711af477ea4f896448cc77ae2b8ac3f4fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Fri, 20 Nov 2026 21:28:33 -0500 Subject: [PATCH 4/6] Review fixes: a job without a job label can still swap; say what the re-check does then A runner's labels cover an unlabelled job of its shape, so swaps end only once every job carries its label; the docs and comments said they could no longer happen. After a swap the launcher logs `served` for the stranded job until the swapped machine ends, not `waiting`/`launched`. jobLabelFault reports the missing label itself, so it no longer throws on a job without one; checkRunner is logSwap; stale launcher comments about the webhook's tag copies are gone. Co-Authored-By: Claude Opus 5.5 --- GLOSSARY.md | 2 +- ...a-job-label-pins-each-runner-to-its-job.md | 2 +- docs/runbook.md | 4 +-- docs/stack.md | 19 +++++++----- docs/traps.md | 2 +- functions/launcher.js | 5 ++- functions/webhook.js | 31 +++++++++---------- 7 files changed, 33 insertions(+), 32 deletions(-) diff --git a/GLOSSARY.md b/GLOSSARY.md index c41f8f8..09fc278 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -33,7 +33,7 @@ The `runs-on` label unique to one job: `ci-job---- _Avoid_: run label (that names a run, shared by its jobs) **Swap**: -A CI runner taking a queued job other than the one it was launched for. Swaps can no longer happen: GitHub gives a job only to a runner whose labels include all of the job's, every job must carry its own job label, and each runner carries only its own job's. The webhook logs one as an anomaly if it ever sees it, and repairs nothing. +A CI runner taking a queued job other than the one it was launched for. GitHub gives a job only to a runner whose labels include all of the job's, and each runner carries its own job's job label, so no job that carries one can swap. A job of our shape without one gets no runner of its own but can still take another job's, so swaps end only once every such job carries its label. The webhook logs a swap as an anomaly, and repairs nothing. **Fargate capacity**: Fargate Spot or Fargate On-Demand, the two capacities with no Docker daemon. A job that picks one promises not to use Docker. diff --git a/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md b/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md index 4379e61..d908600 100644 --- a/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md +++ b/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md @@ -14,7 +14,7 @@ The launcher registers each runner with its job's labels, so the runner launched The rest of the label is the consumer's to make unique within the run. GitHub can't do it for them: inside `runs-on`, `github.job` evaluates to an empty string (measured on 2026-11-20 in a scratch repo, by reading the labels of the queued jobs back from the API), so the job key is written by hand. `strategy.job-index` does work there, and is `0` in a job with no matrix. A reusable workflow sees only its own job keys and its own matrix, so two calls in one run would make the same label: the caller passes something unique into it, as an input. -Rollout was in two steps. First the label was accepted and checked but not required: a job without it was served as before, and swaps were still repaired, including a labelled job's runner taking a job without the label. Once the consumers carried it, the label became required: a job of our shape without one is ignored, like any other job outside the label contract, and the webhook logs why. The swap repair is removed, and with it the webhook's EC2 and ECS permissions. A runner that runs a job other than its own, which should now be impossible, is logged as an anomaly and not repaired. #51's re-check stays: it covers runners that never take their job for other reasons (a concurrency group, a failed boot). +Rollout was in two steps. First the label was accepted and checked but not required: a job without it was served as before, and swaps were still repaired, including a labelled job's runner taking a job without the label. Once the consumers carried it, the label became required: a job of our shape without one is ignored, like any other job outside the label contract, and the webhook logs why. The swap repair is removed, and with it the webhook's EC2 and ECS permissions. A runner that runs a job other than its own, which only a job still without the label can make it do, is logged as an anomaly and not repaired. #51's re-check stays: it covers runners that never take their job for other reasons (a concurrency group, a failed boot). ## Considered Options diff --git a/docs/runbook.md b/docs/runbook.md index 8c3cd3c..e30ed6f 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -356,8 +356,8 @@ for a job from any repo but a private `zeroae` one (fence 3), `{"ignored": "", "job_id": ..., "why": ...}` for a job of our shape with no job label, or a bad one (see "The job label" in `stack.md`), and `{"anomaly": "runner of another job", "runner": ..., "launched_for": , "took": {...}}` if a -runner ever starts a job other than its own, which the job label should make -impossible (`docs/traps.md`). Everything else it ignores without a log line. +runner starts a job other than its own, which only a job without a job label +can make it do (`docs/traps.md`). Everything else it ignores without a log line. **The launcher** logs one JSON line per job, carrying the job's fields plus one of: diff --git a/docs/stack.md b/docs/stack.md index e56a8c0..2d94c39 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -239,7 +239,8 @@ something after that. It goes into the message's `labels` with the rest, the launcher mints the runner's JIT config with those labels, and so the runner launched for the job can take no job with another job label. GitHub gives a job to a runner whose labels include all of the job's, so a job without one -could take any runner of its shape: that is why the label is required. +could take any runner of its shape: that is why the label is required, and +why a job without one, though ignored, can still swap while it waits. A workflow writes it in `runs-on`, with the job key typed by hand: inside `runs-on`, `github.job` evaluates to an empty string. `strategy.job-index` is @@ -300,17 +301,21 @@ ignored, `200`), the webhook invokes the notifier asynchronously about the App changes. The webhook does no GitHub call itself. **A runner running another job** (ADR-0004). A runner can take only a job -whose labels it carries, and each carries its own job's label, so a runner -of this stack never runs another job. On `workflow_job` `in_progress` the +whose labels it carries, and each carries its own job's label, so no job +with a job label can take another job's runner. A job of our shape without +one gets no runner of its own, but can still take one launched for another +job: swaps end only once every such job carries its label. On +`workflow_job` `in_progress` the webhook still reads `runner_name`: a name that isn't `${stack}-<14-digit stamp>-` (a hand launch, a GitHub-hosted runner, another stack), or whose job id is the event's, is ignored (`200`, `ignored: no swap`). Otherwise it logs `{"anomaly": "runner of another job", "runner", "launched_for": , "took": {repo, job_id}}` and answers `200`. It -re-queues and re-tags nothing: the job the runner was launched for is -relaunched by the launcher's next check (#51), and the machine's tags still -name that job, so a reclaim of it is not reported (the notifier finds that -job not running on it). Until ADR-0004 the webhook repaired swaps instead +re-queues and re-tags nothing. The machine's tags still name the job it was +launched for, so the launcher's checks (#51) count that job as served (its +log says `served`) and launch it another runner only once the machine is +gone, at the end of the job it took; and a reclaim of the machine is not +reported (the notifier finds its launch job not running on it). Until ADR-0004 the webhook repaired swaps instead (ADR-0003). Its role allows `GetSecretValue` on the secret, `SendMessage` on the queue and diff --git a/docs/traps.md b/docs/traps.md index 81c3451..88f400d 100644 --- a/docs/traps.md +++ b/docs/traps.md @@ -12,7 +12,7 @@ Symptoms seen on the stack, and their causes. A quoted section name is in | A job sits "Waiting for a runner" | its capacity's vCPU budget is full, the repo isn't in the group, the App isn't installed on it, or the repo is public. Also check the label contract: `self-hosted`, exactly one of `ci-spot`/`ci-ondemand`/`ci-fargate-spot`/`ci-fargate`, exactly one size label (`ci-cpu-2`/`-4`/`-8`/`-16`), exactly one of `x64`/`arm64`, exactly one job label, and otherwise only `linux`. The job label (`ci-job-...`) must be the job's own run and attempt with a job key after them: the webhook logs `ignored` with a `why` otherwise (`no job label` when it has none) (see "The job label" in `stack.md`). The launcher's log tells the first case (`deferred`) apart from the rest (no line at all). A job held behind a concurrency group shows `served` while its runner is up; once that runner idles out, a later check (5 minutes apart, see "Launching for queued jobs" in `stack.md`) launches another, no sooner than 15 minutes after the last launch and backing off to an hour (`waiting` in the log). A job that queued before the repo was turned on stays waiting after the fix: cancel and re-run it | | A job that ran on our runners before now sits "Waiting for a runner", and the webhook's log shows no `enqueued` line for it | missing size or architecture label: its `runs-on` names only its capacity (`self-hosted`, `ci-spot`). There are no defaults; add a size and an architecture label (`ci-cpu-4` and `x64` keep the old shape), then cancel and re-run the job | | An `arm64` runner's instance comes up and terminates without taking its job, and its console shows `no matching manifest for linux/arm64` | `ghcr.io/zeroae/ci-runner:latest` has no arm64 entry yet: the image workflow hasn't pushed a multi-arch `:latest` since it began building arm64. Run the image workflow on `main`, then re-run the job | -| A job's runner was launched for another job (`$RUNNER_NAME` ends in another job id) | a swap, which the job label should make impossible (ADR-0004): the webhook logs `{"anomaly": "runner of another job", ...}` and repairs nothing. Compare the two jobs' `labels` (`gh api repos//actions/jobs/`): a runner registers with its job's labels, so the job it took must carry all of them. The other job is relaunched by the launcher's next check (`waiting`, then `launched`); a reclaim of that machine is not reported, since the notifier finds the job it was launched for not running on it | +| A job's runner was launched for another job (`$RUNNER_NAME` ends in another job id) | a swap: the webhook logs `{"anomaly": "runner of another job", ...}` and repairs nothing. The job the runner took almost surely has no job label (`gh api repos//actions/jobs/ --jq .labels`): a runner registers with its job's labels, so it can take only a job whose labels it carries. Add the label to that workflow. The other job waits: the launcher logs `served` for it until the swapped machine ends, then launches it a runner; cancel and re-run it to skip the wait. A reclaim of the swapped machine is not reported, since the notifier finds the job it was launched for not running on it | | A Fargate Spot job was lost (its task's `stopCode` is `SpotInterruption`) and its commit has no `ci-runner / ` check run | the notifier logged `ignored: is no job task of this stack` for every change of the stop: it found no job tags on the task. ECS keeps a stopped task's tags readable through `DescribeTasks` (measured live), so check the task's tags (`describe-tasks --include TAGS`, within an hour of the stop; `list-tags-for-resource` refuses a stopped task) and the notifier's role. Re-run the job | | A job fails "The runner has received a shutdown signal", "lost communication" or "exit code 130", next to a failed `ci-runner / ` check | Spot reclaim: re-run the job | | A job lost to a Spot reclaim stays in progress after its machine is gone; `gh run cancel` does nothing, and `gh run rerun` says `This workflow is already running` | GitHub didn't fail the job its runner died under. Let the run's other jobs finish (a force-cancel ends them all), then `gh api -X POST repos/zeroae//actions/runs//force-cancel` and re-run the failed jobs (see "Read a lost job") | diff --git a/functions/launcher.js b/functions/launcher.js index 48f0d8a..021981b 100644 --- a/functions/launcher.js +++ b/functions/launcher.js @@ -17,13 +17,12 @@ const CAPACITY = { 'ci-spot': 'spot', 'ci-ondemand': 'on-demand', 'ci-fargate-sp const PROVIDER = { 'ci-fargate-spot': 'FARGATE_SPOT', 'ci-fargate': 'FARGATE' }; // Fargate offers no arm64 in this AZ, so arm64 tasks skip a subnet there. const NO_ARM64_AZ = 'use1-az3'; -// ECS allows only these characters in a tag (EC2 allows any). The webhook's -// re-tag carries a copy. +// ECS allows only these characters in a tag (EC2 allows any). const ECS_TAG_CHARS = /[^\p{L}\p{Z}\p{N}_.:/=+\-@]/gu; const ecsTag = ([key, v]) => ({ key, value: String(v).replace(ECS_TAG_CHARS, '_').slice(0, 256) }); // An EC2 (Key, Value) or ECS (key, value) tag list as an object, keyed -// without the ci-runner: prefix. The webhook and the notifier carry copies. +// without the ci-runner: prefix. The notifier carries a copy. const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value])); // A queued job is checked again this many seconds after each check (#51): a diff --git a/functions/webhook.js b/functions/webhook.js index 2614e72..ba00de6 100644 --- a/functions/webhook.js +++ b/functions/webhook.js @@ -2,8 +2,7 @@ // webhook: GitHub workflow_job receiver behind a Function URL. Verifies the // delivery, applies the label contract, the job label (ADR-0004) and fence 3, // and enqueues the job for the launcher. It launches nothing itself, but -// logs a runner that runs another job than its own, which the job label -// should make impossible, and tells the notifier when a job of this stack's +// logs a runner that runs another job than its own (a swap), and tells the notifier when a job of this stack's // runners succeeds. // Inlined into the template; keep it one dependency-free CommonJS file. const crypto = require('node:crypto'); @@ -50,12 +49,13 @@ function shapeOf(labels) { // index>`: unique to one job, so the runner launched for the job, which // registers with the job's labels, can take no other. Required: GitHub gives // a job to any runner whose labels include all of the job's, so a job without -// one could take any runner of its shape (a swap). Returns why a job's job -// label breaks it, as { label, why }, or null; the handler checks first that -// it has one. +// one, though it gets no runner of its own, could take any runner of its shape +// (a swap). Returns why a job's labels break it, as { label, why }, or null. +const NO_JOB_LABEL = 'no job label'; const isJobLabel = (l) => String(l).toLowerCase().startsWith('ci-job-'); function jobLabelFault(labels, job) { const ls = (labels || []).filter(isJobLabel).map(String); + if (!ls.length) return { why: NO_JOB_LABEL }; if (ls.length > 1) return { label: ls.join(','), why: 'two job labels' }; const label = ls[0]; const m = label.toLowerCase().match(/^ci-job-(\d+)-(\d+)-(.*)$/); @@ -80,11 +80,12 @@ function createHandler({ secret, queue, notifier, env, log = console.log }) { return m && m[1] === env.STACK_NAME ? Number(m[2]) : null; } - // A job on a runner of this stack launched for another job: a swap, which - // the job label should make impossible (ADR-0004). It is logged, not - // repaired; if the runner's own job is stranded, the launcher's re-check - // (#51) launches it another runner. - function checkRunner(repo, job) { + // A job on a runner of this stack launched for another job: a swap. Once + // every job of our shape carries its job label none can happen (ADR-0004), + // so it is logged, not repaired. The runner's own job waits: the launcher's + // re-check (#51) sees the machine tagged with it serving, and launches it + // another runner only once that machine is gone. + function logSwap(repo, job) { const launchedFor = launchJobOf(job.runner_name); if (launchedFor === null || launchedFor === job.id) return reply(200, 'ignored: no swap'); log(JSON.stringify({ @@ -124,21 +125,17 @@ function createHandler({ secret, queue, notifier, env, log = console.log }) { log(JSON.stringify({ refused: repo.full_name, visibility: repo.visibility, job_id: job.id })); return reply(403, 'refused: only private repos of ' + env.GITHUB_ORG); } - if (p.action === 'in_progress') return checkRunner(repo, job); + if (p.action === 'in_progress') return logSwap(repo, job); if (p.action === 'completed') return reportSuccess(p, repo, job); if (p.action !== 'queued') return reply(200, 'ignored: action'); const shape = shapeOf(job.labels); if (!shape) return reply(200, 'ignored: labels'); // A job with no job label breaks the label contract, as one with no size - // does, but its own repo's workflow is waiting on it: say so in the log. - if (!job.labels.some(isJobLabel)) { - log(JSON.stringify({ ignored: repo.full_name, job_id: job.id, why: 'no job label' })); - return reply(200, 'ignored: labels'); - } + // does, but unlike a job outside our shape it is ours: the log says why. const fault = jobLabelFault(job.labels, job); if (fault) { log(JSON.stringify({ ignored: repo.full_name, job_id: job.id, ...fault })); - return reply(200, 'ignored: job label'); + return reply(200, fault.why === NO_JOB_LABEL ? 'ignored: labels' : 'ignored: job label'); } // The shape as the queue message carries it. The message's `label` is the From 983631317b3a74e7cc91b82f11c175175a4f85f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Fri, 20 Nov 2026 21:46:10 -0500 Subject: [PATCH 5/6] Launcher: a check relaunches a job at once when its runner took another job The webhook no longer re-tags a swapped machine, so its tags still name the job it was launched for and the check counted that job as served until the machine ended. Now a check asks GitHub whether each machine's runner is busy: busy while the job is still queued, it runs another job. When every machine of the job is so, the check reads the job again (its runner may have just taken it) and launches it a runner at once, with no relaunch gap, logging `swapped`. Widened into #56 by the owner. Co-Authored-By: Claude Opus 5.5 --- ...a-job-label-pins-each-runner-to-its-job.md | 2 +- docs/runbook.md | 4 +- docs/stack.md | 21 ++++-- docs/traps.md | 2 +- functions/launcher.js | 49 +++++++++---- functions/webhook.js | 6 +- test/launcher.test.js | 73 +++++++++++++++++-- 7 files changed, 126 insertions(+), 31 deletions(-) diff --git a/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md b/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md index d908600..11c3297 100644 --- a/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md +++ b/docs/adr/0004-a-job-label-pins-each-runner-to-its-job.md @@ -14,7 +14,7 @@ The launcher registers each runner with its job's labels, so the runner launched The rest of the label is the consumer's to make unique within the run. GitHub can't do it for them: inside `runs-on`, `github.job` evaluates to an empty string (measured on 2026-11-20 in a scratch repo, by reading the labels of the queued jobs back from the API), so the job key is written by hand. `strategy.job-index` does work there, and is `0` in a job with no matrix. A reusable workflow sees only its own job keys and its own matrix, so two calls in one run would make the same label: the caller passes something unique into it, as an input. -Rollout was in two steps. First the label was accepted and checked but not required: a job without it was served as before, and swaps were still repaired, including a labelled job's runner taking a job without the label. Once the consumers carried it, the label became required: a job of our shape without one is ignored, like any other job outside the label contract, and the webhook logs why. The swap repair is removed, and with it the webhook's EC2 and ECS permissions. A runner that runs a job other than its own, which only a job still without the label can make it do, is logged as an anomaly and not repaired. #51's re-check stays: it covers runners that never take their job for other reasons (a concurrency group, a failed boot). +Rollout was in two steps. First the label was accepted and checked but not required: a job without it was served as before, and swaps were still repaired, including a labelled job's runner taking a job without the label. Once the consumers carried it, the label became required: a job of our shape without one is ignored, like any other job outside the label contract, and the webhook logs why. The swap repair is removed, and with it the webhook's EC2 and ECS permissions. A runner that runs a job other than its own, which only a job still without the label can make it do, is logged as an anomaly and not repaired; the job it was launched for is relaunched by #51's re-check, which takes a runner busy while its job is still queued as no longer serving it. The re-check also covers runners that never take their job for other reasons (a concurrency group, a failed boot). ## Considered Options diff --git a/docs/runbook.md b/docs/runbook.md index e30ed6f..da9adf7 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -364,7 +364,9 @@ of: | Field | Meaning | |---|---| -| `"launched": "i-..."`, `"runner"`, `"label"` | an instance is up for the job | +| `"launched": "i-..."`, `"runner"`, `"label"` | an instance is up for the job; with `"swapped": [...]`, because the machines launched for it ran other jobs (a swap, see `docs/traps.md`) | +| `"served": "i-..."` | a machine launched for the job is still up and its runner idle; the job is checked again in 5 minutes | +| `"waiting": "relaunch from