From 5c5d99a703a06ae1a1f9a3294aa11f3bf83e0b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sat, 14 Nov 2026 18:24:33 -0500 Subject: [PATCH 1/5] Notifier: report only a job the machine's CI runner is running; clear a reclaim the job survived (#44) An idle CI runner (another one served its launch job) kept its launch tags, so a reclaim was reported for a job it never ran. The notifier now reads the tagged job from GitHub and posts only if it is in_progress on the machine's own runner. A job can also finish inside AWS's 2-minute warning. After posting, the notifier watches the job for 150 s and turns the check run neutral when the job ends with success. The Lambda's timeout goes from 30 s to 200 s. Co-Authored-By: Claude Sonnet 5.5 --- docs/runbook.md | 12 +++- docs/stack.md | 35 ++++++++--- functions/notifier.js | 42 ++++++++++--- template.yaml | 2 +- test/notifier.test.js | 134 ++++++++++++++++++++++++++++++++++++++--- test/tasks-ecs.test.js | 13 ++-- test/template.test.js | 7 +++ 7 files changed, 212 insertions(+), 33 deletions(-) diff --git a/docs/runbook.md b/docs/runbook.md index 9c04fc3..748d3b8 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -295,6 +295,14 @@ Its summary names the instance (or the task's id) and links the run. That is the whole story: re-run the failed job. Nothing retries it automatically. One check run per job: the notifier skips a job whose commit has it already. +AWS warns 2 minutes before it takes the machine, and a job can finish inside +that. The notifier watches the job for 150 s after the warning. If the job +ends with `success`, the check run turns `neutral`, titled `The job finished +despite the Spot reclaim warning`: no re-run is needed. A `failure` check run +whose job later passed anyway (the job ended after the watch) is also safe to +ignore. The notifier reports only a job the machine's CI runner was running: a +CI runner that took no job (another one served its launch job) is not reported. + To list those check runs on a commit: ```bash @@ -304,7 +312,9 @@ gh api repos/zeroae//commits//check-runs \ A job lost **without** that check run wasn't a Spot reclaim. Look in the notifier's log for a line `reported : job ` -(or `already reported ...`). An instance from `make hand-launch`, or a task +(or `already reported ...`, or `survived ...`). A line +`ignored : its runner is not running job (...)` means the machine's +CI runner ran no job when the warning came. An instance from `make hand-launch`, or a task run by hand, carries no job tags, so the notifier ignores it and logs `ignored: is no job instance of this stack` (`... no job task ...` for a task). A task's stop code is in `describe-tasks` (below): diff --git a/docs/stack.md b/docs/stack.md index 20f9ebd..c2b6121 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -470,18 +470,37 @@ For each event it: as is a task whose tags are gone. 2. Signs an App JWT, finds the repo's installation (`GET /repos/{repo}/installation`) and gets its token. -3. For a task, reads the job's real name from - `GET /repos/{repo}/actions/jobs/{job_id}` (the tag has `_` for brackets and - commas), falling back to the tag. -4. Skips the job if the commit already has its reclaim check run (same name, - `external_id` the job id, the title below), so a task's later changes and a - repeated event report nothing more. -5. Creates a check run on the head sha: name `ci-runner / ` (or +3. Reads the job from GitHub (`GET /repos/{repo}/actions/jobs/{job_id}`), the + job the tags name: the one the machine was launched for, or the one it took + in a swap (the webhook re-tags it). It **reports only a job the machine's CI + runner is running**: `status` `in_progress` and `runner_name` equal to the + machine's `ci-runner:runner` tag. Otherwise it logs `ignored : its runner + is not running job (...)` and posts nothing: the job is `queued`, or + `completed`, or another CI runner runs it. The usual case is an idle CI + runner: another CI runner took the job it was launched for, it took none, + and so its tags are still the launch tags. A swapped job that the webhook + has not yet re-tagged, in the moments after it started, is missed the same + way. +4. For a task, takes the job's real name from that job (the tag has `_` for + brackets and commas), falling back to the tag. +5. Skips the job if the commit already has its reclaim check run (same name, + `external_id` the job id, the title below or the survived title), so a task's + later changes and a repeated event report nothing more. +6. Creates a check run on the head sha: name `ci-runner / ` (or `job ` with no name), `completed`, conclusion `failure`, title `Spot capacity reclaimed: re-run this job`, a summary naming the job and the instance or task and linking the run, and `details_url` the job's page. An EC2 reclaim and a Fargate one give the same check run, but for the summary's machine. +7. **Watches the job** for 150 s (15 looks, 10 s apart; the Lambda's timeout is + 200 s): AWS warns 2 minutes before it takes the machine, and the job can + finish inside that. A job that completes with `success` was not lost, so the + notifier changes the check run to `neutral`, titled `The job finished + despite the Spot reclaim warning`, and logs `survived : job `. + A job that completes with any other conclusion, or is still running when the + watch ends, keeps the `failure`. A task's later changes find the check run + (either title) and stop at step 5, or find the job no longer running and stop + at step 3, so only the first change watches. A GitHub failure fails the invocation, and EventBridge's async invoke retries it. Two events handled at the same moment can still report twice. @@ -492,7 +511,7 @@ bytes than the copies. `test/conventions.test.js` fails if the copies differ, so change all three at once. Its role allows `GetSecretValue` on the secret, `ec2:DescribeInstances`, and -`ecs:DescribeTasks` only on the cluster's tasks. +`ecs:DescribeTasks` only on the cluster's tasks. The watch needs no more. ## The runner image in ECR diff --git a/functions/notifier.js b/functions/notifier.js index 39caf79..100da2e 100644 --- a/functions/notifier.js +++ b/functions/notifier.js @@ -20,7 +20,14 @@ function appJwt(appId, pem, nowMs) { // keyed without the ci-runner: prefix. const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value])); -function createHandler({ secret, fetch, ec2, ecs, now, env, log = console.log }) { +// AWS warns 2 minutes before it takes a Spot machine, and the job can finish +// inside the warning. After posting, the notifier watches the job for that long. +const WATCH_MS = 10000; +const WATCHES = 15; +const TITLE = 'Spot capacity reclaimed: re-run this job'; +const SURVIVED = 'The job finished despite the Spot reclaim warning'; + +function createHandler({ secret, fetch, sleep, ec2, ecs, now, env, log = console.log }) { async function gh(method, path, token, body) { const res = await fetch(API + path, { method, body: body && JSON.stringify(body), @@ -58,27 +65,47 @@ function createHandler({ secret, fetch, ec2, ecs, now, env, log = console.log }) const { id: installation } = await gh('GET', `/repos/${repo}/installation`, jwt); const { token } = await gh('POST', `/app/installations/${installation}/access_tokens`, jwt); + // Only a job this machine's CI runner is running can be lost with it. The + // tags name the job the machine was launched for, or took in a swap (the + // webhook re-tags it); a CI runner that took no job (another one served + // its launch job) is idle, and a completed job is no loss. + const ghJob = await gh('GET', `/repos/${repo}/actions/jobs/${jobId}`, token); + if (ghJob.status !== 'in_progress' || ghJob.runner_name !== t.runner) { + return log(`ignored ${id}: its runner is not running job ${jobId} (${ghJob.status}, runner ${ghJob.runner_name || 'none'})`); + } // A task's job-name tag has `_` for what ECS refuses (brackets, commas): // the check run carries the real name, from GitHub. - const ghJob = d.taskArn && (await gh('GET', `/repos/${repo}/actions/jobs/${jobId}`, token).catch(() => null)); - const job = (ghJob && ghJob.name) || t['job-name'] || `job ${jobId}`; + const job = (d.taskArn && ghJob.name) || t['job-name'] || `job ${jobId}`; const run = `https://github.com/${repo}/actions/runs/${t['run-id']}`; const name = `ci-runner / ${job}`; - const title = 'Spot capacity reclaimed: re-run this job'; // A task's reclaim matches on each of its changes, and EventBridge may // repeat an event: one check run per job is enough. const { check_runs: existing } = await gh('GET', `/repos/${repo}/commits/${t['head-sha']}/check-runs?filter=all&check_name=${encodeURIComponent(name)}`, token); - if (existing.some((c) => c.external_id === jobId && c.output && c.output.title === title)) return log(`already reported ${id}: ${repo} job ${jobId}`); - await gh('POST', `/repos/${repo}/check-runs`, token, { + if (existing.some((c) => c.external_id === jobId && c.output && [TITLE, SURVIVED].includes(c.output.title))) return log(`already reported ${id}: ${repo} job ${jobId}`); + const { id: checkId } = await gh('POST', `/repos/${repo}/check-runs`, token, { name, head_sha: t['head-sha'], external_id: jobId, status: 'completed', conclusion: 'failure', completed_at: new Date(now()).toISOString(), details_url: `${run}/job/${jobId}`, output: { - title, + title: TITLE, summary: `AWS reclaimed the Spot ${machine} running \`${job}\`. Re-run the job: ${run}`, }, }); log(`reported ${id}: ${repo} job ${jobId}`); + + // A job that finishes with success was not lost: the check run no longer + // asks for a re-run. Any other end, or none in time, keeps the failure. + for (let i = 0; i < WATCHES; i++) { + await sleep(WATCH_MS); + const latest = await gh('GET', `/repos/${repo}/actions/jobs/${jobId}`, token); + if (latest.status !== 'completed') continue; + if (latest.conclusion !== 'success') return; + await gh('PATCH', `/repos/${repo}/check-runs/${checkId}`, token, { + conclusion: 'neutral', + output: { title: SURVIVED, summary: `AWS warned that it would reclaim the Spot ${machine} running \`${job}\`, but the job finished with success. No re-run is needed: ${run}` }, + }); + return log(`survived ${id}: ${repo} job ${jobId}`); + } }; } @@ -96,6 +123,7 @@ module.exports.handler = async (event) => { live = createHandler({ secret: { get: async () => JSON.parse((await smc.send(new sm.GetSecretValueCommand({ SecretId: process.env.SECRET_ARN }))).SecretString) }, fetch, + sleep: (ms) => new Promise((r) => setTimeout(r, ms)), ec2: { describe: (id) => ec2c.send(new EC2.DescribeInstancesCommand({ InstanceIds: [id] })) }, ecs: { describeTasks: (i) => ecsc.send(new ECS.DescribeTasksCommand(i)) }, now: Date.now, diff --git a/template.yaml b/template.yaml index 1fe35db..b65386d 100644 --- a/template.yaml +++ b/template.yaml @@ -570,7 +570,7 @@ Resources: Runtime: nodejs22.x Handler: index.handler Role: !GetAtt NotifierRole.Arn - Timeout: 30 + Timeout: 200 MemorySize: 256 LoggingConfig: LogGroup: !Ref NotifierLogGroup diff --git a/test/notifier.test.js b/test/notifier.test.js index 1b696fb..6f1689a 100644 --- a/test/notifier.test.js +++ b/test/notifier.test.js @@ -31,9 +31,15 @@ function verifyJwt(auth) { return JSON.parse(Buffer.from(p, 'base64url')); } -// existing: the check runs already on the commit. -function fakeGitHub({ checkStatus = 201, jobStatus = 200, existing = [] } = {}) { +// The job the tags name, as GitHub reports it when this machine's CI runner +// is running it. +const RUNNING = { name: 'build (x64, a)', status: 'in_progress', runner_name: 'ci-runner-20261112115500-555' }; + +// existing: the check runs already on the commit. jobs: what GitHub says of +// the job on each look, the last one repeating. +function fakeGitHub({ checkStatus = 201, jobStatus = 200, existing = [], jobs = [RUNNING] } = {}) { const calls = []; + let looks = 0; const fetch = async (url, init = {}) => { const u = new URL(url); const key = `${init.method || 'GET'} ${u.pathname}`; @@ -54,7 +60,11 @@ function fakeGitHub({ checkStatus = 201, jobStatus = 200, existing = [] } = {}) const name = u.searchParams.get('check_name'); return res(200, { check_runs: existing.filter((c) => c.name === name) }); } - if (key === 'GET /repos/zeroae/app/actions/jobs/555') return res(jobStatus, { id: 555, name: 'build (x64, a)' }); + if (key === 'PATCH /repos/zeroae/app/check-runs/9') return res(200, { id: 9 }); + if (key === 'GET /repos/zeroae/app/actions/jobs/555') { + looks += 1; + return res(jobStatus, { id: 555, ...jobs[Math.min(looks, jobs.length) - 1] }); + } throw new Error('unexpected ' + key); }; return { fetch, calls, keys: () => calls.map((c) => c.key) }; @@ -65,6 +75,7 @@ function fakeGitHub({ checkStatus = 201, jobStatus = 200, existing = [] } = {}) function setup({ inst = instance(TAGS), raw = { Reservations: inst ? [{ Instances: [inst] }] : [] }, tasks = [], gh = fakeGitHub(), creds = { app_id: 42, private_key: PEM } } = {}) { const described = []; const logs = []; + const slept = []; const handler = createHandler({ secret: { get: async () => creds }, fetch: gh.fetch, @@ -75,22 +86,24 @@ function setup({ inst = instance(TAGS), raw = { Reservations: inst ? [{ Instance return { tasks: tasks.filter((t) => i.tasks.includes(t.taskArn)), failures: [] }; }, }, + sleep: async (ms) => { slept.push(ms); }, now: () => T0, env: ENV, log: (l) => logs.push(l), }); - return { handler, gh, described, logs }; + return { handler, gh, described, logs, slept }; } test('an interruption of one of our instances creates a failed check run on the job\'s commit', async () => { - const { handler, gh, described } = setup(); + const { handler, gh, described } = setup({ gh: fakeGitHub({ jobs: [RUNNING, DONE('failure')] }) }); await handler(EVENT); assert.deepEqual(described, ['i-0123']); assert.deepEqual(gh.keys(), [ 'GET /repos/zeroae/app/installation', 'POST /app/installations/181/access_tokens', - 'GET /repos/zeroae/app/commits/abc123/check-runs', 'POST /repos/zeroae/app/check-runs', + 'GET /repos/zeroae/app/actions/jobs/555', 'GET /repos/zeroae/app/commits/abc123/check-runs', 'POST /repos/zeroae/app/check-runs', + 'GET /repos/zeroae/app/actions/jobs/555', // the watch: the job failed too, so the check run stays ]); - const { body } = gh.calls[3]; + const { body } = gh.calls[4]; const runUrl = 'https://github.com/zeroae/app/actions/runs/444'; assert.equal(body.name, 'ci-runner / tests'); assert.equal(body.head_sha, 'abc123'); @@ -109,8 +122,76 @@ test('an interruption of one of our instances creates a failed check run on the } }); +// A machine is reported only for the job its CI runner is running. The tags +// say which job it was launched for, or took in a swap (the webhook re-tags it). +for (const [why, job] of [ + ['an idle CI runner, whose launch job another CI runner took', { ...RUNNING, runner_name: 'ci-runner-20261112115400-777' }], + ['a CI runner whose job is still queued', { ...RUNNING, status: 'queued', runner_name: null }], + ['a CI runner whose job already completed', { ...RUNNING, status: 'completed', conclusion: 'success' }], +]) { + test(`${why} is not reported`, async () => { + const { handler, gh, logs } = setup({ gh: fakeGitHub({ jobs: [job] }) }); + await handler(EVENT); + assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); + assert.match(logs.join('\n'), /not running job 555/); + }); +} + +test('a swapped CI runner is reported for the job it took, not the one it was launched for', async () => { + // The webhook re-tagged the instance with job 555; its runner is still named for job 999. + const runner = 'ci-runner-20261112115400-999'; + const inst = instance({ ...TAGS, 'ci-runner:runner': runner }); + const { handler, gh } = setup({ inst, gh: fakeGitHub({ jobs: [{ ...RUNNING, runner_name: runner }] }) }); + await handler(EVENT); + const { body } = gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs'); + assert.equal(body.external_id, '555'); + assert.equal(body.name, 'ci-runner / tests'); +}); + +// AWS gives a 2-minute warning, and a job can finish inside it. The check run +// is posted at the warning, then the notifier watches the job. +const SURVIVED = 'The job finished despite the Spot reclaim warning'; +const DONE = (conclusion) => ({ ...RUNNING, status: 'completed', conclusion }); +const patchOf = (gh) => gh.calls.find((c) => c.key === 'PATCH /repos/zeroae/app/check-runs/9'); + +test('a job that finishes with success after the warning turns its check run neutral', async () => { + const gh = fakeGitHub({ jobs: [RUNNING, RUNNING, DONE('success')] }); + const { handler, slept, logs } = setup({ gh }); + await handler(EVENT); + const { body } = patchOf(gh); + assert.equal(body.conclusion, 'neutral'); + assert.equal(body.output.title, SURVIVED); + assert.match(body.output.summary, /`tests`/); + assert.deepEqual(slept, [10000, 10000]); + assert.match(logs.join('\n'), /survived i-0123: zeroae\/app job 555/); +}); + +for (const conclusion of ['failure', 'cancelled']) { + test(`a job that completes with ${conclusion} after the warning keeps its failure`, async () => { + const gh = fakeGitHub({ jobs: [RUNNING, DONE(conclusion)] }); + await setup({ gh }).handler(EVENT); + assert.equal(patchOf(gh), undefined); + }); +} + +test('a job still running when the watch ends keeps its failure', async () => { + const gh = fakeGitHub({ jobs: [RUNNING] }); + const { handler, slept } = setup({ gh }); + await handler(EVENT); + assert.equal(patchOf(gh), undefined); + assert.equal(slept.length, 15); + assert.equal(slept.reduce((a, b) => a + b), 150000); +}); + +test('a reclaim whose check run turned neutral is not reported again', async () => { + const gh = fakeGitHub({ existing: [{ name: 'ci-runner / tests', external_id: '555', output: { title: SURVIVED } }] }); + const { handler } = setup({ gh }); + await handler(EVENT); + assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); +}); + test('a job name falls back to the job id when the tag is empty', async () => { - const { handler, gh } = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': '' }) }); + const { handler, gh } = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': '' })}); await handler(EVENT); assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.name, 'ci-runner / job 555'); }); @@ -214,8 +295,8 @@ test('a Fargate Spot reclaim creates the same check run as an EC2 one, with the assert.match(fargate.logs.join('\n'), /reported 0f1e2d3c4b5a69788796a5b4c3d2e1f0: zeroae\/app job 555/); }); -test('a task\'s job name falls back to its tag when GitHub can\'t say', async () => { - const { handler, gh } = setup({ inst: null, tasks: [task(TASK_TAGS)], gh: fakeGitHub({ jobStatus: 404 }) }); +test('a task\'s job name falls back to its tag when GitHub gives none', async () => { + const { handler, gh } = setup({ inst: null, tasks: [task(TASK_TAGS)], gh: fakeGitHub({ jobs: [{ ...RUNNING, name: undefined }] }) }); await handler(TASK_EVENT); assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.name, 'ci-runner / build _x64_ a_'); }); @@ -232,3 +313,36 @@ for (const [why, tasks] of [ assert.match(logs.join('\n'), /ignored/); }); } + +// A task's runner is named for its launch job too, and the webhook re-tags a +// swapped task the same way, so the same rules hold. +const TASK_RUNNING = { ...RUNNING, name: 'build (x64, a)' }; + +for (const [why, job] of [ + ['an idle Fargate CI runner', { ...TASK_RUNNING, runner_name: 'ci-runner-20261112115400-777' }], + ['a Fargate CI runner whose job is still queued', { ...TASK_RUNNING, status: 'queued', runner_name: null }], + ['a Fargate CI runner whose job already completed', { ...TASK_RUNNING, status: 'completed', conclusion: 'success' }], +]) { + test(`${why} is not reported`, async () => { + const { handler, gh, logs } = setup({ inst: null, tasks: [task(TASK_TAGS)], gh: fakeGitHub({ jobs: [job] }) }); + await handler(TASK_EVENT); + assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); + assert.match(logs.join('\n'), /not running job 555/); + }); +} + +test('a swapped Fargate CI runner is reported for the job it took', async () => { + const runner = 'ci-runner-20261112115400-999'; + const tags = { ...TASK_TAGS, 'ci-runner:runner': runner }; + const { handler, gh } = setup({ inst: null, tasks: [task(tags)], gh: fakeGitHub({ jobs: [{ ...TASK_RUNNING, runner_name: runner }] }) }); + await handler(TASK_EVENT); + assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.external_id, '555'); +}); + +test('a Fargate job that finishes with success after the warning turns its check run neutral', async () => { + const gh = fakeGitHub({ jobs: [TASK_RUNNING, { ...TASK_RUNNING, status: 'completed', conclusion: 'success' }] }); + await setup({ inst: null, tasks: [task(TASK_TAGS)], gh }).handler(TASK_EVENT); + const { body } = patchOf(gh); + assert.equal(body.conclusion, 'neutral'); + assert.match(body.output.summary, /task \(0f1e2d3c4b5a69788796a5b4c3d2e1f0\)/); +}); diff --git a/test/tasks-ecs.test.js b/test/tasks-ecs.test.js index 7395327..dcb933b 100644 --- a/test/tasks-ecs.test.js +++ b/test/tasks-ecs.test.js @@ -90,7 +90,7 @@ const secret = { get: async () => ({ app_id: 42, private_key: privateKey.export( // exists, JIT configs mint, and runners and check runs are recorded (and a // commit's check runs listed). const JOB_NAMES = { 999: 'build (x64, a)', 555: 'test (x64, b)' }; -function fakeGitHub(runners = []) { +function fakeGitHub(runners = [], job = {}) { const calls = []; const fetch = async (url, init = {}) => { const { pathname } = new URL(url); @@ -99,8 +99,8 @@ function fakeGitHub(runners = []) { const res = (status, json) => ({ ok: status < 300, status, json: async () => json }); if (key.endsWith('/installation')) return res(200, { id: 181 }); if (key.endsWith('/access_tokens')) return res(201, { token: 't', expires_at: new Date(Date.now() + 3600e3).toISOString() }); - const job = pathname.match(/\/actions\/jobs\/(\d+)$/); - if (job) return res(200, { id: Number(job[1]), status: 'queued', name: JOB_NAMES[job[1]] }); + const jobPath = pathname.match(/\/actions\/jobs\/(\d+)$/); + if (jobPath) return res(200, { id: Number(jobPath[1]), status: 'queued', name: JOB_NAMES[jobPath[1]], ...job }); if (key.endsWith('/actions/runner-groups')) return res(200, { runner_groups: [{ id: 3, name: 'ci-runner' }] }); if (key.endsWith('/runner-groups/3/runners')) return res(200, { total_count: runners.length, runners }); if (key.endsWith('/generate-jitconfig')) return res(201, { runner: { id: 77 }, encoded_jit_config: 'ZW5jb2RlZGppdA==' }); @@ -204,7 +204,7 @@ async function changesOf(arn, done) { } test('a Fargate Spot reclaim: template.yaml\'s rule matches each change of the stop, and the notifier reports it once', { skip }, async () => { - const { arn } = await launch(); + const { arn, runner } = await launch(); const pattern = JSON.stringify(notifierTaskPattern()); const matches = async (e) => (await aws.events('TestEventPattern', { EventPattern: pattern, Event: JSON.stringify(e) })).Result; // AWS stops a reclaimed task with SpotInterruption, the emulator only with @@ -215,9 +215,10 @@ test('a Fargate Spot reclaim: template.yaml\'s rule matches each change of the s const warning = { ...first, detail: { ...first.detail, desiredStatus: 'STOPPED', stopCode: 'SpotInterruption' } }; assert.equal(await matches(warning), true); - const gh = fakeGitHub(); + // The task's runner is running job 999 (the notifier reports only a running job). + const gh = fakeGitHub([], { status: 'in_progress', runner_name: runner }); const logs = []; - const h = notifier.createHandler({ secret, fetch: gh.fetch, ec2: { describe: assert.fail }, ecs, now: Date.now, env, log: (l) => logs.push(l) }); + const h = notifier.createHandler({ secret, fetch: gh.fetch, sleep: async () => {}, ec2: { describe: assert.fail }, ecs, now: Date.now, env, log: (l) => logs.push(l) }); await h(warning); const check = gh.calls.find((c) => c.key === 'POST /repos/zeroae/lib/check-runs'); assert.ok(check, logs.join('\n')); diff --git a/test/template.test.js b/test/template.test.js index 11e188f..f277191 100644 --- a/test/template.test.js +++ b/test/template.test.js @@ -169,6 +169,13 @@ test('a second notifier rule matches every change of a Fargate Spot reclaim in t assert.match(perm, /SourceArn: !GetAtt NotifierTaskRule\.Arn\n/); }); +test('the notifier\'s timeout outlasts its watch of a job after a Spot warning', () => { + const src = fs.readFileSync(path.join(__dirname, '..', 'functions', 'notifier.js'), 'utf8'); + const watchMs = Number(src.match(/^const WATCH_MS = (\d+);/m)[1]) * Number(src.match(/^const WATCHES = (\d+);/m)[1]); + const timeout = Number(resource('NotifierFunction').match(/\n {6}Timeout: (\d+)\n/)[1]); + assert.ok(timeout * 1000 >= watchMs + 30000, `${timeout} s is too short for a ${watchMs / 1000} s watch plus the calls around it`); +}); + test('the notifier may describe only the cluster\'s tasks, and nothing else on ECS', () => { const ecs = statements(resource('NotifierRole')).filter((s) => /ecs:/.test(s)); assert.deepEqual(ecs, [ From b0b2a74f0198ab479c4a85e2c6f1e13911520601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sat, 14 Nov 2026 18:26:13 -0500 Subject: [PATCH 2/5] Notifier: a retry or later event clears a standing reclaim report; review fixes (#44) A report left standing because the watch ended or a GitHub call failed mid-watch is now turned neutral by a retry or a later event once its job has finished with success. Names say what they hold (interval, looks, titles), the glossary's Spot reclaim allows a surviving job, and tests that repeated the EC2 fixtures share them. Co-Authored-By: Claude Sonnet 5.5 --- GLOSSARY.md | 2 +- docs/runbook.md | 4 +-- docs/stack.md | 6 ++-- functions/notifier.js | 71 ++++++++++++++++++++++++++----------------- test/notifier.test.js | 62 ++++++++++++++++++++++++++++--------- test/template.test.js | 2 +- 6 files changed, 99 insertions(+), 48 deletions(-) diff --git a/GLOSSARY.md b/GLOSSARY.md index e853615..e0217c5 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -40,4 +40,4 @@ The most vCPUs that the CI runners of one capacity may hold at once. A job that _Avoid_: "cap" in prose (that was the old runner count). The parameters that set the budgets are still named `*VcpuCap` (`SpotVcpuCap`, `OnDemandVcpuCap`, `FargateSpotVcpuCap`, `FargateVcpuCap`). **Spot reclaim**: -AWS taking back Spot capacity, EC2 or Fargate, while a job runs. The job is lost and is reported as one on its commit. +AWS taking back Spot capacity, EC2 or Fargate, while a job runs. The job is lost, unless it finishes inside AWS's 2-minute warning, and a lost job is reported as a reclaim on its commit. diff --git a/docs/runbook.md b/docs/runbook.md index 748d3b8..9b84e09 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -314,8 +314,8 @@ A job lost **without** that check run wasn't a Spot reclaim. Look in the notifier's log for a line `reported : job ` (or `already reported ...`, or `survived ...`). A line `ignored : its runner is not running job (...)` means the machine's -CI runner ran no job when the warning came. An instance from `make hand-launch`, or a task -run by hand, carries no job tags, so the notifier ignores it and logs +CI runner ran no job when the warning came. An instance from `make hand-launch`, +or a task run by hand, carries no job tags, so the notifier ignores it and logs `ignored: is no job instance of this stack` (`... no job task ...` for a task). A task's stop code is in `describe-tasks` (below): `SpotInterruption` is a reclaim. A job that sits diff --git a/docs/stack.md b/docs/stack.md index c2b6121..d693f42 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -499,8 +499,10 @@ For each event it: despite the Spot reclaim warning`, and logs `survived : job `. A job that completes with any other conclusion, or is still running when the watch ends, keeps the `failure`. A task's later changes find the check run - (either title) and stop at step 5, or find the job no longer running and stop - at step 3, so only the first change watches. + (either title) and stop at step 5, so only the first change watches. A + standing `failure` whose job has since finished with `success` (the watch + ended or failed, and a retry or a later change found it) is also changed to + `neutral` at step 5. A GitHub failure fails the invocation, and EventBridge's async invoke retries it. Two events handled at the same moment can still report twice. diff --git a/functions/notifier.js b/functions/notifier.js index 100da2e..f3190d6 100644 --- a/functions/notifier.js +++ b/functions/notifier.js @@ -20,12 +20,13 @@ function appJwt(appId, pem, nowMs) { // keyed without the ci-runner: prefix. const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value])); -// AWS warns 2 minutes before it takes a Spot machine, and the job can finish -// inside the warning. After posting, the notifier watches the job for that long. -const WATCH_MS = 10000; -const WATCHES = 15; -const TITLE = 'Spot capacity reclaimed: re-run this job'; -const SURVIVED = 'The job finished despite the Spot reclaim warning'; +// AWS warns just over 2 minutes before it takes a Spot machine, and the job +// can finish inside the warning. After posting, the notifier looks at the job +// WATCH_LOOKS times, WATCH_INTERVAL_MS apart: 150 s. +const WATCH_INTERVAL_MS = 10000; +const WATCH_LOOKS = 15; +const RECLAIMED_TITLE = 'Spot capacity reclaimed: re-run this job'; +const SURVIVED_TITLE = 'The job finished despite the Spot reclaim warning'; function createHandler({ secret, fetch, sleep, ec2, ecs, now, env, log = console.log }) { async function gh(method, path, token, body) { @@ -65,46 +66,60 @@ function createHandler({ secret, fetch, sleep, ec2, ecs, now, env, log = console const { id: installation } = await gh('GET', `/repos/${repo}/installation`, jwt); const { token } = await gh('POST', `/app/installations/${installation}/access_tokens`, jwt); + const jobPath = `/repos/${repo}/actions/jobs/${jobId}`; + const ghJob = await gh('GET', jobPath, token); + // A task's job-name tag has `_` for what ECS refuses (brackets, commas): + // the check run carries the real name, from GitHub. + const job = (d.taskArn && ghJob.name) || t['job-name'] || `job ${jobId}`; + const run = `https://github.com/${repo}/actions/runs/${t['run-id']}`; + const name = `ci-runner / ${job}`; + // The commit's reclaim check run for this job, if it has one: a task's + // reclaim matches on each of its changes, and EventBridge may repeat or + // retry an event. + const { check_runs: existing } = await gh('GET', `/repos/${repo}/commits/${t['head-sha']}/check-runs?filter=all&check_name=${encodeURIComponent(name)}`, token); + const reported = existing.find((c) => c.external_id === jobId && c.output && [RECLAIMED_TITLE, SURVIVED_TITLE].includes(c.output.title)); + + // A job that finishes with success was not lost: the check run no longer + // asks for a re-run. + async function clear(checkId) { + await gh('PATCH', `/repos/${repo}/check-runs/${checkId}`, token, { + conclusion: 'neutral', + output: { title: SURVIVED_TITLE, summary: `AWS warned that it would reclaim the Spot ${machine} running \`${job}\`, but the job finished with success. No re-run is needed: ${run}` }, + }); + log(`survived ${id}: ${repo} job ${jobId}`); + } + + if (reported) { + // An earlier invocation's report, left standing because its watch ended + // or failed: a job that has since finished with success clears it. + if (reported.output.title === RECLAIMED_TITLE && ghJob.status === 'completed' && ghJob.conclusion === 'success') return clear(reported.id); + return log(`already reported ${id}: ${repo} job ${jobId}`); + } // Only a job this machine's CI runner is running can be lost with it. The // tags name the job the machine was launched for, or took in a swap (the // webhook re-tags it); a CI runner that took no job (another one served // its launch job) is idle, and a completed job is no loss. - const ghJob = await gh('GET', `/repos/${repo}/actions/jobs/${jobId}`, token); if (ghJob.status !== 'in_progress' || ghJob.runner_name !== t.runner) { return log(`ignored ${id}: its runner is not running job ${jobId} (${ghJob.status}, runner ${ghJob.runner_name || 'none'})`); } - // A task's job-name tag has `_` for what ECS refuses (brackets, commas): - // the check run carries the real name, from GitHub. - const job = (d.taskArn && ghJob.name) || t['job-name'] || `job ${jobId}`; - const run = `https://github.com/${repo}/actions/runs/${t['run-id']}`; - const name = `ci-runner / ${job}`; - // A task's reclaim matches on each of its changes, and EventBridge may - // repeat an event: one check run per job is enough. - const { check_runs: existing } = await gh('GET', `/repos/${repo}/commits/${t['head-sha']}/check-runs?filter=all&check_name=${encodeURIComponent(name)}`, token); - if (existing.some((c) => c.external_id === jobId && c.output && [TITLE, SURVIVED].includes(c.output.title))) return log(`already reported ${id}: ${repo} job ${jobId}`); const { id: checkId } = await gh('POST', `/repos/${repo}/check-runs`, token, { name, head_sha: t['head-sha'], external_id: jobId, status: 'completed', conclusion: 'failure', completed_at: new Date(now()).toISOString(), details_url: `${run}/job/${jobId}`, output: { - title: TITLE, + title: RECLAIMED_TITLE, summary: `AWS reclaimed the Spot ${machine} running \`${job}\`. Re-run the job: ${run}`, }, }); log(`reported ${id}: ${repo} job ${jobId}`); - // A job that finishes with success was not lost: the check run no longer - // asks for a re-run. Any other end, or none in time, keeps the failure. - for (let i = 0; i < WATCHES; i++) { - await sleep(WATCH_MS); - const latest = await gh('GET', `/repos/${repo}/actions/jobs/${jobId}`, token); + // Any other end of the job, or none in time, keeps the failure. + for (let i = 0; i < WATCH_LOOKS; i++) { + await sleep(WATCH_INTERVAL_MS); + const latest = await gh('GET', jobPath, token); if (latest.status !== 'completed') continue; - if (latest.conclusion !== 'success') return; - await gh('PATCH', `/repos/${repo}/check-runs/${checkId}`, token, { - conclusion: 'neutral', - output: { title: SURVIVED, summary: `AWS warned that it would reclaim the Spot ${machine} running \`${job}\`, but the job finished with success. No re-run is needed: ${run}` }, - }); - return log(`survived ${id}: ${repo} job ${jobId}`); + if (latest.conclusion === 'success') await clear(checkId); + return; } }; } diff --git a/test/notifier.test.js b/test/notifier.test.js index 6f1689a..c099dae 100644 --- a/test/notifier.test.js +++ b/test/notifier.test.js @@ -35,6 +35,11 @@ function verifyJwt(auth) { // is running it. const RUNNING = { name: 'build (x64, a)', status: 'in_progress', runner_name: 'ci-runner-20261112115500-555' }; +const RECLAIMED = 'Spot capacity reclaimed: re-run this job'; +const SURVIVED = 'The job finished despite the Spot reclaim warning'; +const DONE = (conclusion) => ({ ...RUNNING, status: 'completed', conclusion }); +const patchOf = (gh) => gh.calls.find((c) => c.key === 'PATCH /repos/zeroae/app/check-runs/9'); + // existing: the check runs already on the commit. jobs: what GitHub says of // the job on each look, the last one repeating. function fakeGitHub({ checkStatus = 201, jobStatus = 200, existing = [], jobs = [RUNNING] } = {}) { @@ -101,7 +106,8 @@ test('an interruption of one of our instances creates a failed check run on the assert.deepEqual(gh.keys(), [ 'GET /repos/zeroae/app/installation', 'POST /app/installations/181/access_tokens', 'GET /repos/zeroae/app/actions/jobs/555', 'GET /repos/zeroae/app/commits/abc123/check-runs', 'POST /repos/zeroae/app/check-runs', - 'GET /repos/zeroae/app/actions/jobs/555', // the watch: the job failed too, so the check run stays + // The watch: the job failed too, so the check run stays. + 'GET /repos/zeroae/app/actions/jobs/555', ]); const { body } = gh.calls[4]; const runUrl = 'https://github.com/zeroae/app/actions/runs/444'; @@ -150,9 +156,6 @@ test('a swapped CI runner is reported for the job it took, not the one it was la // AWS gives a 2-minute warning, and a job can finish inside it. The check run // is posted at the warning, then the notifier watches the job. -const SURVIVED = 'The job finished despite the Spot reclaim warning'; -const DONE = (conclusion) => ({ ...RUNNING, status: 'completed', conclusion }); -const patchOf = (gh) => gh.calls.find((c) => c.key === 'PATCH /repos/zeroae/app/check-runs/9'); test('a job that finishes with success after the warning turns its check run neutral', async () => { const gh = fakeGitHub({ jobs: [RUNNING, RUNNING, DONE('success')] }); @@ -183,15 +186,47 @@ test('a job still running when the watch ends keeps its failure', async () => { assert.equal(slept.reduce((a, b) => a + b), 150000); }); +// A report an earlier invocation left standing (its watch ended, or a GitHub +// call failed mid-watch and EventBridge retried). +const STANDING = { id: 9, name: 'ci-runner / tests', external_id: '555', output: { title: RECLAIMED } }; + +test('a retry or a later event clears a standing report whose job has since finished with success', async () => { + const gh = fakeGitHub({ existing: [STANDING], jobs: [DONE('success')] }); + const { handler, logs } = setup({ gh }); + await handler(EVENT); + assert.equal(patchOf(gh).body.conclusion, 'neutral'); + assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); + assert.match(logs.join('\n'), /survived i-0123/); +}); + +for (const [why, job] of [['still runs', RUNNING], ['failed', DONE('failure')]]) { + test(`a retry or a later event leaves a standing report whose job ${why}`, async () => { + const gh = fakeGitHub({ existing: [STANDING], jobs: [job] }); + await setup({ gh }).handler(EVENT); + assert.equal(patchOf(gh), undefined); + assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); + }); +} + +test('a report that is already neutral is left alone', async () => { + const gh = fakeGitHub({ existing: [{ ...STANDING, output: { title: SURVIVED } }], jobs: [DONE('success')] }); + await setup({ gh }).handler(EVENT); + assert.equal(patchOf(gh), undefined); +}); + +test('a GitHub failure while looking at the job fails the invocation, so EventBridge retries it', async () => { + await assert.rejects(setup({ gh: fakeGitHub({ jobStatus: 500 }) }).handler(EVENT), /jobs\/555.*500/); +}); + test('a reclaim whose check run turned neutral is not reported again', async () => { - const gh = fakeGitHub({ existing: [{ name: 'ci-runner / tests', external_id: '555', output: { title: SURVIVED } }] }); + const gh = fakeGitHub({ existing: [{ id: 9, name: 'ci-runner / tests', external_id: '555', output: { title: SURVIVED } }] }); const { handler } = setup({ gh }); await handler(EVENT); assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); }); test('a job name falls back to the job id when the tag is empty', async () => { - const { handler, gh } = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': '' })}); + const { handler, gh } = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': '' }) }); await handler(EVENT); assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.name, 'ci-runner / job 555'); }); @@ -228,9 +263,9 @@ test('an event with no instance id is ignored without describing anything', asyn }); test('a reclaim already reported on the commit is not reported again', async () => { - const title = 'Spot capacity reclaimed: re-run this job'; + const title = RECLAIMED; const gh = fakeGitHub({ existing: [ - { name: 'ci-runner / tests', external_id: '555', output: { title } }, + { id: 9, name: 'ci-runner / tests', external_id: '555', output: { title } }, ] }); const { handler, logs } = setup({ gh }); await handler(EVENT); @@ -316,12 +351,11 @@ for (const [why, tasks] of [ // A task's runner is named for its launch job too, and the webhook re-tags a // swapped task the same way, so the same rules hold. -const TASK_RUNNING = { ...RUNNING, name: 'build (x64, a)' }; for (const [why, job] of [ - ['an idle Fargate CI runner', { ...TASK_RUNNING, runner_name: 'ci-runner-20261112115400-777' }], - ['a Fargate CI runner whose job is still queued', { ...TASK_RUNNING, status: 'queued', runner_name: null }], - ['a Fargate CI runner whose job already completed', { ...TASK_RUNNING, status: 'completed', conclusion: 'success' }], + ['an idle Fargate CI runner', { ...RUNNING, runner_name: 'ci-runner-20261112115400-777' }], + ['a Fargate CI runner whose job is still queued', { ...RUNNING, status: 'queued', runner_name: null }], + ['a Fargate CI runner whose job already completed', { ...RUNNING, status: 'completed', conclusion: 'success' }], ]) { test(`${why} is not reported`, async () => { const { handler, gh, logs } = setup({ inst: null, tasks: [task(TASK_TAGS)], gh: fakeGitHub({ jobs: [job] }) }); @@ -334,13 +368,13 @@ for (const [why, job] of [ test('a swapped Fargate CI runner is reported for the job it took', async () => { const runner = 'ci-runner-20261112115400-999'; const tags = { ...TASK_TAGS, 'ci-runner:runner': runner }; - const { handler, gh } = setup({ inst: null, tasks: [task(tags)], gh: fakeGitHub({ jobs: [{ ...TASK_RUNNING, runner_name: runner }] }) }); + const { handler, gh } = setup({ inst: null, tasks: [task(tags)], gh: fakeGitHub({ jobs: [{ ...RUNNING, runner_name: runner }] }) }); await handler(TASK_EVENT); assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.external_id, '555'); }); test('a Fargate job that finishes with success after the warning turns its check run neutral', async () => { - const gh = fakeGitHub({ jobs: [TASK_RUNNING, { ...TASK_RUNNING, status: 'completed', conclusion: 'success' }] }); + const gh = fakeGitHub({ jobs: [RUNNING, { ...RUNNING, status: 'completed', conclusion: 'success' }] }); await setup({ inst: null, tasks: [task(TASK_TAGS)], gh }).handler(TASK_EVENT); const { body } = patchOf(gh); assert.equal(body.conclusion, 'neutral'); diff --git a/test/template.test.js b/test/template.test.js index f277191..f018031 100644 --- a/test/template.test.js +++ b/test/template.test.js @@ -171,7 +171,7 @@ test('a second notifier rule matches every change of a Fargate Spot reclaim in t test('the notifier\'s timeout outlasts its watch of a job after a Spot warning', () => { const src = fs.readFileSync(path.join(__dirname, '..', 'functions', 'notifier.js'), 'utf8'); - const watchMs = Number(src.match(/^const WATCH_MS = (\d+);/m)[1]) * Number(src.match(/^const WATCHES = (\d+);/m)[1]); + const watchMs = Number(src.match(/^const WATCH_INTERVAL_MS = (\d+);/m)[1]) * Number(src.match(/^const WATCH_LOOKS = (\d+);/m)[1]); const timeout = Number(resource('NotifierFunction').match(/\n {6}Timeout: (\d+)\n/)[1]); assert.ok(timeout * 1000 >= watchMs + 30000, `${timeout} s is too short for a ${watchMs / 1000} s watch plus the calls around it`); }); From 3fabe55d2e5ef1004cb3d988fb715eb0c41e37bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sat, 14 Nov 2026 18:38:52 -0500 Subject: [PATCH 3/5] Webhook tells the notifier a job succeeded; the notifier no longer watches (#44) Watching the job for 150 s inside the notifier held a Lambda per reclaim and missed a job that ended after the watch. GitHub already delivers workflow_job completed to the webhook (about 3 deliveries per job), so the webhook now invokes the notifier when a job succeeds on a runner of this stack, and the notifier turns that job's reclaim report neutral. The notifier re-reads the job once after posting, to close the race with that word, and a retry or later event still clears a standing report. The notifier's timeout returns to 30 s. The webhook gains lambda:InvokeFunction on the notifier. Co-Authored-By: Claude Sonnet 5.5 --- docs/runbook.md | 14 +++---- docs/stack.md | 46 ++++++++++++++------- functions/notifier.js | 91 +++++++++++++++++++++++------------------- functions/webhook.js | 24 ++++++++++- template.yaml | 7 +++- test/notifier.test.js | 88 +++++++++++++++++++++++++++------------- test/tasks-ecs.test.js | 2 +- test/template.test.js | 7 ---- test/webhook.test.js | 31 +++++++++++++- 9 files changed, 209 insertions(+), 101 deletions(-) diff --git a/docs/runbook.md b/docs/runbook.md index 9b84e09..6882bc1 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -296,12 +296,12 @@ the whole story: re-run the failed job. Nothing retries it automatically. One check run per job: the notifier skips a job whose commit has it already. AWS warns 2 minutes before it takes the machine, and a job can finish inside -that. The notifier watches the job for 150 s after the warning. If the job -ends with `success`, the check run turns `neutral`, titled `The job finished -despite the Spot reclaim warning`: no re-run is needed. A `failure` check run -whose job later passed anyway (the job ended after the watch) is also safe to -ignore. The notifier reports only a job the machine's CI runner was running: a -CI runner that took no job (another one served its launch job) is not reported. +that. When the job ends with `success`, the check run turns `neutral`, titled +`The job finished despite the Spot reclaim warning`: no re-run is needed (the +webhook tells the notifier; see `docs/stack.md`, "Reporting a Spot reclaim"). +A `failure` check run whose job passed anyway is safe to ignore. The notifier reports +only a job the machine's CI runner was running: a CI runner that took no job +(another one served its launch job) is not reported. To list those check runs on a commit: @@ -312,7 +312,7 @@ gh api repos/zeroae//commits//check-runs \ A job lost **without** that check run wasn't a Spot reclaim. Look in the notifier's log for a line `reported : job ` -(or `already reported ...`, or `survived ...`). A line +(or `already reported ...`, or `survived: ...`). A line `ignored : its runner is not running job (...)` means the machine's CI runner ran no job when the warning came. An instance from `make hand-launch`, or a task run by hand, carries no job tags, so the notifier ignores it and logs diff --git a/docs/stack.md b/docs/stack.md index d693f42..10c8623 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -209,16 +209,28 @@ App's webhook points at), for each delivery: 3. **Fence 3:** refuses (`403`, logged) unless `repository.visibility` is `private` and the owner is `GITHUB_ORG`. 4. On `in_progress`, repairs a swap (below) and answers. -5. Ignores (`200`) actions other than `queued`, and jobs outside the label +5. On `completed`, tells the notifier when the job succeeded on a runner of + this stack (below) and answers. +6. Ignores (`200`) actions other than `queued`, and jobs outside the label contract (case-insensitive): `self-hosted`, exactly one of `ci-spot`/`ci-ondemand`/`ci-fargate-spot`/`ci-fargate`, exactly one size label (`ci-cpu-2`, `-4`, `-8`, `-16`), exactly one of `x64`/`arm64`, and optionally `linux`. None has a default: a missing size or architecture label, an unknown `ci-cpu-*`, or both `x64` and `arm64`, are outside it. -6. Enqueues `{repo, job_id, run_id, head_sha, job_name, label, vcpus, arch, +7. Enqueues `{repo, job_id, run_id, head_sha, job_name, label, vcpus, arch, labels, installation_id}` (`label` is the capacity label, lower case; `vcpus` is 2, 4, 8 or 16; `arch` is `x64` or `arm64`) and answers `202`. +**Telling the notifier a job succeeded.** On `workflow_job` `completed` with +conclusion `success`, and a `runner_name` that is `${stack}-<14-digit +stamp>-` (a job on a GitHub-hosted runner, another stack's, or none is +ignored, `200`), the webhook invokes the notifier asynchronously +(`lambda:InvokeFunction` on it, env `NOTIFIER_FUNCTION`) with `{jobSucceeded: +{repo, job_id, run_id, head_sha, job_name, installation_id}}` and answers +`202`. The App's `workflow_job` subscription already delivers every action +(about 3 deliveries per job: `queued`, `in_progress`, `completed`), so nothing +about the App changes. The webhook does no GitHub call itself. + **Swap repair** (ADR-0003). GitHub gives a queued job to any idle runner whose labels cover it, so a runner launched for job Y may take job X. Left alone, X's own message is later dropped as not queued, and Y never gets a runner. @@ -492,17 +504,23 @@ For each event it: the instance or task and linking the run, and `details_url` the job's page. An EC2 reclaim and a Fargate one give the same check run, but for the summary's machine. -7. **Watches the job** for 150 s (15 looks, 10 s apart; the Lambda's timeout is - 200 s): AWS warns 2 minutes before it takes the machine, and the job can - finish inside that. A job that completes with `success` was not lost, so the - notifier changes the check run to `neutral`, titled `The job finished - despite the Spot reclaim warning`, and logs `survived : job `. - A job that completes with any other conclusion, or is still running when the - watch ends, keeps the `failure`. A task's later changes find the check run - (either title) and stop at step 5, so only the first change watches. A - standing `failure` whose job has since finished with `success` (the watch - ended or failed, and a retry or a later change found it) is also changed to - `neutral` at step 5. +7. Reads the job once more after posting: a job that finished with `success` + while this ran is cleared as in step 8. +8. **Clears a report the job survived.** AWS warns 2 minutes before it takes + the machine, and a job can finish inside that. A job that finishes with + `success` was not lost, so its reclaim check run changes to `neutral`, + titled `The job finished despite the Spot reclaim warning`, and the notifier + logs `survived: `. Three things get a report there: + - **the webhook's word** (above): the notifier gets the token (the event + carries the installation id), finds the job's reclaim check run, and + clears it. A job with no reclaim report logs `nothing to clear` and + changes nothing. The check run's name uses the event's job name; + - **a retry or a later change** of a reclaim event that finds its own + standing `failure` (step 5) for a job that has since finished with + `success`; + - **step 7**, for the word that came before the report existed. + + A job that ends with any other conclusion keeps the `failure`. A GitHub failure fails the invocation, and EventBridge's async invoke retries it. Two events handled at the same moment can still report twice. @@ -513,7 +531,7 @@ bytes than the copies. `test/conventions.test.js` fails if the copies differ, so change all three at once. Its role allows `GetSecretValue` on the secret, `ec2:DescribeInstances`, and -`ecs:DescribeTasks` only on the cluster's tasks. The watch needs no more. +`ecs:DescribeTasks` only on the cluster's tasks. ## The runner image in ECR diff --git a/functions/notifier.js b/functions/notifier.js index f3190d6..beb127d 100644 --- a/functions/notifier.js +++ b/functions/notifier.js @@ -3,6 +3,8 @@ // Fargate Spot task's stop (ECS Task State Change, stopCode SpotInterruption). // If the instance or task is this stack's and carries a job's launch tags (see // launcher.js), it creates a failed check run on the job's commit through the App. +// The webhook also invokes it when a job of this stack's runners succeeds: a +// reclaim report for that job turns neutral. // The App JWT is a copy of the launcher's: bundling a shared module would cost // more template bytes than the copy does. const crypto = require('node:crypto'); @@ -20,15 +22,10 @@ function appJwt(appId, pem, nowMs) { // keyed without the ci-runner: prefix. const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value])); -// AWS warns just over 2 minutes before it takes a Spot machine, and the job -// can finish inside the warning. After posting, the notifier looks at the job -// WATCH_LOOKS times, WATCH_INTERVAL_MS apart: 150 s. -const WATCH_INTERVAL_MS = 10000; -const WATCH_LOOKS = 15; const RECLAIMED_TITLE = 'Spot capacity reclaimed: re-run this job'; const SURVIVED_TITLE = 'The job finished despite the Spot reclaim warning'; -function createHandler({ secret, fetch, sleep, ec2, ecs, now, env, log = console.log }) { +function createHandler({ secret, fetch, ec2, ecs, now, env, log = console.log }) { async function gh(method, path, token, body) { const res = await fetch(API + path, { method, body: body && JSON.stringify(body), @@ -39,7 +36,46 @@ function createHandler({ secret, fetch, sleep, ec2, ecs, now, env, log = console return json; } + // An installation token for the repo. The webhook passes the installation id + // it already has; an EventBridge event carries none. + async function tokenFor(repo, installationId) { + const s = await secret.get(); + if (!s.private_key) throw new Error('the App is not set up'); + const jwt = appJwt(s.app_id, s.private_key, now()); + const installation = installationId || (await gh('GET', `/repos/${repo}/installation`, jwt)).id; + return (await gh('POST', `/app/installations/${installation}/access_tokens`, jwt)).token; + } + + // The commit's reclaim check run for a job, if it has one. A task's reclaim + // matches on each of its changes, and EventBridge may repeat or retry an + // event: one check run per job is enough. + async function reportedFor(repo, token, sha, name, jobId) { + const { check_runs: existing } = await gh('GET', `/repos/${repo}/commits/${sha}/check-runs?filter=all&check_name=${encodeURIComponent(name)}`, token); + return existing.find((c) => c.external_id === jobId && c.output && [RECLAIMED_TITLE, SURVIVED_TITLE].includes(c.output.title)); + } + + // A job that finished with success was not lost: its check run no longer + // asks for a re-run. + async function clear(repo, token, checkId, job, run) { + await gh('PATCH', `/repos/${repo}/check-runs/${checkId}`, token, { + conclusion: 'neutral', + output: { title: SURVIVED_TITLE, summary: `AWS warned that it would reclaim the Spot machine running \`${job}\`, but the job finished with success. No re-run is needed: ${run}` }, + }); + log(`survived: ${repo} ${job}`); + } + + // The webhook says a job of this stack's runners finished with success. A + // job that had a reclaim report (AWS warns 2 minutes before it takes the + // machine, and a job can finish inside that) is cleared. + async function jobSucceeded(j) { + const token = await tokenFor(j.repo, j.installation_id); + const reported = await reportedFor(j.repo, token, j.head_sha, `ci-runner / ${j.job_name}`, String(j.job_id)); + if (!reported || reported.output.title !== RECLAIMED_TITLE) return log(`nothing to clear: ${j.repo} job ${j.job_id}`); + return clear(j.repo, token, reported.id, j.job_name, `https://github.com/${j.repo}/actions/runs/${j.run_id}`); + } + return async (event) => { + if (event.jobSucceeded) return jobSucceeded(event.jobSucceeded); const d = event.detail || {}; let id, machine, t; if (d.taskArn) { @@ -60,12 +96,7 @@ function createHandler({ secret, fetch, sleep, ec2, ecs, now, env, log = console return log(`ignored: ${id} is no job ${d.taskArn ? 'task' : 'instance'} of this stack`); } - const s = await secret.get(); - if (!s.private_key) throw new Error('the App is not set up'); - const jwt = appJwt(s.app_id, s.private_key, now()); - const { id: installation } = await gh('GET', `/repos/${repo}/installation`, jwt); - const { token } = await gh('POST', `/app/installations/${installation}/access_tokens`, jwt); - + const token = await tokenFor(repo); const jobPath = `/repos/${repo}/actions/jobs/${jobId}`; const ghJob = await gh('GET', jobPath, token); // A task's job-name tag has `_` for what ECS refuses (brackets, commas): @@ -73,26 +104,13 @@ function createHandler({ secret, fetch, sleep, ec2, ecs, now, env, log = console const job = (d.taskArn && ghJob.name) || t['job-name'] || `job ${jobId}`; const run = `https://github.com/${repo}/actions/runs/${t['run-id']}`; const name = `ci-runner / ${job}`; - // The commit's reclaim check run for this job, if it has one: a task's - // reclaim matches on each of its changes, and EventBridge may repeat or - // retry an event. - const { check_runs: existing } = await gh('GET', `/repos/${repo}/commits/${t['head-sha']}/check-runs?filter=all&check_name=${encodeURIComponent(name)}`, token); - const reported = existing.find((c) => c.external_id === jobId && c.output && [RECLAIMED_TITLE, SURVIVED_TITLE].includes(c.output.title)); - - // A job that finishes with success was not lost: the check run no longer - // asks for a re-run. - async function clear(checkId) { - await gh('PATCH', `/repos/${repo}/check-runs/${checkId}`, token, { - conclusion: 'neutral', - output: { title: SURVIVED_TITLE, summary: `AWS warned that it would reclaim the Spot ${machine} running \`${job}\`, but the job finished with success. No re-run is needed: ${run}` }, - }); - log(`survived ${id}: ${repo} job ${jobId}`); - } + const finishedWithSuccess = (j) => j.status === 'completed' && j.conclusion === 'success'; + const reported = await reportedFor(repo, token, t['head-sha'], name, jobId); if (reported) { - // An earlier invocation's report, left standing because its watch ended - // or failed: a job that has since finished with success clears it. - if (reported.output.title === RECLAIMED_TITLE && ghJob.status === 'completed' && ghJob.conclusion === 'success') return clear(reported.id); + // An earlier event's report, left standing: the job finished with + // success after the webhook's word, or before the report existed. + if (reported.output.title === RECLAIMED_TITLE && finishedWithSuccess(ghJob)) return clear(repo, token, reported.id, job, run); return log(`already reported ${id}: ${repo} job ${jobId}`); } // Only a job this machine's CI runner is running can be lost with it. The @@ -112,15 +130,9 @@ function createHandler({ secret, fetch, sleep, ec2, ecs, now, env, log = console }, }); log(`reported ${id}: ${repo} job ${jobId}`); - - // Any other end of the job, or none in time, keeps the failure. - for (let i = 0; i < WATCH_LOOKS; i++) { - await sleep(WATCH_INTERVAL_MS); - const latest = await gh('GET', jobPath, token); - if (latest.status !== 'completed') continue; - if (latest.conclusion === 'success') await clear(checkId); - return; - } + // The job may have finished while this ran, and the webhook's word then + // found no report to clear. + if (finishedWithSuccess(await gh('GET', jobPath, token))) await clear(repo, token, checkId, job, run); }; } @@ -138,7 +150,6 @@ module.exports.handler = async (event) => { live = createHandler({ secret: { get: async () => JSON.parse((await smc.send(new sm.GetSecretValueCommand({ SecretId: process.env.SECRET_ARN }))).SecretString) }, fetch, - sleep: (ms) => new Promise((r) => setTimeout(r, ms)), ec2: { describe: (id) => ec2c.send(new EC2.DescribeInstancesCommand({ InstanceIds: [id] })) }, ecs: { describeTasks: (i) => ecsc.send(new ECS.DescribeTasksCommand(i)) }, now: Date.now, diff --git a/functions/webhook.js b/functions/webhook.js index 61033fd..e793fb5 100644 --- a/functions/webhook.js +++ b/functions/webhook.js @@ -1,7 +1,8 @@ 'use strict'; // webhook: GitHub workflow_job receiver behind a Function URL. Verifies the // delivery, applies the label contract and fence 3, and enqueues the job for -// the launcher. It launches nothing itself, but repairs swaps (ADR-0003). +// the launcher. It launches nothing itself, but repairs swaps (ADR-0003) and +// tells the notifier when a job of this stack's runners succeeds. // Inlined into the template; keep it one dependency-free CommonJS file. const crypto = require('node:crypto'); @@ -56,7 +57,7 @@ const ecsTag = ([key, v]) => ({ key, value: String(v).replace(ECS_TAG_CHARS, '_' // keyed without the ci-runner: prefix. const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value])); -function createHandler({ secret, queue, ec2, ecs, env, log = console.log }) { +function createHandler({ secret, queue, notifier, ec2, ecs, env, log = console.log }) { // The runner's task in the cluster, not yet told to stop, with its tags. async function findTask(name) { let nextToken; @@ -117,6 +118,19 @@ function createHandler({ secret, queue, ec2, ecs, env, log = console.log }) { return reply(202, 'swap repaired'); } + // A job that finished with success on a runner of this stack. The notifier + // clears its reclaim report for the job, if the job had one: AWS warns 2 + // minutes before it takes a Spot machine, and a job can finish inside that. + async function reportSuccess(p, repo, job) { + const m = String(job.runner_name || '').match(/^(.+)-\d{14}-\d+$/); + if (job.conclusion !== 'success' || !m || m[1] !== env.STACK_NAME) return reply(200, 'ignored: not a success of ours'); + await notifier.send({ jobSucceeded: { + repo: repo.full_name, job_id: job.id, run_id: job.run_id, head_sha: job.head_sha, job_name: job.name, + installation_id: p.installation && p.installation.id, + } }); + return reply(202, 'notifier told'); + } + return async (event) => { if (!key) key = (await secret.get()).webhook_secret; if (!key) return reply(503, 'not set up'); @@ -136,6 +150,7 @@ function createHandler({ secret, queue, ec2, ecs, env, log = console.log }) { return reply(403, 'refused: only private repos of ' + env.GITHUB_ORG); } if (p.action === 'in_progress') return repairSwap(p, repo, job); + if (p.action === 'completed') return reportSuccess(p, repo, job); if (p.action !== 'queued') return reply(200, 'ignored: action'); const shape = shapeOf(job.labels); if (!shape) return reply(200, 'ignored: labels'); @@ -160,15 +175,20 @@ module.exports.handler = async (event) => { if (!live) { const sm = require('@aws-sdk/client-secrets-manager'); const sqs = require('@aws-sdk/client-sqs'); + const LAMBDA = require('@aws-sdk/client-lambda'); const EC2 = require('@aws-sdk/client-ec2'); const smc = new sm.SecretsManagerClient({}); const sqsc = new sqs.SQSClient({}); + const lambdac = new LAMBDA.LambdaClient({}); const ECS = require('@aws-sdk/client-ecs'); const ec2c = new EC2.EC2Client({}); const ecsc = new ECS.ECSClient({}); live = createHandler({ secret: { get: async () => JSON.parse((await smc.send(new sm.GetSecretValueCommand({ SecretId: process.env.SECRET_ARN }))).SecretString) }, queue: { send: async (m) => { await sqsc.send(new sqs.SendMessageCommand({ QueueUrl: process.env.QUEUE_URL, MessageBody: JSON.stringify(m) })); } }, + notifier: { + send: async (m) => { await lambdac.send(new LAMBDA.InvokeCommand({ FunctionName: process.env.NOTIFIER_FUNCTION, InvocationType: 'Event', Payload: JSON.stringify(m) })); }, + }, ec2: { describeInstances: (i) => ec2c.send(new EC2.DescribeInstancesCommand(i)), createTags: (i) => ec2c.send(new EC2.CreateTagsCommand(i)), diff --git a/template.yaml b/template.yaml index b65386d..caf288c 100644 --- a/template.yaml +++ b/template.yaml @@ -192,6 +192,10 @@ Resources: - Effect: Allow Action: sqs:SendMessage Resource: !GetAtt JobQueue.Arn + # Tell the notifier a job succeeded, so it can clear a reclaim report. + - Effect: Allow + Action: lambda:InvokeFunction + Resource: !GetAtt NotifierFunction.Arn # Swap repair: find a runner's instance, and re-tag it with the # job it really runs. Only the job tags, only the stack's instances. - Effect: Allow @@ -239,6 +243,7 @@ Resources: Variables: SECRET_ARN: !Ref AppSecret QUEUE_URL: !Ref JobQueue + NOTIFIER_FUNCTION: !Ref NotifierFunction GITHUB_ORG: !Ref GitHubOrg STACK_NAME: !Ref AWS::StackName CLUSTER: !Ref RunnerCluster @@ -570,7 +575,7 @@ Resources: Runtime: nodejs22.x Handler: index.handler Role: !GetAtt NotifierRole.Arn - Timeout: 200 + Timeout: 30 MemorySize: 256 LoggingConfig: LogGroup: !Ref NotifierLogGroup diff --git a/test/notifier.test.js b/test/notifier.test.js index c099dae..1aeea19 100644 --- a/test/notifier.test.js +++ b/test/notifier.test.js @@ -80,7 +80,6 @@ function fakeGitHub({ checkStatus = 201, jobStatus = 200, existing = [], jobs = function setup({ inst = instance(TAGS), raw = { Reservations: inst ? [{ Instances: [inst] }] : [] }, tasks = [], gh = fakeGitHub(), creds = { app_id: 42, private_key: PEM } } = {}) { const described = []; const logs = []; - const slept = []; const handler = createHandler({ secret: { get: async () => creds }, fetch: gh.fetch, @@ -91,12 +90,11 @@ function setup({ inst = instance(TAGS), raw = { Reservations: inst ? [{ Instance return { tasks: tasks.filter((t) => i.tasks.includes(t.taskArn)), failures: [] }; }, }, - sleep: async (ms) => { slept.push(ms); }, now: () => T0, env: ENV, log: (l) => logs.push(l), }); - return { handler, gh, described, logs, slept }; + return { handler, gh, described, logs }; } test('an interruption of one of our instances creates a failed check run on the job\'s commit', async () => { @@ -106,7 +104,7 @@ test('an interruption of one of our instances creates a failed check run on the assert.deepEqual(gh.keys(), [ 'GET /repos/zeroae/app/installation', 'POST /app/installations/181/access_tokens', 'GET /repos/zeroae/app/actions/jobs/555', 'GET /repos/zeroae/app/commits/abc123/check-runs', 'POST /repos/zeroae/app/check-runs', - // The watch: the job failed too, so the check run stays. + // The re-read after posting: the job failed too, so the check run stays. 'GET /repos/zeroae/app/actions/jobs/555', ]); const { body } = gh.calls[4]; @@ -154,38 +152,28 @@ test('a swapped CI runner is reported for the job it took, not the one it was la assert.equal(body.name, 'ci-runner / tests'); }); -// AWS gives a 2-minute warning, and a job can finish inside it. The check run -// is posted at the warning, then the notifier watches the job. - -test('a job that finishes with success after the warning turns its check run neutral', async () => { - const gh = fakeGitHub({ jobs: [RUNNING, RUNNING, DONE('success')] }); - const { handler, slept, logs } = setup({ gh }); +// AWS gives a 2-minute warning, and a job can finish inside it. The webhook +// tells the notifier when a job succeeds. If that word came before the report +// existed, the notifier's own re-read of the job, right after posting, clears it. +test('a job that has finished with success by the time the report is posted turns its check run neutral', async () => { + const gh = fakeGitHub({ jobs: [RUNNING, DONE('success')] }); + const { handler, logs } = setup({ gh }); await handler(EVENT); const { body } = patchOf(gh); assert.equal(body.conclusion, 'neutral'); assert.equal(body.output.title, SURVIVED); assert.match(body.output.summary, /`tests`/); - assert.deepEqual(slept, [10000, 10000]); - assert.match(logs.join('\n'), /survived i-0123: zeroae\/app job 555/); + assert.match(logs.join('\n'), /survived: zeroae\/app tests/); }); -for (const conclusion of ['failure', 'cancelled']) { - test(`a job that completes with ${conclusion} after the warning keeps its failure`, async () => { - const gh = fakeGitHub({ jobs: [RUNNING, DONE(conclusion)] }); +for (const [why, job] of [['still runs', RUNNING], ['failed', DONE('failure')], ['was cancelled', DONE('cancelled')]]) { + test(`a job that ${why} after the report is posted keeps its failure`, async () => { + const gh = fakeGitHub({ jobs: [RUNNING, job] }); await setup({ gh }).handler(EVENT); assert.equal(patchOf(gh), undefined); }); } -test('a job still running when the watch ends keeps its failure', async () => { - const gh = fakeGitHub({ jobs: [RUNNING] }); - const { handler, slept } = setup({ gh }); - await handler(EVENT); - assert.equal(patchOf(gh), undefined); - assert.equal(slept.length, 15); - assert.equal(slept.reduce((a, b) => a + b), 150000); -}); - // A report an earlier invocation left standing (its watch ended, or a GitHub // call failed mid-watch and EventBridge retried). const STANDING = { id: 9, name: 'ci-runner / tests', external_id: '555', output: { title: RECLAIMED } }; @@ -196,7 +184,7 @@ test('a retry or a later event clears a standing report whose job has since fini await handler(EVENT); assert.equal(patchOf(gh).body.conclusion, 'neutral'); assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); - assert.match(logs.join('\n'), /survived i-0123/); + assert.match(logs.join('\n'), /survived: zeroae\/app tests/); }); for (const [why, job] of [['still runs', RUNNING], ['failed', DONE('failure')]]) { @@ -373,10 +361,56 @@ test('a swapped Fargate CI runner is reported for the job it took', async () => assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.external_id, '555'); }); -test('a Fargate job that finishes with success after the warning turns its check run neutral', async () => { +test('a Fargate job that has finished with success by the time the report is posted turns its check run neutral', async () => { const gh = fakeGitHub({ jobs: [RUNNING, { ...RUNNING, status: 'completed', conclusion: 'success' }] }); await setup({ inst: null, tasks: [task(TASK_TAGS)], gh }).handler(TASK_EVENT); const { body } = patchOf(gh); assert.equal(body.conclusion, 'neutral'); - assert.match(body.output.summary, /task \(0f1e2d3c4b5a69788796a5b4c3d2e1f0\)/); + assert.match(body.output.summary, /`build \(x64, a\)`/); +}); + +// --- the webhook: a job of this stack's runners succeeded ----------------------- +const SUCCEEDED = { jobSucceeded: { repo: 'zeroae/app', job_id: 555, run_id: 444, head_sha: 'abc123', job_name: 'tests', installation_id: 181 } }; + +test('a job that succeeded clears its standing reclaim report', async () => { + const gh = fakeGitHub({ existing: [STANDING] }); + const { handler, logs } = setup({ gh }); + await handler(SUCCEEDED); + // The webhook's installation id saves the installation lookup. + assert.deepEqual(gh.keys(), [ + 'POST /app/installations/181/access_tokens', + 'GET /repos/zeroae/app/commits/abc123/check-runs', + 'PATCH /repos/zeroae/app/check-runs/9', + ]); + const { body } = patchOf(gh); + assert.equal(body.conclusion, 'neutral'); + assert.equal(body.output.title, SURVIVED); + assert.ok(body.output.summary.includes('https://github.com/zeroae/app/actions/runs/444')); + assert.match(logs.join('\n'), /survived: zeroae\/app tests/); +}); + +for (const [why, existing] of [ + ['a job that never had a reclaim report', []], + ['a report that is already neutral', [{ ...STANDING, output: { title: SURVIVED } }]], + ['another job\'s report', [{ ...STANDING, external_id: '554' }]], + ['a check run of that name that isn\'t ours', [{ ...STANDING, output: { title: 'Lint' } }]], +]) { + test(`a job that succeeded changes nothing for ${why}`, async () => { + const gh = fakeGitHub({ existing }); + const { handler, logs } = setup({ gh }); + await handler(SUCCEEDED); + assert.equal(patchOf(gh), undefined); + assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); + assert.match(logs.join('\n'), /nothing to clear: zeroae\/app job 555/); + }); +} + +test('a job that succeeded, with no installation id, looks the installation up', async () => { + const gh = fakeGitHub({ existing: [STANDING] }); + await setup({ gh }).handler({ jobSucceeded: { ...SUCCEEDED.jobSucceeded, installation_id: undefined } }); + assert.equal(gh.keys()[0], 'GET /repos/zeroae/app/installation'); +}); + +test('before the App exists a job that succeeded fails the invocation', async () => { + await assert.rejects(setup({ creds: { state_key: 'x' } }).handler(SUCCEEDED), /not set up/); }); diff --git a/test/tasks-ecs.test.js b/test/tasks-ecs.test.js index dcb933b..679965d 100644 --- a/test/tasks-ecs.test.js +++ b/test/tasks-ecs.test.js @@ -218,7 +218,7 @@ test('a Fargate Spot reclaim: template.yaml\'s rule matches each change of the s // The task's runner is running job 999 (the notifier reports only a running job). const gh = fakeGitHub([], { status: 'in_progress', runner_name: runner }); const logs = []; - const h = notifier.createHandler({ secret, fetch: gh.fetch, sleep: async () => {}, ec2: { describe: assert.fail }, ecs, now: Date.now, env, log: (l) => logs.push(l) }); + const h = notifier.createHandler({ secret, fetch: gh.fetch, ec2: { describe: assert.fail }, ecs, now: Date.now, env, log: (l) => logs.push(l) }); await h(warning); const check = gh.calls.find((c) => c.key === 'POST /repos/zeroae/lib/check-runs'); assert.ok(check, logs.join('\n')); diff --git a/test/template.test.js b/test/template.test.js index f018031..11e188f 100644 --- a/test/template.test.js +++ b/test/template.test.js @@ -169,13 +169,6 @@ test('a second notifier rule matches every change of a Fargate Spot reclaim in t assert.match(perm, /SourceArn: !GetAtt NotifierTaskRule\.Arn\n/); }); -test('the notifier\'s timeout outlasts its watch of a job after a Spot warning', () => { - const src = fs.readFileSync(path.join(__dirname, '..', 'functions', 'notifier.js'), 'utf8'); - const watchMs = Number(src.match(/^const WATCH_INTERVAL_MS = (\d+);/m)[1]) * Number(src.match(/^const WATCH_LOOKS = (\d+);/m)[1]); - const timeout = Number(resource('NotifierFunction').match(/\n {6}Timeout: (\d+)\n/)[1]); - assert.ok(timeout * 1000 >= watchMs + 30000, `${timeout} s is too short for a ${watchMs / 1000} s watch plus the calls around it`); -}); - test('the notifier may describe only the cluster\'s tasks, and nothing else on ECS', () => { const ecs = statements(resource('NotifierRole')).filter((s) => /ecs:/.test(s)); assert.deepEqual(ecs, [ diff --git a/test/webhook.test.js b/test/webhook.test.js index e910f95..4f7da40 100644 --- a/test/webhook.test.js +++ b/test/webhook.test.js @@ -11,6 +11,7 @@ const SECRET = 'whsec-test'; // tasks: the cluster's tasks, as { taskArn, desiredStatus, tags: { key: value } }. function setup({ secretValue = { webhook_secret: SECRET }, instances = [], tasks = [] } = {}) { const sent = []; + const notified = []; const described = []; const tagged = []; const taskTagged = []; @@ -19,6 +20,7 @@ function setup({ secretValue = { webhook_secret: SECRET }, instances = [], tasks const handler = createHandler({ secret: { get: async () => { reads++; return secretValue; } }, queue: { send: async (msg) => { sent.push(msg); } }, + notifier: { send: async (msg) => { notified.push(msg); } }, ec2: { describeInstances: async (input) => { described.push(input); @@ -51,16 +53,17 @@ function setup({ secretValue = { webhook_secret: SECRET }, instances = [], tasks env: { GITHUB_ORG: 'zeroae', STACK_NAME: 'ci-runner', CLUSTER: 'ci-runner' }, log: (line) => logs.push(JSON.parse(line)), }); - return { handler, sent, described, tagged, taskTagged, logs, reads: () => reads }; + return { handler, sent, notified, described, tagged, taskTagged, logs, reads: () => reads }; } // A workflow_job payload as GitHub sends it (only the fields that matter). -function job({ action = 'queued', labels = ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64'], visibility = 'private', owner = 'zeroae', runner } = {}) { +function job({ action = 'queued', labels = ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64'], visibility = 'private', owner = 'zeroae', runner, conclusion } = {}) { return { action, workflow_job: { id: 555, run_id: 444, head_sha: 'abc123', name: 'tests', labels, ...(runner && { runner_name: runner, runner_id: 77 }), + ...(conclusion && { conclusion }), }, repository: { full_name: `${owner}/app`, name: 'app', visibility, private: visibility !== 'public', owner: { login: owner } }, installation: { id: 181 }, @@ -241,6 +244,30 @@ test('other actions and other events are ignored', async () => { // A runner the launcher started for job 999 of run 888, and its instance. const RUNNER = 'ci-runner-20261113120000-999'; + +// The notifier is told when a job of this stack's runners finishes with +// success, so it can clear a reclaim report for a job that survived the warning. +test('a job that completes with success on one of our runners is passed to the notifier', async () => { + const { handler, notified, sent } = setup(); + const res = await handler(delivery(job({ action: 'completed', runner: RUNNER, conclusion: 'success' }))); + assert.equal(res.statusCode, 202); + assert.deepEqual(notified, [{ jobSucceeded: { repo: 'zeroae/app', job_id: 555, run_id: 444, head_sha: 'abc123', job_name: 'tests', installation_id: 181 } }]); + assert.equal(sent.length, 0); +}); + +for (const [why, opts] of [ + ['a job that failed', { runner: RUNNER, conclusion: 'failure' }], + ['a job that was cancelled', { runner: RUNNER, conclusion: 'cancelled' }], + ['a job on a GitHub-hosted runner', { runner: 'GitHub Actions 12', conclusion: 'success' }], + ['a job on another stack\'s runner', { runner: 'other-20261113120000-999', conclusion: 'success' }], + ['a job that never had a runner', { conclusion: 'success' }], +]) { + test(`${why} is not passed to the notifier`, async () => { + const { handler, notified } = setup(); + assert.equal((await handler(delivery(job({ action: 'completed', ...opts })))).statusCode, 200); + assert.deepEqual(notified, []); + }); +} const instance = (tags = {}) => ({ InstanceId: 'i-0abc', Tags: { From 693cb3f476b74f94b131871f5060dee9462ce4c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sat, 14 Nov 2026 18:41:16 -0500 Subject: [PATCH 4/5] Review fixes: a re-run clears its earlier attempt's report; GitHub's job name everywhere (#44) A re-run's job has a new id, so its first attempt's reclaim report stayed failure after the re-run passed. The webhook's word now clears every reclaim failure under the job's run with the job's name. The notifier names the check run from GitHub's job name for EC2 too, so a tag cut at 256 characters cannot miss the webhook's word. One launchJobOf() reads a runner's name in the webhook. stack.md lists the webhook's new grant and the failure path. Co-Authored-By: Claude Sonnet 5.5 --- docs/runbook.md | 3 ++- docs/stack.md | 22 ++++++++++++++-------- functions/notifier.js | 27 ++++++++++++++++----------- functions/webhook.js | 14 ++++++++++---- test/notifier.test.js | 37 +++++++++++++++++++++++++++---------- 5 files changed, 69 insertions(+), 34 deletions(-) diff --git a/docs/runbook.md b/docs/runbook.md index 6882bc1..ec7ed1d 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -299,7 +299,8 @@ AWS warns 2 minutes before it takes the machine, and a job can finish inside that. When the job ends with `success`, the check run turns `neutral`, titled `The job finished despite the Spot reclaim warning`: no re-run is needed (the webhook tells the notifier; see `docs/stack.md`, "Reporting a Spot reclaim"). -A `failure` check run whose job passed anyway is safe to ignore. The notifier reports +A `failure` check run whose job passed anyway is safe to ignore. A re-run that +passes turns the first attempt's report `neutral` too. The notifier reports only a job the machine's CI runner was running: a CI runner that took no job (another one served its launch job) is not reported. diff --git a/docs/stack.md b/docs/stack.md index 10c8623..4645b2c 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -266,7 +266,7 @@ tags and repairs again. Re-queueing a job that another runner already took is harmless: the launcher drops it. Its role allows `GetSecretValue` on the secret, `SendMessage` on the queue, -`ec2:DescribeInstances`, and `ec2:CreateTags` only on instances tagged +`lambda:InvokeFunction` on the notifier, `ec2:DescribeInstances`, and `ec2:CreateTags` only on instances tagged `ci-runner:stack=` and only for the five job tag keys. On ECS: `ecs:ListTasks` with the condition `ecs:cluster` = the cluster, and `ecs:DescribeTasks` and `ecs:TagResource` only on the cluster's tasks, the @@ -493,8 +493,9 @@ For each event it: and so its tags are still the launch tags. A swapped job that the webhook has not yet re-tagged, in the moments after it started, is missed the same way. -4. For a task, takes the job's real name from that job (the tag has `_` for - brackets and commas), falling back to the tag. +4. Takes the job's real name from that job, falling back to the tag: a tag is + cut at 256 characters, and a task's has `_` for brackets and commas. The + webhook's word (step 8) carries GitHub's name too, so the two meet. 5. Skips the job if the commit already has its reclaim check run (same name, `external_id` the job id, the title below or the survived title), so a task's later changes and a repeated event report nothing more. @@ -510,17 +511,22 @@ For each event it: the machine, and a job can finish inside that. A job that finishes with `success` was not lost, so its reclaim check run changes to `neutral`, titled `The job finished despite the Spot reclaim warning`, and the notifier - logs `survived: `. Three things get a report there: + logs `survived: `. Three paths clear a report: - **the webhook's word** (above): the notifier gets the token (the event - carries the installation id), finds the job's reclaim check run, and - clears it. A job with no reclaim report logs `nothing to clear` and - changes nothing. The check run's name uses the event's job name; + carries the installation id), lists the commit's check runs named `ci-runner + / `, and clears each reclaim `failure` whose `details_url` is + under the job's run. That takes in the report of an earlier attempt of the + job: a re-run's job has a new id, but the same run. With none to clear it + logs `nothing to clear` and changes nothing; - **a retry or a later change** of a reclaim event that finds its own standing `failure` (step 5) for a job that has since finished with `success`; - **step 7**, for the word that came before the report existed. - A job that ends with any other conclusion keeps the `failure`. + A job that ends with any other conclusion keeps the `failure`. If the + webhook's invoke of the notifier fails, the delivery fails (`502`): redeliver + it from the App's advanced settings, or the retry and later-change paths + clear the report. A GitHub failure fails the invocation, and EventBridge's async invoke retries it. Two events handled at the same moment can still report twice. diff --git a/functions/notifier.js b/functions/notifier.js index beb127d..8b43714 100644 --- a/functions/notifier.js +++ b/functions/notifier.js @@ -46,12 +46,12 @@ function createHandler({ secret, fetch, ec2, ecs, now, env, log = console.log }) return (await gh('POST', `/app/installations/${installation}/access_tokens`, jwt)).token; } - // The commit's reclaim check run for a job, if it has one. A task's reclaim + // The commit's reclaim check runs under a job's name. A task's reclaim // matches on each of its changes, and EventBridge may repeat or retry an // event: one check run per job is enough. - async function reportedFor(repo, token, sha, name, jobId) { + async function reportsNamed(repo, token, sha, name) { const { check_runs: existing } = await gh('GET', `/repos/${repo}/commits/${sha}/check-runs?filter=all&check_name=${encodeURIComponent(name)}`, token); - return existing.find((c) => c.external_id === jobId && c.output && [RECLAIMED_TITLE, SURVIVED_TITLE].includes(c.output.title)); + return existing.filter((c) => c.output && [RECLAIMED_TITLE, SURVIVED_TITLE].includes(c.output.title)); } // A job that finished with success was not lost: its check run no longer @@ -66,12 +66,16 @@ function createHandler({ secret, fetch, ec2, ecs, now, env, log = console.log }) // The webhook says a job of this stack's runners finished with success. A // job that had a reclaim report (AWS warns 2 minutes before it takes the - // machine, and a job can finish inside that) is cleared. + // machine, and a job can finish inside that) is cleared. So is the report + // of an earlier attempt of the job, which a re-run of the run left behind: + // its details_url is under the same run. async function jobSucceeded(j) { const token = await tokenFor(j.repo, j.installation_id); - const reported = await reportedFor(j.repo, token, j.head_sha, `ci-runner / ${j.job_name}`, String(j.job_id)); - if (!reported || reported.output.title !== RECLAIMED_TITLE) return log(`nothing to clear: ${j.repo} job ${j.job_id}`); - return clear(j.repo, token, reported.id, j.job_name, `https://github.com/${j.repo}/actions/runs/${j.run_id}`); + const run = `https://github.com/${j.repo}/actions/runs/${j.run_id}`; + const standing = (await reportsNamed(j.repo, token, j.head_sha, `ci-runner / ${j.job_name}`)) + .filter((c) => c.output.title === RECLAIMED_TITLE && (c.details_url || '').startsWith(`${run}/`)); + if (!standing.length) return log(`nothing to clear: ${j.repo} job ${j.job_id}`); + for (const c of standing) await clear(j.repo, token, c.id, j.job_name, run); } return async (event) => { @@ -99,14 +103,15 @@ function createHandler({ secret, fetch, ec2, ecs, now, env, log = console.log }) const token = await tokenFor(repo); const jobPath = `/repos/${repo}/actions/jobs/${jobId}`; const ghJob = await gh('GET', jobPath, token); - // A task's job-name tag has `_` for what ECS refuses (brackets, commas): - // the check run carries the real name, from GitHub. - const job = (d.taskArn && ghJob.name) || t['job-name'] || `job ${jobId}`; + // The check run carries the job's real name, from GitHub, as the webhook's + // word does: a task's job-name tag has `_` for what ECS refuses (brackets, + // commas), and every tag is cut at 256 characters. + const job = ghJob.name || t['job-name'] || `job ${jobId}`; const run = `https://github.com/${repo}/actions/runs/${t['run-id']}`; const name = `ci-runner / ${job}`; const finishedWithSuccess = (j) => j.status === 'completed' && j.conclusion === 'success'; - const reported = await reportedFor(repo, token, t['head-sha'], name, jobId); + const reported = (await reportsNamed(repo, token, t['head-sha'], name)).find((c) => c.external_id === jobId); if (reported) { // An earlier event's report, left standing: the job finished with // success after the webhook's word, or before the report existed. diff --git a/functions/webhook.js b/functions/webhook.js index e793fb5..c79a662 100644 --- a/functions/webhook.js +++ b/functions/webhook.js @@ -75,14 +75,21 @@ function createHandler({ secret, queue, notifier, ec2, ecs, env, log = console.l let key; // the webhook secret, cached once the App exists + // The job id a runner of this stack was launched for, from its name + // `${stack}-<14-digit stamp>-`; null for any other runner. + function launchJobOf(runnerName) { + const m = String(runnerName || '').match(/^(.+)-\d{14}-(\d+)$/); + return m && m[1] === env.STACK_NAME ? Number(m[2]) : null; + } + // A swap: a runner of this stack, named `${stack}--`, started // a job other than the one it was launched for. Re-queue that one, so the // launcher's "still queued?" check launches it a runner or drops it, and // re-tag the instance or task so the notifier names the job it runs. async function repairSwap(p, repo, job) { const name = String(job.runner_name || ''); - const m = name.match(/^(.+)-\d{14}-(\d+)$/); - if (!m || m[1] !== env.STACK_NAME || Number(m[2]) === job.id) return reply(200, 'ignored: no swap'); + const launchJob = launchJobOf(name); + if (launchJob === null || launchJob === job.id) return reply(200, 'ignored: no swap'); const r = await ec2.describeInstances({ Filters: [ { Name: 'tag:ci-runner:stack', Values: [env.STACK_NAME] }, @@ -122,8 +129,7 @@ function createHandler({ secret, queue, notifier, ec2, ecs, env, log = console.l // clears its reclaim report for the job, if the job had one: AWS warns 2 // minutes before it takes a Spot machine, and a job can finish inside that. async function reportSuccess(p, repo, job) { - const m = String(job.runner_name || '').match(/^(.+)-\d{14}-\d+$/); - if (job.conclusion !== 'success' || !m || m[1] !== env.STACK_NAME) return reply(200, 'ignored: not a success of ours'); + if (job.conclusion !== 'success' || launchJobOf(job.runner_name) === null) return reply(200, 'ignored: not a success of ours'); await notifier.send({ jobSucceeded: { repo: repo.full_name, job_id: job.id, run_id: job.run_id, head_sha: job.head_sha, job_name: job.name, installation_id: p.installation && p.installation.id, diff --git a/test/notifier.test.js b/test/notifier.test.js index 1aeea19..1d2c691 100644 --- a/test/notifier.test.js +++ b/test/notifier.test.js @@ -33,7 +33,8 @@ function verifyJwt(auth) { // The job the tags name, as GitHub reports it when this machine's CI runner // is running it. -const RUNNING = { name: 'build (x64, a)', status: 'in_progress', runner_name: 'ci-runner-20261112115500-555' }; +const RUNNING = { name: 'tests', status: 'in_progress', runner_name: 'ci-runner-20261112115500-555' }; +const TASK_JOB = { ...RUNNING, name: 'build (x64, a)' }; const RECLAIMED = 'Spot capacity reclaimed: re-run this job'; const SURVIVED = 'The job finished despite the Spot reclaim warning'; @@ -174,9 +175,10 @@ for (const [why, job] of [['still runs', RUNNING], ['failed', DONE('failure')], }); } -// A report an earlier invocation left standing (its watch ended, or a GitHub -// call failed mid-watch and EventBridge retried). -const STANDING = { id: 9, name: 'ci-runner / tests', external_id: '555', output: { title: RECLAIMED } }; +// A report an earlier event left standing: the job finished with success after +// the webhook's word, or before the report existed. +const RUN_URL = 'https://github.com/zeroae/app/actions/runs/444'; +const STANDING = { id: 9, name: 'ci-runner / tests', external_id: '555', details_url: `${RUN_URL}/job/555`, output: { title: RECLAIMED } }; test('a retry or a later event clears a standing report whose job has since finished with success', async () => { const gh = fakeGitHub({ existing: [STANDING], jobs: [DONE('success')] }); @@ -213,8 +215,14 @@ test('a reclaim whose check run turned neutral is not reported again', async () assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); }); -test('a job name falls back to the job id when the tag is empty', async () => { - const { handler, gh } = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': '' }) }); +test('a job\'s name is GitHub\'s, which the tag may have cut or changed', async () => { + const { handler, gh } = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': 'cut at 256' }), gh: fakeGitHub({ jobs: [{ ...RUNNING, name: 'the whole name' }] }) }); + await handler(EVENT); + assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.name, 'ci-runner / the whole name'); +}); + +test('a job name falls back to the tag, then to the job id, when GitHub gives none', async () => { + const { handler, gh } = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': '' }), gh: fakeGitHub({ jobs: [{ ...RUNNING, name: undefined }] }) }); await handler(EVENT); assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.name, 'ci-runner / job 555'); }); @@ -302,9 +310,10 @@ const TASK_TAGS = { const task = (tags) => ({ taskArn: TASK_ARN, lastStatus: 'RUNNING', tags: Object.entries(tags).map(([key, value]) => ({ key, value })) }); test('a Fargate Spot reclaim creates the same check run as an EC2 one, with the job\'s real name', async () => { - const ec2 = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': 'build (x64, a)' }) }); + const github = () => fakeGitHub({ jobs: [TASK_JOB] }); + const ec2 = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': 'build (x64, a)' }), gh: github() }); await ec2.handler(EVENT); - const fargate = setup({ inst: null, tasks: [task(TASK_TAGS)] }); + const fargate = setup({ inst: null, tasks: [task(TASK_TAGS)], gh: github() }); await fargate.handler(TASK_EVENT); assert.deepEqual(fargate.described, [{ cluster: CLUSTER_ARN, tasks: [TASK_ARN], include: ['TAGS'] }]); @@ -362,7 +371,7 @@ test('a swapped Fargate CI runner is reported for the job it took', async () => }); test('a Fargate job that has finished with success by the time the report is posted turns its check run neutral', async () => { - const gh = fakeGitHub({ jobs: [RUNNING, { ...RUNNING, status: 'completed', conclusion: 'success' }] }); + const gh = fakeGitHub({ jobs: [TASK_JOB, { ...TASK_JOB, status: 'completed', conclusion: 'success' }] }); await setup({ inst: null, tasks: [task(TASK_TAGS)], gh }).handler(TASK_EVENT); const { body } = patchOf(gh); assert.equal(body.conclusion, 'neutral'); @@ -392,7 +401,7 @@ test('a job that succeeded clears its standing reclaim report', async () => { for (const [why, existing] of [ ['a job that never had a reclaim report', []], ['a report that is already neutral', [{ ...STANDING, output: { title: SURVIVED } }]], - ['another job\'s report', [{ ...STANDING, external_id: '554' }]], + ['a report of another run\'s job of that name', [{ ...STANDING, details_url: 'https://github.com/zeroae/app/actions/runs/443/job/554' }]], ['a check run of that name that isn\'t ours', [{ ...STANDING, output: { title: 'Lint' } }]], ]) { test(`a job that succeeded changes nothing for ${why}`, async () => { @@ -405,6 +414,14 @@ for (const [why, existing] of [ }); } +test('a job that succeeded in a re-run clears the report of its earlier attempt', async () => { + // The re-run attempt's job has a new id (555); the report names the first attempt's job (500) of the same run. + const earlier = { ...STANDING, external_id: '500', details_url: `${RUN_URL}/job/500` }; + const gh = fakeGitHub({ existing: [earlier] }); + await setup({ gh }).handler(SUCCEEDED); + assert.equal(patchOf(gh).body.conclusion, 'neutral'); +}); + test('a job that succeeded, with no installation id, looks the installation up', async () => { const gh = fakeGitHub({ existing: [STANDING] }); await setup({ gh }).handler({ jobSucceeded: { ...SUCCEEDED.jobSucceeded, installation_id: undefined } }); From e9abe861b6406450ae5b3d0f53b3834c4f7ef332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= Date: Sat, 14 Nov 2026 18:46:26 -0500 Subject: [PATCH 5/5] Notifier fences a success word to the org; test the webhook's grant on the notifier (#44) jobSucceeded now ignores a repo outside GITHUB_ORG before any GitHub call, as a reclaim event does. A template test pins the webhook's lambda:InvokeFunction to the notifier alone, and NOTIFIER_FUNCTION on the webhook. Co-Authored-By: Claude Sonnet 5.5 --- functions/notifier.js | 2 ++ test/notifier.test.js | 8 ++++++++ test/template.test.js | 7 +++++++ 3 files changed, 17 insertions(+) diff --git a/functions/notifier.js b/functions/notifier.js index 8b43714..ba1dcfc 100644 --- a/functions/notifier.js +++ b/functions/notifier.js @@ -70,6 +70,8 @@ function createHandler({ secret, fetch, ec2, ecs, now, env, log = console.log }) // of an earlier attempt of the job, which a re-run of the run left behind: // its details_url is under the same run. async function jobSucceeded(j) { + // The same fence as a reclaim event's: only the org's repos. + if (String(j.repo).split('/')[0] !== env.GITHUB_ORG) return log(`ignored: ${j.repo} is outside ${env.GITHUB_ORG}`); const token = await tokenFor(j.repo, j.installation_id); const run = `https://github.com/${j.repo}/actions/runs/${j.run_id}`; const standing = (await reportsNamed(j.repo, token, j.head_sha, `ci-runner / ${j.job_name}`)) diff --git a/test/notifier.test.js b/test/notifier.test.js index 1d2c691..046ac2a 100644 --- a/test/notifier.test.js +++ b/test/notifier.test.js @@ -422,6 +422,14 @@ test('a job that succeeded in a re-run clears the report of its earlier attempt' assert.equal(patchOf(gh).body.conclusion, 'neutral'); }); +test('a job that succeeded in a repo outside the org is ignored without a GitHub call', async () => { + const gh = fakeGitHub({ existing: [STANDING] }); + const { handler, logs } = setup({ gh }); + await handler({ jobSucceeded: { ...SUCCEEDED.jobSucceeded, repo: 'elsewhere/app' } }); + assert.deepEqual(gh.keys(), []); + assert.match(logs.join('\n'), /ignored: elsewhere\/app is outside zeroae/); +}); + test('a job that succeeded, with no installation id, looks the installation up', async () => { const gh = fakeGitHub({ existing: [STANDING] }); await setup({ gh }).handler({ jobSucceeded: { ...SUCCEEDED.jobSucceeded, installation_id: undefined } }); diff --git a/test/template.test.js b/test/template.test.js index 11e188f..1f894ad 100644 --- a/test/template.test.js +++ b/test/template.test.js @@ -169,6 +169,13 @@ test('a second notifier rule matches every change of a Fargate Spot reclaim in t assert.match(perm, /SourceArn: !GetAtt NotifierTaskRule\.Arn\n/); }); +test('the webhook may invoke only the notifier, and knows its name', () => { + const lambda = statements(resource('WebhookRole')).filter((s) => /lambda:/.test(s)); + assert.equal(lambda.length, 1); + assert.match(lambda[0], /^Allow\n {16}Action: lambda:InvokeFunction\n {16}Resource: !GetAtt NotifierFunction\.Arn\n/); + assert.match(resource('WebhookFunction'), /\n {10}NOTIFIER_FUNCTION: !Ref NotifierFunction\n/); +}); + test('the notifier may describe only the cluster\'s tasks, and nothing else on ECS', () => { const ecs = statements(resource('NotifierRole')).filter((s) => /ecs:/.test(s)); assert.deepEqual(ecs, [