diff --git a/evals/skilld-nikita-reference/case.yaml b/evals/skilld-nikita-reference/case.yaml index dfd0ca5a..631863b3 100644 --- a/evals/skilld-nikita-reference/case.yaml +++ b/evals/skilld-nikita-reference/case.yaml @@ -7,7 +7,7 @@ execution: max_turns: 15 allowed_tools: [Read, Glob, Grep, Skill, Bash] prompt: | - Use the Agent Skill `heyimjames/nikita-bier-consumer-apps/nikita-bier-consumer-apps` + Use the Agent Skill `github:heyimjames/nikita-bier-consumer-apps/.#commit:f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0` once for a source lookup. Do not install it. Which four alternatives to full contact access does its `onboarding.md` diff --git a/evals/skilld-nikita-reference/graders/loaded-skill.md b/evals/skilld-nikita-reference/graders/loaded-skill.md index 1a3f930c..230dc98b 100644 --- a/evals/skilld-nikita-reference/graders/loaded-skill.md +++ b/evals/skilld-nikita-reference/graders/loaded-skill.md @@ -1,7 +1,7 @@ --- type: tool_used tool: Bash -input_match: "skilld run heyimjames/nikita-bier-consumer-apps/nikita-bier-consumer-apps --json" +input_match: 'skilld run [^\n]*github:heyimjames/nikita-bier-consumer-apps/\.#commit:f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0[^\n]*--json' min: 1 weight: 2 --- diff --git a/evals/skilld-nikita-reference/graders/pinned-reference.md b/evals/skilld-nikita-reference/graders/pinned-reference.md index 2187a1f9..3cbd9719 100644 --- a/evals/skilld-nikita-reference/graders/pinned-reference.md +++ b/evals/skilld-nikita-reference/graders/pinned-reference.md @@ -1,7 +1,7 @@ --- type: tool_used tool: Bash -input_match: 'skilld run [^\n]*heyimjames/nikita-bier-consumer-apps/[^\n]*--revision [0-9a-f]{40}[^\n]*--file[= ]onboarding\.md[^\n]*--json' +input_match: 'skilld run [^\n]*heyimjames/nikita-bier-consumer-apps/[^\n]*--revision f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0[^\n]*--file[= ]onboarding\.md[^\n]*--json' min: 1 weight: 3 --- diff --git a/evals/skilld-nikita-reference/graders/revision-reported.md b/evals/skilld-nikita-reference/graders/revision-reported.md index 5b5c539d..663ed45d 100644 --- a/evals/skilld-nikita-reference/graders/revision-reported.md +++ b/evals/skilld-nikita-reference/graders/revision-reported.md @@ -1,6 +1,6 @@ --- type: regex target: last_message -pattern: "[0-9a-f]{40}" +pattern: 'f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0' match: contains --- diff --git a/evals/skilld-nikita-reference/graders/same-commit.md b/evals/skilld-nikita-reference/graders/same-commit.md deleted file mode 100644 index 8f01705c..00000000 --- a/evals/skilld-nikita-reference/graders/same-commit.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: regex -target: trace -pattern: '"--revision","([0-9a-f]{40})"[\s\S]*?\[tool bash\][^\n]*--revision \1[^\n]*--file[= ]onboarding\.md' -match: contains -weight: 2 ---- diff --git a/tests/evals/nikita-reference.test.ts b/tests/evals/nikita-reference.test.ts new file mode 100644 index 00000000..655f855f --- /dev/null +++ b/tests/evals/nikita-reference.test.ts @@ -0,0 +1,41 @@ +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { loadCase } from '../../scripts/eval-opencode/case.ts' +import { gradeRun, runScore } from '../../scripts/eval-opencode/grade.ts' +import { parseTranscript } from '../../scripts/eval-opencode/transcript.ts' + +const commit = 'f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0' +const selector = 'heyimjames/nikita-bier-consumer-apps/nikita-bier-consumer-apps' + +function grade(referenceCommit = commit, skillCommit = commit) { + const loaded = loadCase(fileURLToPath(new URL('../../evals/skilld-nikita-reference', import.meta.url))) + if (loaded._tag === 'Err') + throw new Error(loaded.message) + const tool = (command: string, output: string) => JSON.stringify({ + type: 'tool_use', + part: { tool: 'bash', state: { input: { command }, output, status: 'completed', metadata: { exit: 0 } } }, + }) + const transcript = parseTranscript([ + tool(`skilld run 'github:heyimjames/nikita-bier-consumer-apps/.#commit:${skillCommit}' --json`, + `{"readArgv":["--revision","${skillCommit}"]}${'long supporting content'.repeat(300)}`), + tool(`skilld run ${selector} --revision ${referenceCommit} --file=onboarding.md --json`, 'reference'), + JSON.stringify({ type: 'text', part: { text: `Community codes, QR codes, deep links, phone numbers. ${commit}` } }), + ].join('\n')) + return gradeRun(loaded.value.graders, { transcript, readProjectFile: () => null, verdicts: new Map(), arm: 'without' }) +} + +describe('Nikita reference eval', () => { + it('accepts the pinned source workflow when tool output exceeds the trace limit', () => { + expect(runScore(grade())).toBe(1) + }) + + it('rejects a reference from another commit', () => { + expect(grade('0'.repeat(40)).find(result => result.name === 'pinned-reference')?.outcome) + .toMatchObject({ _tag: 'Scored', score: 0 }) + }) + + it('rejects a Skill loaded from another commit', () => { + expect(grade(commit, '0'.repeat(40)).find(result => result.name === 'loaded-skill')?.outcome) + .toMatchObject({ _tag: 'Scored', score: 0 }) + }) +})