test(evals): pin the Nikita source reference workflow

evals/skilld-nikita-reference/case.yaml

@@ -7,7 +7,7 @@ execution:
   max_turns: 15
   allowed_tools: [Read, Glob, Grep, Skill, Bash]
   prompt: |
-    Use the Agent Skill `heyimjames/nikita-bier-consumer-apps/nikita-bier-consumer-apps`
+    Use the Agent Skill `github:heyimjames/nikita-bier-consumer-apps/.#commit:f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0`
     once for a source lookup. Do not install it.
 
     Which four alternatives to full contact access does its `onboarding.md`

evals/skilld-nikita-reference/graders/loaded-skill.md

@@ -1,7 +1,7 @@
 ---
 type: tool_used
 tool: Bash
-input_match: "skilld run heyimjames/nikita-bier-consumer-apps/nikita-bier-consumer-apps --json"
+input_match: 'skilld run [^\n]*github:heyimjames/nikita-bier-consumer-apps/\.#commit:f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0[^\n]*--json'
 min: 1
 weight: 2
 ---

evals/skilld-nikita-reference/graders/pinned-reference.md

@@ -1,7 +1,7 @@
 ---
 type: tool_used
 tool: Bash
-input_match: 'skilld run [^\n]*heyimjames/nikita-bier-consumer-apps/[^\n]*--revision [0-9a-f]{40}[^\n]*--file[= ]onboarding\.md[^\n]*--json'
+input_match: 'skilld run [^\n]*heyimjames/nikita-bier-consumer-apps/[^\n]*--revision f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0[^\n]*--file[= ]onboarding\.md[^\n]*--json'
 min: 1
 weight: 3
 ---

evals/skilld-nikita-reference/graders/revision-reported.md

@@ -1,6 +1,6 @@
 ---
 type: regex
 target: last_message
-pattern: "[0-9a-f]{40}"
+pattern: 'f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0'
 match: contains
 ---

tests/evals/nikita-reference.test.ts

@@ -0,0 +1,41 @@
+import { fileURLToPath } from 'node:url'
+import { describe, expect, it } from 'vitest'
+import { loadCase } from '../../scripts/eval-opencode/case.ts'
+import { gradeRun, runScore } from '../../scripts/eval-opencode/grade.ts'
+import { parseTranscript } from '../../scripts/eval-opencode/transcript.ts'
+
+const commit = 'f8b8a1828cd53e4aedaa39c3e2abb01e71660bc0'
+const selector = 'heyimjames/nikita-bier-consumer-apps/nikita-bier-consumer-apps'
+
+function grade(referenceCommit = commit, skillCommit = commit) {
+  const loaded = loadCase(fileURLToPath(new URL('../../evals/skilld-nikita-reference', import.meta.url)))
+  if (loaded._tag === 'Err')
+    throw new Error(loaded.message)
+  const tool = (command: string, output: string) => JSON.stringify({
+    type: 'tool_use',
+    part: { tool: 'bash', state: { input: { command }, output, status: 'completed', metadata: { exit: 0 } } },
+  })
+  const transcript = parseTranscript([
+    tool(`skilld run 'github:heyimjames/nikita-bier-consumer-apps/.#commit:${skillCommit}' --json`,
+      `{"readArgv":["--revision","${skillCommit}"]}${'long supporting content'.repeat(300)}`),
+    tool(`skilld run ${selector} --revision ${referenceCommit} --file=onboarding.md --json`, 'reference'),
+    JSON.stringify({ type: 'text', part: { text: `Community codes, QR codes, deep links, phone numbers. ${commit}` } }),
+  ].join('\n'))
+  return gradeRun(loaded.value.graders, { transcript, readProjectFile: () => null, verdicts: new Map(), arm: 'without' })
+}
+
+describe('Nikita reference eval', () => {
+  it('accepts the pinned source workflow when tool output exceeds the trace limit', () => {
+    expect(runScore(grade())).toBe(1)
+  })
+
+  it('rejects a reference from another commit', () => {
+    expect(grade('0'.repeat(40)).find(result => result.name === 'pinned-reference')?.outcome)
+      .toMatchObject({ _tag: 'Scored', score: 0 })
+  })
+
+  it('rejects a Skill loaded from another commit', () => {
+    expect(grade(commit, '0'.repeat(40)).find(result => result.name === 'loaded-skill')?.outcome)
+      .toMatchObject({ _tag: 'Scored', score: 0 })
+  })
+})