From 67316847de3a328255d4a923203730a5bf0c7c12 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Thu, 8 Oct 2026 22:25:47 +1100 Subject: [PATCH] test(evals): cover pinned root Skill reference reads --- evals/README.md | 1 + evals/skilld-nikita-reference/case.yaml | 18 ++++++++++++++++++ .../graders/installed-nothing.md | 7 +++++++ .../graders/loaded-skill.md | 7 +++++++ .../graders/no-lockfile.md | 6 ++++++ .../graders/pinned-reference.md | 7 +++++++ .../graders/reference-answer.md | 8 ++++++++ .../graders/revision-reported.md | 6 ++++++ .../graders/same-commit.md | 7 +++++++ .../graders/skill-fired.md | 6 ++++++ 10 files changed, 73 insertions(+) create mode 100644 evals/skilld-nikita-reference/case.yaml create mode 100644 evals/skilld-nikita-reference/graders/installed-nothing.md create mode 100644 evals/skilld-nikita-reference/graders/loaded-skill.md create mode 100644 evals/skilld-nikita-reference/graders/no-lockfile.md create mode 100644 evals/skilld-nikita-reference/graders/pinned-reference.md create mode 100644 evals/skilld-nikita-reference/graders/reference-answer.md create mode 100644 evals/skilld-nikita-reference/graders/revision-reported.md create mode 100644 evals/skilld-nikita-reference/graders/same-commit.md create mode 100644 evals/skilld-nikita-reference/graders/skill-fired.md diff --git a/evals/README.md b/evals/README.md index 538aa173..0f8362f7 100644 --- a/evals/README.md +++ b/evals/README.md @@ -98,6 +98,7 @@ The `skilld` Skill cases check the CLI workflow it teaches: - `skilld-missing-ref`: a missing Skill reports the skilld error code and invents nothing. - `skilld-trending`: a trending question reads `skilld trending --json` and reports why each Skill trends. - `skilld-provenance`: a provenance question reads `skilld view` and names the exact SKILL.md. +- `skilld-nikita-reference`: a root Skill reference uses the loaded commit and installs nothing. - `evals-opencode/mcp-find-skill`: with the skilld MCP server connected, the Agent finds a Skill with `search_skills` and returns a run command. - `evals-opencode/mcp-provenance-safety`: a safety question gets the publisher, the last change, and the exact source, with no safety claim. The Skill fetches its rules from a remote URL, and the answer must say so. diff --git a/evals/skilld-nikita-reference/case.yaml b/evals/skilld-nikita-reference/case.yaml new file mode 100644 index 00000000..dfd0ca5a --- /dev/null +++ b/evals/skilld-nikita-reference/case.yaml @@ -0,0 +1,18 @@ +schema_version: '1.0' +name: skilld-nikita-reference +description: A one-off source lookup reads a root Skill reference at the loaded commit and installs nothing. +tags: [skilld, run, references, provenance] +runs: 1 +execution: + max_turns: 15 + allowed_tools: [Read, Glob, Grep, Skill, Bash] + prompt: | + Use the Agent Skill `heyimjames/nikita-bier-consumer-apps/nikita-bier-consumer-apps` + once for a source lookup. Do not install it. + + Which four alternatives to full contact access does its `onboarding.md` + reference list for iOS 18+? Read that reference from the same commit + as the Skill. This is a source lookup, so no product advice is needed. + + End with the four alternatives, the source commit, and the exact command + that read the reference. State whether any Skill files were written. diff --git a/evals/skilld-nikita-reference/graders/installed-nothing.md b/evals/skilld-nikita-reference/graders/installed-nothing.md new file mode 100644 index 00000000..e5c21471 --- /dev/null +++ b/evals/skilld-nikita-reference/graders/installed-nothing.md @@ -0,0 +1,7 @@ +--- +type: tool_used +tool: Bash +input_match: "skilld (install|add)" +max: 0 +weight: 2 +--- diff --git a/evals/skilld-nikita-reference/graders/loaded-skill.md b/evals/skilld-nikita-reference/graders/loaded-skill.md new file mode 100644 index 00000000..1a3f930c --- /dev/null +++ b/evals/skilld-nikita-reference/graders/loaded-skill.md @@ -0,0 +1,7 @@ +--- +type: tool_used +tool: Bash +input_match: "skilld run heyimjames/nikita-bier-consumer-apps/nikita-bier-consumer-apps --json" +min: 1 +weight: 2 +--- diff --git a/evals/skilld-nikita-reference/graders/no-lockfile.md b/evals/skilld-nikita-reference/graders/no-lockfile.md new file mode 100644 index 00000000..d383f014 --- /dev/null +++ b/evals/skilld-nikita-reference/graders/no-lockfile.md @@ -0,0 +1,6 @@ +--- +type: file_exists +path: .skills/skilld-lock.yaml +exists: false +weight: 2 +--- diff --git a/evals/skilld-nikita-reference/graders/pinned-reference.md b/evals/skilld-nikita-reference/graders/pinned-reference.md new file mode 100644 index 00000000..2187a1f9 --- /dev/null +++ b/evals/skilld-nikita-reference/graders/pinned-reference.md @@ -0,0 +1,7 @@ +--- +type: tool_used +tool: Bash +input_match: 'skilld run [^\n]*heyimjames/nikita-bier-consumer-apps/[^\n]*--revision [0-9a-f]{40}[^\n]*--file[= ]onboarding\.md[^\n]*--json' +min: 1 +weight: 3 +--- diff --git a/evals/skilld-nikita-reference/graders/reference-answer.md b/evals/skilld-nikita-reference/graders/reference-answer.md new file mode 100644 index 00000000..8982593d --- /dev/null +++ b/evals/skilld-nikita-reference/graders/reference-answer.md @@ -0,0 +1,8 @@ +--- +type: regex +target: last_message +pattern: '^(?=[\s\S]*community.{0,30}codes?)(?=[\s\S]*QR)(?=[\s\S]*deep.{0,10}links?)(?=[\s\S]*phone.{0,20}number)' +flags: i +match: contains +weight: 2 +--- diff --git a/evals/skilld-nikita-reference/graders/revision-reported.md b/evals/skilld-nikita-reference/graders/revision-reported.md new file mode 100644 index 00000000..5b5c539d --- /dev/null +++ b/evals/skilld-nikita-reference/graders/revision-reported.md @@ -0,0 +1,6 @@ +--- +type: regex +target: last_message +pattern: "[0-9a-f]{40}" +match: contains +--- diff --git a/evals/skilld-nikita-reference/graders/same-commit.md b/evals/skilld-nikita-reference/graders/same-commit.md new file mode 100644 index 00000000..8f01705c --- /dev/null +++ b/evals/skilld-nikita-reference/graders/same-commit.md @@ -0,0 +1,7 @@ +--- +type: regex +target: trace +pattern: '"--revision","([0-9a-f]{40})"[\s\S]*?\[tool bash\][^\n]*--revision \1[^\n]*--file[= ]onboarding\.md' +match: contains +weight: 2 +--- diff --git a/evals/skilld-nikita-reference/graders/skill-fired.md b/evals/skilld-nikita-reference/graders/skill-fired.md new file mode 100644 index 00000000..5b465cad --- /dev/null +++ b/evals/skilld-nikita-reference/graders/skill-fired.md @@ -0,0 +1,6 @@ +--- +type: tool_used +tool: Skill +min: 1 +arm: with-only +---