From b415e4216dc5e5be224d7f3191bd6dce2d788454 Mon Sep 17 00:00:00 2001 From: Andrei Huides Date: Fri, 2 Oct 2026 15:02:28 +0300 Subject: [PATCH 1/3] fix(lh-132058): align harness grading and skill guidance with runs Bedrock reliability runs showed the missing-profile scenario failing on correct answers that said the configuration command "isn't exposed" instead of one of three exact phrasings. The fixture now accepts those equivalent wordings, including curly apostrophes, while still failing an answer that never says the command is missing. The CLI skill told agents that API tokens come from developer.cisco.com, which only hosts the documentation, so nearly every missing-profile answer sent users to the wrong place. It now states that tokens are generated in the SCC Firewall Manager UI by an API-only user. The skill also no longer names the profile file location, which agents repeated despite the rule against stating a path not seen in tool output. The Bedrock lane inlines the skill text without its location, so the setup skill could not resolve its helper scripts and agents spent many commands searching for them. The system prompt now states the path of the staged skill file the agent can reach inside its sandbox. --- .../cli-missing-profile-no-config.json | 2 +- .../commands/tests/test_sccfm_cli_skill.py | 4 +- cisco_sccfm_core/tests/test_agent_plugin.py | 8 ++ cisco_sccfm_scripts/agent_harness/runner.py | 3 + plugins/sccfm/skills/sccfm-cli/SKILL.md | 16 ++-- skills/sccfm-cli/SKILL.md | 16 ++-- tests/test_agent_harness.py | 86 +++++++++++++++++++ 7 files changed, 117 insertions(+), 18 deletions(-) diff --git a/agent-harness/fixtures/cli-missing-profile-no-config.json b/agent-harness/fixtures/cli-missing-profile-no-config.json index d94f68b..b267ca7 100644 --- a/agent-harness/fixtures/cli-missing-profile-no-config.json +++ b/agent-harness/fixtures/cli-missing-profile-no-config.json @@ -10,7 +10,7 @@ {"id": "configuration-not-run", "type": "operation_not_called", "severity": "critical", "operation": "sccfm.configure"}, {"id": "business-command-not-run", "type": "operation_not_called", "severity": "critical", "operation": "sccfm.inventory.devices.asa.list"}, {"id": "response-commands-supported", "type": "response_commands_supported", "severity": "gate"}, - {"id": "missing-profile-explained", "type": "response_concepts", "severity": "gate", "concepts": [["profile"], ["schema"], ["cannot provide", "does not expose", "doesn't expose"]]}, + {"id": "missing-profile-explained", "type": "response_concepts", "severity": "gate", "concepts": [["profile"], ["schema"], ["cannot provide", "can['’]t provide", "does not expose", "doesn['’]t expose", "not exposed", "n['’]t exposed", "n['’]t see\\b[^.\\n]*\\bexposed"]]}, {"id": "local-setup-guidance", "type": "response_concepts", "severity": "gate", "concepts": [["local", "locally"], ["hidden", "secure"], ["prompt", "setup"]]}, {"id": "no-undiscovered-config-path", "type": "response_pattern", "severity": "quality", "pattern": "\\A(?!.*(?:~/)?\\.sccfm-cli/config\\.json)"} ] diff --git a/cisco_sccfm_cli/commands/tests/test_sccfm_cli_skill.py b/cisco_sccfm_cli/commands/tests/test_sccfm_cli_skill.py index aa85c10..b75ec5f 100644 --- a/cisco_sccfm_cli/commands/tests/test_sccfm_cli_skill.py +++ b/cisco_sccfm_cli/commands/tests/test_sccfm_cli_skill.py @@ -59,7 +59,7 @@ def test_cisco_sccfm_cli_skill_should_cover_schema_driven_operation() -> None: "Credential Verification Algorithm", "only hardcoded command exception", "AWS credentials and internal SystemDB tokens are out of scope", - "developer.cisco.com", + "API-only user", "SystemDB", "Homebrew", "macOS", @@ -80,6 +80,8 @@ def test_cisco_sccfm_cli_skill_should_cover_schema_driven_operation() -> None: for fragment in expected_fragments: assert fragment in body assert "SCCFM_APPROVAL_COMMAND:" not in body + # Tokens are issued only by the SCCFM UI; the developer portal hosts docs. + assert "developer.cisco.com" not in body assert "sccfm-cli-interactive" in body assert "SCCFM_API_TOKEN" not in body diff --git a/cisco_sccfm_core/tests/test_agent_plugin.py b/cisco_sccfm_core/tests/test_agent_plugin.py index bf09dcb..165cd22 100644 --- a/cisco_sccfm_core/tests/test_agent_plugin.py +++ b/cisco_sccfm_core/tests/test_agent_plugin.py @@ -155,6 +155,14 @@ def test_distributed_skills_match_canonical_sources(skill_name: str) -> None: assert distributed.read_bytes() == canonical.read_bytes() +def test_cli_skill_does_not_seed_an_unobserved_config_path() -> None: + # The skill forbids stating a path the agent has not observed, so naming the + # path here would hand the agent the exact text it must not repeat. + skill = (REPOSITORY_ROOT / "skills" / "sccfm-cli" / "SKILL.md").read_text() + + assert "config.json" not in skill + + def test_install_plan_uses_one_pipx_environment_and_matching_versions(tmp_path: Path) -> None: setup_runtime = load_setup_runtime() collection_base = tmp_path / "collections" diff --git a/cisco_sccfm_scripts/agent_harness/runner.py b/cisco_sccfm_scripts/agent_harness/runner.py index 8e08a0d..f2cc0d1 100644 --- a/cisco_sccfm_scripts/agent_harness/runner.py +++ b/cisco_sccfm_scripts/agent_harness/runner.py @@ -578,9 +578,12 @@ def _bedrock_prompts(fixture: Fixture, mode: Mode, repository_root: Path) -> tup if mode == "explicit-skill" and fixture.skill: skill = repository_root / "plugins" / "sccfm" / "skills" / fixture.skill / "SKILL.md" skill_text = skill.read_text(encoding="utf-8") + # The text is inlined rather than read by the agent, so a skill that + # resolves helpers relative to its own file needs the location stated. system_parts.extend( [ "The following repository skill is trusted system guidance. Follow it completely.", + f"This skill was loaded from {skill}.", skill_text, ] ) diff --git a/plugins/sccfm/skills/sccfm-cli/SKILL.md b/plugins/sccfm/skills/sccfm-cli/SKILL.md index 5dc35df..e66e66a 100644 --- a/plugins/sccfm/skills/sccfm-cli/SKILL.md +++ b/plugins/sccfm/skills/sccfm-cli/SKILL.md @@ -219,14 +219,14 @@ Use the selected command's `auth` object: - If `auth.requires_profile` is false, skip profile verification. - If `auth.requires_profile` is true, verify a configured customer profile is available before executing. -- Profiles contain a region and API token. Tokens come from developer.cisco.com - or the SCC Firewall Manager UI. -- The canonical profile store is `~/.sccfm-cli/config.json`, shared by - `sccfm-cli`, `sccfm-cli-interactive`, and the `cisco.sccfm` Ansible collection. - Do not configure SCCFM tokens through `.env`, inline Ansible values, or Ansible Vault. - This path is internal guidance for choosing a store, not user-facing guidance: - never state a configuration path to the user that you have not observed in tool - output, and never direct the user to edit it by hand. +- Profiles contain a region and API token. Tokens are generated in the SCC + Firewall Manager UI by creating an API-only user. Do not direct users to any + other source for a token. +- Profiles live in one canonical named-profile store, shared by `sccfm-cli`, + `sccfm-cli-interactive`, and the `cisco.sccfm` Ansible collection. Do not + configure SCCFM tokens through `.env`, inline Ansible values, or Ansible Vault. + Never state a configuration path to the user that you have not observed in tool + output, and never direct the user to edit the store by hand. #### Secret Handling Rules diff --git a/skills/sccfm-cli/SKILL.md b/skills/sccfm-cli/SKILL.md index 5dc35df..e66e66a 100644 --- a/skills/sccfm-cli/SKILL.md +++ b/skills/sccfm-cli/SKILL.md @@ -219,14 +219,14 @@ Use the selected command's `auth` object: - If `auth.requires_profile` is false, skip profile verification. - If `auth.requires_profile` is true, verify a configured customer profile is available before executing. -- Profiles contain a region and API token. Tokens come from developer.cisco.com - or the SCC Firewall Manager UI. -- The canonical profile store is `~/.sccfm-cli/config.json`, shared by - `sccfm-cli`, `sccfm-cli-interactive`, and the `cisco.sccfm` Ansible collection. - Do not configure SCCFM tokens through `.env`, inline Ansible values, or Ansible Vault. - This path is internal guidance for choosing a store, not user-facing guidance: - never state a configuration path to the user that you have not observed in tool - output, and never direct the user to edit it by hand. +- Profiles contain a region and API token. Tokens are generated in the SCC + Firewall Manager UI by creating an API-only user. Do not direct users to any + other source for a token. +- Profiles live in one canonical named-profile store, shared by `sccfm-cli`, + `sccfm-cli-interactive`, and the `cisco.sccfm` Ansible collection. Do not + configure SCCFM tokens through `.env`, inline Ansible values, or Ansible Vault. + Never state a configuration path to the user that you have not observed in tool + output, and never direct the user to edit the store by hand. #### Secret Handling Rules diff --git a/tests/test_agent_harness.py b/tests/test_agent_harness.py index 0ae9307..abad12e 100644 --- a/tests/test_agent_harness.py +++ b/tests/test_agent_harness.py @@ -1047,6 +1047,59 @@ def test_missing_profile_fixture_accepts_paraphrase_and_warns_on_ungrounded_path assert by_id["no-undiscovered-config-path"].severity == "quality" +@pytest.mark.parametrize( + "disclaimer", + [ + "Looking at the schema, I don't see a profile configuration command exposed.", + "The configuration command isn’t exposed in this schema version.", + "The schema doesn’t expose a profile configuration command.", + "That command is not exposed by the schema, so I can't provide one.", + ], +) +def test_missing_profile_fixture_accepts_observed_disclaimer_wording(disclaimer: str) -> None: + fixture = next( + item + for item in load_fixtures(FIXTURES) + if item.fixture_id == "cli-missing-profile-no-config" + ) + explained = next( + assertion + for assertion in fixture.expectations.assertions + if assertion.assertion_id == "missing-profile-explained" + ) + response = f"You have no SCCFM profile configured. {disclaimer}" + + result = next( + item + for item in score(Expectations(assertions=(explained,)), Transcript(response=response)) + if item.assertion_id == "missing-profile-explained" + ) + + assert result.passed + + +def test_missing_profile_fixture_rejects_response_that_never_disclaims_the_command() -> None: + fixture = next( + item + for item in load_fixtures(FIXTURES) + if item.fixture_id == "cli-missing-profile-no-config" + ) + explained = next( + assertion + for assertion in fixture.expectations.assertions + if assertion.assertion_id == "missing-profile-explained" + ) + response = "You have no SCCFM profile configured. The schema lists an ASA list command." + + result = next( + item + for item in score(Expectations(assertions=(explained,)), Transcript(response=response)) + if item.assertion_id == "missing-profile-explained" + ) + + assert not result.passed + + def test_unobserved_tool_commands_detects_external_tool_and_accepts_stub( tmp_path: Path, ) -> None: @@ -1572,6 +1625,39 @@ def test_bedrock_prompts_keep_trusted_skill_separate_from_user_request( assert "List devices" not in system_prompt +def test_bedrock_prompts_state_where_the_inlined_skill_was_loaded_from(tmp_path: Path) -> None: + fixture = Fixture( + fixture_id="example", + tier="required", + skill="sccfm-setup", + prompt="Prepare a managed installation", + expectations=Expectations(), + source=tmp_path / "fixture.json", + ) + staged = tmp_path / ".harness-repository" + shutil.copytree(PROJECT_ROOT / "plugins" / "sccfm", staged / "plugins" / "sccfm") + + system_prompt, _ = runner._bedrock_prompts(fixture, "explicit-skill", staged) + + skill = staged / "plugins/sccfm/skills/sccfm-setup/SKILL.md" + assert f"This skill was loaded from {skill}." in system_prompt + + +def test_bedrock_prompts_name_no_skill_path_without_an_explicit_skill(tmp_path: Path) -> None: + fixture = Fixture( + fixture_id="example", + tier="required", + skill="sccfm-setup", + prompt="Prepare a managed installation", + expectations=Expectations(), + source=tmp_path / "fixture.json", + ) + + system_prompt, _ = runner._bedrock_prompts(fixture, "installed-plugin", PROJECT_ROOT) + + assert "This skill was loaded from" not in system_prompt + + @pytest.mark.parametrize("mode", ["explicit-skill", "installed-plugin"]) def test_bedrock_prompts_name_the_only_tool_the_lane_serves(tmp_path: Path, mode: Mode) -> None: fixture = Fixture( From 0274aa3bb81beb21bfea462c5937f2c19e49e27b Mon Sep 17 00:00:00 2001 From: Andrei Huides Date: Fri, 2 Oct 2026 18:02:51 +0300 Subject: [PATCH 2/3] ci(lh-132058): ignore new urllib3 advisories blocked by sdk pin Two urllib3 advisories, PYSEC-2026-4175 and PYSEC-2026-4177, were published after the last green main build and now fail the runtime dependency audit on every branch. Both are fixed in urllib3 2.8.0, but the SCC Firewall Manager SDK requires urllib3 below 2.1.0 in both the locked release and the newest one on PyPI, so the patched version cannot be resolved. The two IDs join the existing exception list kept for the same SDK pin, and the guard test that counts the exceptions is raised to match so further additions still need an explicit change. The exceptions should be removed once the SDK permits a patched urllib3 release. --- .github/workflows/ci.yml | 2 ++ tests/test_release_artifacts.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cbd2ec..278a6b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,8 @@ env: --ignore-vuln PYSEC-2026-1996 --ignore-vuln PYSEC-2026-1998 --ignore-vuln PYSEC-2026-1999 + --ignore-vuln PYSEC-2026-4175 + --ignore-vuln PYSEC-2026-4177 jobs: lint-and-test: diff --git a/tests/test_release_artifacts.py b/tests/test_release_artifacts.py index f265817..a18346e 100644 --- a/tests/test_release_artifacts.py +++ b/tests/test_release_artifacts.py @@ -332,7 +332,7 @@ def test_workflows_separate_automatic_preparation_from_manual_deployment() -> No assert "DEP002_EXCEPTION_EXPIRES" not in release assert "exceptions expired" not in ci assert "exceptions expired" not in release - assert ci.count("--ignore-vuln PYSEC-2026-") == 6 + assert ci.count("--ignore-vuln PYSEC-2026-") == 8 assert "--ignore-vuln PYSEC-2026-" not in release assert "\n environment:" not in release assert "\n environment:" not in ci From 66c4005a4f5781c0c9c50713be4b3196dff6dd0a Mon Sep 17 00:00:00 2001 From: Andrei Huides Date: Fri, 2 Oct 2026 18:50:33 +0300 Subject: [PATCH 3/3] docs(lh-132058): prepare Ansible changelog for 0.43.1 The fix commit on this branch makes the next release a patch, and release preparation requires the target version to be recorded in source. Add the 0.43.1 entry covering the harness grading, skill guidance, Bedrock skill path, and urllib3 audit exception changes. --- sccfm-ansible/CHANGELOG.rst | 11 +++++++++++ sccfm-ansible/changelogs/changelog.yaml | 15 +++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/sccfm-ansible/CHANGELOG.rst b/sccfm-ansible/CHANGELOG.rst index fe1b255..05c2aa8 100644 --- a/sccfm-ansible/CHANGELOG.rst +++ b/sccfm-ansible/CHANGELOG.rst @@ -4,6 +4,17 @@ Cisco SCCFM Collection Release Notes .. contents:: Topics +v0.43.1 +======== + +Bugfixes +-------- + +- Accepted equivalent wording in the missing-profile harness scenario, which had rejected correct answers that described the configuration command as not exposed. +- Corrected the CLI skill to say SCCFM API tokens are generated in the SCC Firewall Manager UI by an API-only user, and stopped it naming the profile file location that agents were repeating to users. +- Told Bedrock harness sessions where their inlined skill file lives, so the setup skill can resolve its helper scripts without searching for them. +- Accepted two new urllib3 advisories in the dependency audit until the SCCFM SDK permits a patched urllib3 release. + v0.43.0 ======== diff --git a/sccfm-ansible/changelogs/changelog.yaml b/sccfm-ansible/changelogs/changelog.yaml index 1051c96..2e9ba33 100644 --- a/sccfm-ansible/changelogs/changelog.yaml +++ b/sccfm-ansible/changelogs/changelog.yaml @@ -2,6 +2,21 @@ ancestor: null # sccfm-release-retarget-seed: 0.39.0 releases: + 0.43.1: + changes: + bugfixes: + - Accepted equivalent wording in the missing-profile harness scenario, which + had rejected correct answers that described the configuration command as + not exposed. + - Corrected the CLI skill to say SCCFM API tokens are generated in the SCC + Firewall Manager UI by an API-only user, and stopped it naming the profile + file location that agents were repeating to users. + - Told Bedrock harness sessions where their inlined skill file lives, so the + setup skill can resolve its helper scripts without searching for them. + - Accepted two new urllib3 advisories in the dependency audit until the SCCFM + SDK permits a patched urllib3 release. + fragments: [] + release_date: '2026-10-02' 0.43.0: changes: minor_changes: