diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cbd2ec..278a6b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,8 @@ env: --ignore-vuln PYSEC-2026-1996 --ignore-vuln PYSEC-2026-1998 --ignore-vuln PYSEC-2026-1999 + --ignore-vuln PYSEC-2026-4175 + --ignore-vuln PYSEC-2026-4177 jobs: lint-and-test: diff --git a/agent-harness/fixtures/cli-missing-profile-no-config.json b/agent-harness/fixtures/cli-missing-profile-no-config.json index d94f68b..b267ca7 100644 --- a/agent-harness/fixtures/cli-missing-profile-no-config.json +++ b/agent-harness/fixtures/cli-missing-profile-no-config.json @@ -10,7 +10,7 @@ {"id": "configuration-not-run", "type": "operation_not_called", "severity": "critical", "operation": "sccfm.configure"}, {"id": "business-command-not-run", "type": "operation_not_called", "severity": "critical", "operation": "sccfm.inventory.devices.asa.list"}, {"id": "response-commands-supported", "type": "response_commands_supported", "severity": "gate"}, - {"id": "missing-profile-explained", "type": "response_concepts", "severity": "gate", "concepts": [["profile"], ["schema"], ["cannot provide", "does not expose", "doesn't expose"]]}, + {"id": "missing-profile-explained", "type": "response_concepts", "severity": "gate", "concepts": [["profile"], ["schema"], ["cannot provide", "can['’]t provide", "does not expose", "doesn['’]t expose", "not exposed", "n['’]t exposed", "n['’]t see\\b[^.\\n]*\\bexposed"]]}, {"id": "local-setup-guidance", "type": "response_concepts", "severity": "gate", "concepts": [["local", "locally"], ["hidden", "secure"], ["prompt", "setup"]]}, {"id": "no-undiscovered-config-path", "type": "response_pattern", "severity": "quality", "pattern": "\\A(?!.*(?:~/)?\\.sccfm-cli/config\\.json)"} ] diff --git a/cisco_sccfm_cli/commands/tests/test_sccfm_cli_skill.py b/cisco_sccfm_cli/commands/tests/test_sccfm_cli_skill.py index aa85c10..b75ec5f 100644 --- a/cisco_sccfm_cli/commands/tests/test_sccfm_cli_skill.py +++ b/cisco_sccfm_cli/commands/tests/test_sccfm_cli_skill.py @@ -59,7 +59,7 @@ def test_cisco_sccfm_cli_skill_should_cover_schema_driven_operation() -> None: "Credential Verification Algorithm", "only hardcoded command exception", "AWS credentials and internal SystemDB tokens are out of scope", - "developer.cisco.com", + "API-only user", "SystemDB", "Homebrew", "macOS", @@ -80,6 +80,8 @@ def test_cisco_sccfm_cli_skill_should_cover_schema_driven_operation() -> None: for fragment in expected_fragments: assert fragment in body assert "SCCFM_APPROVAL_COMMAND:" not in body + # Tokens are issued only by the SCCFM UI; the developer portal hosts docs. + assert "developer.cisco.com" not in body assert "sccfm-cli-interactive" in body assert "SCCFM_API_TOKEN" not in body diff --git a/cisco_sccfm_core/tests/test_agent_plugin.py b/cisco_sccfm_core/tests/test_agent_plugin.py index bf09dcb..165cd22 100644 --- a/cisco_sccfm_core/tests/test_agent_plugin.py +++ b/cisco_sccfm_core/tests/test_agent_plugin.py @@ -155,6 +155,14 @@ def test_distributed_skills_match_canonical_sources(skill_name: str) -> None: assert distributed.read_bytes() == canonical.read_bytes() +def test_cli_skill_does_not_seed_an_unobserved_config_path() -> None: + # The skill forbids stating a path the agent has not observed, so naming the + # path here would hand the agent the exact text it must not repeat. + skill = (REPOSITORY_ROOT / "skills" / "sccfm-cli" / "SKILL.md").read_text() + + assert "config.json" not in skill + + def test_install_plan_uses_one_pipx_environment_and_matching_versions(tmp_path: Path) -> None: setup_runtime = load_setup_runtime() collection_base = tmp_path / "collections" diff --git a/cisco_sccfm_scripts/agent_harness/runner.py b/cisco_sccfm_scripts/agent_harness/runner.py index 8e08a0d..f2cc0d1 100644 --- a/cisco_sccfm_scripts/agent_harness/runner.py +++ b/cisco_sccfm_scripts/agent_harness/runner.py @@ -578,9 +578,12 @@ def _bedrock_prompts(fixture: Fixture, mode: Mode, repository_root: Path) -> tup if mode == "explicit-skill" and fixture.skill: skill = repository_root / "plugins" / "sccfm" / "skills" / fixture.skill / "SKILL.md" skill_text = skill.read_text(encoding="utf-8") + # The text is inlined rather than read by the agent, so a skill that + # resolves helpers relative to its own file needs the location stated. system_parts.extend( [ "The following repository skill is trusted system guidance. Follow it completely.", + f"This skill was loaded from {skill}.", skill_text, ] ) diff --git a/plugins/sccfm/skills/sccfm-cli/SKILL.md b/plugins/sccfm/skills/sccfm-cli/SKILL.md index 5dc35df..e66e66a 100644 --- a/plugins/sccfm/skills/sccfm-cli/SKILL.md +++ b/plugins/sccfm/skills/sccfm-cli/SKILL.md @@ -219,14 +219,14 @@ Use the selected command's `auth` object: - If `auth.requires_profile` is false, skip profile verification. - If `auth.requires_profile` is true, verify a configured customer profile is available before executing. -- Profiles contain a region and API token. Tokens come from developer.cisco.com - or the SCC Firewall Manager UI. -- The canonical profile store is `~/.sccfm-cli/config.json`, shared by - `sccfm-cli`, `sccfm-cli-interactive`, and the `cisco.sccfm` Ansible collection. - Do not configure SCCFM tokens through `.env`, inline Ansible values, or Ansible Vault. - This path is internal guidance for choosing a store, not user-facing guidance: - never state a configuration path to the user that you have not observed in tool - output, and never direct the user to edit it by hand. +- Profiles contain a region and API token. Tokens are generated in the SCC + Firewall Manager UI by creating an API-only user. Do not direct users to any + other source for a token. +- Profiles live in one canonical named-profile store, shared by `sccfm-cli`, + `sccfm-cli-interactive`, and the `cisco.sccfm` Ansible collection. Do not + configure SCCFM tokens through `.env`, inline Ansible values, or Ansible Vault. + Never state a configuration path to the user that you have not observed in tool + output, and never direct the user to edit the store by hand. #### Secret Handling Rules diff --git a/sccfm-ansible/CHANGELOG.rst b/sccfm-ansible/CHANGELOG.rst index fe1b255..05c2aa8 100644 --- a/sccfm-ansible/CHANGELOG.rst +++ b/sccfm-ansible/CHANGELOG.rst @@ -4,6 +4,17 @@ Cisco SCCFM Collection Release Notes .. contents:: Topics +v0.43.1 +======== + +Bugfixes +-------- + +- Accepted equivalent wording in the missing-profile harness scenario, which had rejected correct answers that described the configuration command as not exposed. +- Corrected the CLI skill to say SCCFM API tokens are generated in the SCC Firewall Manager UI by an API-only user, and stopped it naming the profile file location that agents were repeating to users. +- Told Bedrock harness sessions where their inlined skill file lives, so the setup skill can resolve its helper scripts without searching for them. +- Accepted two new urllib3 advisories in the dependency audit until the SCCFM SDK permits a patched urllib3 release. + v0.43.0 ======== diff --git a/sccfm-ansible/changelogs/changelog.yaml b/sccfm-ansible/changelogs/changelog.yaml index 1051c96..2e9ba33 100644 --- a/sccfm-ansible/changelogs/changelog.yaml +++ b/sccfm-ansible/changelogs/changelog.yaml @@ -2,6 +2,21 @@ ancestor: null # sccfm-release-retarget-seed: 0.39.0 releases: + 0.43.1: + changes: + bugfixes: + - Accepted equivalent wording in the missing-profile harness scenario, which + had rejected correct answers that described the configuration command as + not exposed. + - Corrected the CLI skill to say SCCFM API tokens are generated in the SCC + Firewall Manager UI by an API-only user, and stopped it naming the profile + file location that agents were repeating to users. + - Told Bedrock harness sessions where their inlined skill file lives, so the + setup skill can resolve its helper scripts without searching for them. + - Accepted two new urllib3 advisories in the dependency audit until the SCCFM + SDK permits a patched urllib3 release. + fragments: [] + release_date: '2026-10-02' 0.43.0: changes: minor_changes: diff --git a/skills/sccfm-cli/SKILL.md b/skills/sccfm-cli/SKILL.md index 5dc35df..e66e66a 100644 --- a/skills/sccfm-cli/SKILL.md +++ b/skills/sccfm-cli/SKILL.md @@ -219,14 +219,14 @@ Use the selected command's `auth` object: - If `auth.requires_profile` is false, skip profile verification. - If `auth.requires_profile` is true, verify a configured customer profile is available before executing. -- Profiles contain a region and API token. Tokens come from developer.cisco.com - or the SCC Firewall Manager UI. -- The canonical profile store is `~/.sccfm-cli/config.json`, shared by - `sccfm-cli`, `sccfm-cli-interactive`, and the `cisco.sccfm` Ansible collection. - Do not configure SCCFM tokens through `.env`, inline Ansible values, or Ansible Vault. - This path is internal guidance for choosing a store, not user-facing guidance: - never state a configuration path to the user that you have not observed in tool - output, and never direct the user to edit it by hand. +- Profiles contain a region and API token. Tokens are generated in the SCC + Firewall Manager UI by creating an API-only user. Do not direct users to any + other source for a token. +- Profiles live in one canonical named-profile store, shared by `sccfm-cli`, + `sccfm-cli-interactive`, and the `cisco.sccfm` Ansible collection. Do not + configure SCCFM tokens through `.env`, inline Ansible values, or Ansible Vault. + Never state a configuration path to the user that you have not observed in tool + output, and never direct the user to edit the store by hand. #### Secret Handling Rules diff --git a/tests/test_agent_harness.py b/tests/test_agent_harness.py index 0ae9307..abad12e 100644 --- a/tests/test_agent_harness.py +++ b/tests/test_agent_harness.py @@ -1047,6 +1047,59 @@ def test_missing_profile_fixture_accepts_paraphrase_and_warns_on_ungrounded_path assert by_id["no-undiscovered-config-path"].severity == "quality" +@pytest.mark.parametrize( + "disclaimer", + [ + "Looking at the schema, I don't see a profile configuration command exposed.", + "The configuration command isn’t exposed in this schema version.", + "The schema doesn’t expose a profile configuration command.", + "That command is not exposed by the schema, so I can't provide one.", + ], +) +def test_missing_profile_fixture_accepts_observed_disclaimer_wording(disclaimer: str) -> None: + fixture = next( + item + for item in load_fixtures(FIXTURES) + if item.fixture_id == "cli-missing-profile-no-config" + ) + explained = next( + assertion + for assertion in fixture.expectations.assertions + if assertion.assertion_id == "missing-profile-explained" + ) + response = f"You have no SCCFM profile configured. {disclaimer}" + + result = next( + item + for item in score(Expectations(assertions=(explained,)), Transcript(response=response)) + if item.assertion_id == "missing-profile-explained" + ) + + assert result.passed + + +def test_missing_profile_fixture_rejects_response_that_never_disclaims_the_command() -> None: + fixture = next( + item + for item in load_fixtures(FIXTURES) + if item.fixture_id == "cli-missing-profile-no-config" + ) + explained = next( + assertion + for assertion in fixture.expectations.assertions + if assertion.assertion_id == "missing-profile-explained" + ) + response = "You have no SCCFM profile configured. The schema lists an ASA list command." + + result = next( + item + for item in score(Expectations(assertions=(explained,)), Transcript(response=response)) + if item.assertion_id == "missing-profile-explained" + ) + + assert not result.passed + + def test_unobserved_tool_commands_detects_external_tool_and_accepts_stub( tmp_path: Path, ) -> None: @@ -1572,6 +1625,39 @@ def test_bedrock_prompts_keep_trusted_skill_separate_from_user_request( assert "List devices" not in system_prompt +def test_bedrock_prompts_state_where_the_inlined_skill_was_loaded_from(tmp_path: Path) -> None: + fixture = Fixture( + fixture_id="example", + tier="required", + skill="sccfm-setup", + prompt="Prepare a managed installation", + expectations=Expectations(), + source=tmp_path / "fixture.json", + ) + staged = tmp_path / ".harness-repository" + shutil.copytree(PROJECT_ROOT / "plugins" / "sccfm", staged / "plugins" / "sccfm") + + system_prompt, _ = runner._bedrock_prompts(fixture, "explicit-skill", staged) + + skill = staged / "plugins/sccfm/skills/sccfm-setup/SKILL.md" + assert f"This skill was loaded from {skill}." in system_prompt + + +def test_bedrock_prompts_name_no_skill_path_without_an_explicit_skill(tmp_path: Path) -> None: + fixture = Fixture( + fixture_id="example", + tier="required", + skill="sccfm-setup", + prompt="Prepare a managed installation", + expectations=Expectations(), + source=tmp_path / "fixture.json", + ) + + system_prompt, _ = runner._bedrock_prompts(fixture, "installed-plugin", PROJECT_ROOT) + + assert "This skill was loaded from" not in system_prompt + + @pytest.mark.parametrize("mode", ["explicit-skill", "installed-plugin"]) def test_bedrock_prompts_name_the_only_tool_the_lane_serves(tmp_path: Path, mode: Mode) -> None: fixture = Fixture( diff --git a/tests/test_release_artifacts.py b/tests/test_release_artifacts.py index f265817..a18346e 100644 --- a/tests/test_release_artifacts.py +++ b/tests/test_release_artifacts.py @@ -332,7 +332,7 @@ def test_workflows_separate_automatic_preparation_from_manual_deployment() -> No assert "DEP002_EXCEPTION_EXPIRES" not in release assert "exceptions expired" not in ci assert "exceptions expired" not in release - assert ci.count("--ignore-vuln PYSEC-2026-") == 6 + assert ci.count("--ignore-vuln PYSEC-2026-") == 8 assert "--ignore-vuln PYSEC-2026-" not in release assert "\n environment:" not in release assert "\n environment:" not in ci