From ff46920befe5fe4f96aafd7e12a1d6e5df9084ea Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 21 Oct 2026 21:55:09 +0000 Subject: [PATCH 1/2] test(security): scan generated Python artifacts --- .github/workflows/README.md | 6 +- .github/workflows/ci.yml | 7 + .../workflows/generated-artifact-security.yml | 63 +++++ .../workflows/release-main-and-preview.yml | 11 +- e2eTest/README.md | 7 + .../generated-artifact-security.test.ts | 237 ++++++++++++++++++ e2eTest/setup.ts | 14 +- package.json | 1 + ...ted-artifact-vulnerability-exceptions.yaml | 69 +++++ 9 files changed, 406 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/generated-artifact-security.yml create mode 100644 e2eTest/project/generated-artifact-security.test.ts create mode 100644 security/generated-artifact-vulnerability-exceptions.yaml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 1250f12628..5210ac8480 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -8,7 +8,7 @@ Workflows are organized into two roles: Orchestrators (ci.yml, canary.yml, prerelease.yml, release-prepare.yml, release-main-and-preview.yml, pr-automation.yml, codeql.yml, slack-pr-review-notification.yml, workflow-metrics.yml) -Jobs (verify.yml, e2e-test.yml) +Jobs (verify.yml, e2e-test.yml, generated-artifact-security.yml) ``` **Orchestrators** respond to events and coordinate work. They define _when_ and @@ -27,6 +27,7 @@ An orchestrator calls jobs via `workflow_call`. ci.yml |-- verify.yml (one job per platform: static checks on Linux, then bundle, | package, compile, smoke test and unit tests) + |-- generated-artifact-security.yml (fresh Python lockfiles and ARM64 image scans) `-- e2e-test.yml (deploys and invokes real AgentCore resources) canary.yml (scheduled) @@ -35,7 +36,8 @@ canary.yml (scheduled) release-prepare.yml (version bump, vended CDK pin, release PR) release-main-and-preview.yml (npm publish and GitHub release when a release PR merges) - `-- verify.yml + |-- verify.yml + `-- generated-artifact-security.yml ``` `verify.yml` and `e2e-test.yml` each appear in two orchestrators. This is the diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64d307e956..92a5e293bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,13 @@ concurrency: cancel-in-progress: true jobs: + generated-artifact-security: + uses: ./.github/workflows/generated-artifact-security.yml + permissions: + contents: read + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + verify: uses: ./.github/workflows/verify.yml permissions: diff --git a/.github/workflows/generated-artifact-security.yml b/.github/workflows/generated-artifact-security.yml new file mode 100644 index 0000000000..61878a7d72 --- /dev/null +++ b/.github/workflows/generated-artifact-security.yml @@ -0,0 +1,63 @@ +# Generates every supported Python project through the released CLI path, resolves fresh lockfiles, +# and scans both those dependencies and generated ARM64 container images. +name: generated-artifact-security +on: + workflow_call: + inputs: + ref: + required: true + type: string + workflow_dispatch: + schedule: + - cron: "23 7 * * 1" + +permissions: + contents: read + +env: + AGENTCORE_TELEMETRY_DISABLED: "1" + +jobs: + scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + - uses: oven-sh/setup-bun@v2 + - uses: actions/setup-node@v6 + with: + node-version: "24" + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.14" + - run: bun install --frozen-lockfile + - run: bun run build + + - name: Enable ARM64 container builds + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + platforms: arm64 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Install Trivy + uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1 + with: + version: v0.76.1 + cache: true + + - name: Scan generated projects and images + env: + AGENTCORE_CLI_PATH: node ${{ github.workspace }}/dist/index.js + GENERATED_ARTIFACT_SECURITY_REPORT_DIR: ${{ github.workspace }}/out/generated-artifact-security/reports + run: bun run test:e2e:generated-artifact-security + + - name: Upload scan reports + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: generated-artifact-security-${{ github.sha }} + path: out/generated-artifact-security + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/release-main-and-preview.yml b/.github/workflows/release-main-and-preview.yml index 4bc01ed46a..7d8bbec802 100644 --- a/.github/workflows/release-main-and-preview.yml +++ b/.github/workflows/release-main-and-preview.yml @@ -51,8 +51,17 @@ jobs: ref: ${{ github.sha }} macos: false + generated-artifact-security: + needs: check-release + if: needs.check-release.outputs.is_release == 'true' + uses: ./.github/workflows/generated-artifact-security.yml + permissions: + contents: read + with: + ref: ${{ github.sha }} + publish: - needs: verify + needs: [verify, generated-artifact-security] # npm provenance is only issued from GitHub-hosted runners. runs-on: ubuntu-latest environment: diff --git a/e2eTest/README.md b/e2eTest/README.md index ea7f54dd9a..25dea6eba1 100644 --- a/e2eTest/README.md +++ b/e2eTest/README.md @@ -15,6 +15,13 @@ To run tagged tests: bun run test:e2e -- --tagsFilter='runtime || canary' ``` +The generated-project vulnerability E2E is Linux-only and additionally requires Docker with +ARM64 build support and Trivy: + +```sh +bun run test:e2e:generated-artifact-security +``` + Set `AGENTCORE_CLI_PATH` to use a different executable. The value is split on whitespace and is not run through a shell, so paths containing spaces, quoting, and shell syntax are not supported: diff --git a/e2eTest/project/generated-artifact-security.test.ts b/e2eTest/project/generated-artifact-security.test.ts new file mode 100644 index 0000000000..64015aaeb0 --- /dev/null +++ b/e2eTest/project/generated-artifact-security.test.ts @@ -0,0 +1,237 @@ +import { createHash } from "node:crypto"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, test } from "vitest"; +import { parse } from "yaml"; +import z from "zod"; +import { RUNTIME_TEMPLATE_SHORTCUTS } from "../../src/handlers/project/shortcuts"; +import { CliRunner, parseResult, type RunResult } from "../helpers/run"; + +type ScanCase = { + template: string; + projectName: string; + container: boolean; +}; + +type VulnerabilityException = { + id?: unknown; + statement?: unknown; + expired_at?: unknown; +}; + +const SCAN_CASES: readonly ScanCase[] = [ + { template: "agent-python-minimal", projectName: "ScanPythonMinimal", container: false }, + { template: "agent-python-strands", projectName: "ScanPythonStrands", container: false }, + { + template: "agent-python-strands-container", + projectName: "ScanStrandsContainer", + container: true, + }, + { template: "agent-python-langchain", projectName: "ScanPythonLangchain", container: false }, + { template: "mcp-python-fastmcp", projectName: "ScanPythonFastMcp", container: false }, + { template: "a2a-python-strands", projectName: "ScanPythonA2a", container: false }, + { template: "agui-python-strands", projectName: "ScanPythonAgui", container: false }, + { template: "environment-python-bma", projectName: "ScanPythonBma", container: true }, +] as const; + +const ProjectCreatedSchema = z.object({ + project: z.object({ path: z.string() }), +}); + +const repositoryRoot = resolve("."); +const reportRoot = resolve( + process.env.GENERATED_ARTIFACT_SECURITY_REPORT_DIR ?? "out/generated-artifact-security/reports", +); +const exceptionsPath = join( + repositoryRoot, + "security", + "generated-artifact-vulnerability-exceptions.yaml", +); + +function run(command: readonly string[], cwd: string): Promise { + return new Promise((resolveResult, reject) => { + const [executable, ...args] = command; + if (!executable) throw new Error("cannot run an empty command"); + const child = spawn(executable, args, { + cwd, + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (exitCode) => resolveResult({ stdout, stderr, exitCode: exitCode ?? -1 })); + }); +} + +async function runSuccessfully(command: readonly string[], cwd: string): Promise { + const result = await run(command, cwd); + expect( + result.exitCode, + `${command.join(" ")} failed\nstdout: ${result.stdout}\nstderr: ${result.stderr}`, + ).toBe(0); + return result; +} + +async function validateExceptions(): Promise { + const document = parse(await readFile(exceptionsPath, "utf8")) as { + vulnerabilities?: unknown; + } | null; + const vulnerabilities = document?.vulnerabilities; + expect(Array.isArray(vulnerabilities)).toBe(true); + if (!Array.isArray(vulnerabilities)) throw new Error("exceptions must be a list"); + + const today = new Date().toISOString().slice(0, 10); + for (const exception of vulnerabilities as VulnerabilityException[]) { + expect(exception.id, "every exception must identify a vulnerability").toBeTypeOf("string"); + expect(exception.statement, `${String(exception.id)} must document its rationale`).toBeTypeOf( + "string", + ); + expect( + (exception.statement as string).trim().length, + `${String(exception.id)} must document a non-empty rationale`, + ).toBeGreaterThan(0); + expect(exception.expired_at, `${String(exception.id)} must have an expiration`).toMatch( + /^\d{4}-\d{2}-\d{2}$/, + ); + expect( + (exception.expired_at as string) >= today, + `${String(exception.id)} expired on ${String(exception.expired_at)}`, + ).toBe(true); + } +} + +describe.runIf(process.platform === "linux")("generated artifact vulnerability scanning", () => { + const cli = new CliRunner(); + + test("matrix covers every supported Python template", () => { + const supported = Object.entries(RUNTIME_TEMPLATE_SHORTCUTS) + .filter(([, template]) => template.language === "Python") + .map(([name]) => name) + .sort(); + expect(SCAN_CASES.map(({ template }) => template).sort()).toEqual(supported); + }); + + test.each(SCAN_CASES)( + "$template", + { timeout: 15 * 60 * 1000 }, + async ({ template, projectName, container }) => { + await validateExceptions(); + await mkdir(reportRoot, { recursive: true }); + const parent = await mkdtemp(join(tmpdir(), "agentcore-security-e2e-")); + try { + const created = parseResult( + ProjectCreatedSchema, + await cli.run( + [ + "create", + "--name", + projectName, + "--template", + template, + "--skip-install", + "--skip-git", + "--region", + "us-west-2", + "--json", + ], + parent, + ), + ); + const runtimeRoot = join(created.project.path, "app", "agent"); + await runSuccessfully(["uv", "lock", "--refresh"], runtimeRoot); + + const dependencyReport = join(reportRoot, `${template}-dependencies.json`); + await runSuccessfully( + [ + "trivy", + "fs", + "--scanners", + "vuln", + "--severity", + "HIGH,CRITICAL", + "--ignore-unfixed", + "--show-suppressed", + "--exit-code", + "1", + "--format", + "json", + "--output", + dependencyReport, + "--ignorefile", + exceptionsPath, + runtimeRoot, + ], + repositoryRoot, + ); + + const metadata: Record = { + template, + architecture: container ? "linux/arm64" : undefined, + lockfileSha256: createHash("sha256") + .update(await readFile(join(runtimeRoot, "uv.lock"))) + .digest("hex"), + dependencyReport, + }; + if (container) { + const imageName = `agentcore-generated-scan:${template}`; + await runSuccessfully( + [ + "docker", + "buildx", + "build", + "--platform", + "linux/arm64", + "--pull", + "--no-cache", + "--load", + "--tag", + imageName, + runtimeRoot, + ], + repositoryRoot, + ); + const image = await runSuccessfully( + ["docker", "image", "inspect", "--format", "{{.Id}}", imageName], + repositoryRoot, + ); + const imageReport = join(reportRoot, `${template}-image.json`); + await runSuccessfully( + [ + "trivy", + "image", + "--scanners", + "vuln", + "--severity", + "HIGH,CRITICAL", + "--ignore-unfixed", + "--show-suppressed", + "--exit-code", + "1", + "--format", + "json", + "--output", + imageReport, + "--ignorefile", + exceptionsPath, + imageName, + ], + repositoryRoot, + ); + metadata.imageDigest = image.stdout.trim(); + metadata.imageReport = imageReport; + } + await writeFile( + join(reportRoot, `${template}-metadata.json`), + `${JSON.stringify(metadata, null, 2)}\n`, + ); + } finally { + await rm(parent, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/e2eTest/setup.ts b/e2eTest/setup.ts index 1772d4e2d1..f79d38b96a 100644 --- a/e2eTest/setup.ts +++ b/e2eTest/setup.ts @@ -43,10 +43,12 @@ export async function cleanupStaleStacks(cfn: CloudFormationClient): Promise Date: Mon, 26 Oct 2026 18:01:58 +0000 Subject: [PATCH 2/2] test(security): discover generated templates automatically --- .../generated-artifact-security.test.ts | 44 +++++++++---------- .../bedrock-managed-agents/Dockerfile | 3 +- .../project/templates/pythonSecurity.test.ts | 6 +++ 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/e2eTest/project/generated-artifact-security.test.ts b/e2eTest/project/generated-artifact-security.test.ts index 64015aaeb0..408b8dd7b1 100644 --- a/e2eTest/project/generated-artifact-security.test.ts +++ b/e2eTest/project/generated-artifact-security.test.ts @@ -6,11 +6,15 @@ import { join, resolve } from "node:path"; import { describe, expect, test } from "vitest"; import { parse } from "yaml"; import z from "zod"; -import { RUNTIME_TEMPLATE_SHORTCUTS } from "../../src/handlers/project/shortcuts"; +import { + RUNTIME_TEMPLATE_SHORTCUT_NAMES, + RUNTIME_TEMPLATE_SHORTCUTS, + type RuntimeTemplateShortcutName, +} from "../../src/handlers/project/shortcuts"; import { CliRunner, parseResult, type RunResult } from "../helpers/run"; type ScanCase = { - template: string; + template: RuntimeTemplateShortcutName; projectName: string; container: boolean; }; @@ -21,20 +25,20 @@ type VulnerabilityException = { expired_at?: unknown; }; -const SCAN_CASES: readonly ScanCase[] = [ - { template: "agent-python-minimal", projectName: "ScanPythonMinimal", container: false }, - { template: "agent-python-strands", projectName: "ScanPythonStrands", container: false }, - { - template: "agent-python-strands-container", - projectName: "ScanStrandsContainer", - container: true, +const SCAN_CASES: readonly ScanCase[] = RUNTIME_TEMPLATE_SHORTCUT_NAMES.flatMap( + (template, index) => { + const metadata = RUNTIME_TEMPLATE_SHORTCUTS[template]; + return metadata.language === "Python" + ? [ + { + template, + projectName: `SecurityScan${index}`, + container: metadata.build === "Container", + }, + ] + : []; }, - { template: "agent-python-langchain", projectName: "ScanPythonLangchain", container: false }, - { template: "mcp-python-fastmcp", projectName: "ScanPythonFastMcp", container: false }, - { template: "a2a-python-strands", projectName: "ScanPythonA2a", container: false }, - { template: "agui-python-strands", projectName: "ScanPythonAgui", container: false }, - { template: "environment-python-bma", projectName: "ScanPythonBma", container: true }, -] as const; +); const ProjectCreatedSchema = z.object({ project: z.object({ path: z.string() }), @@ -108,15 +112,7 @@ async function validateExceptions(): Promise { describe.runIf(process.platform === "linux")("generated artifact vulnerability scanning", () => { const cli = new CliRunner(); - test("matrix covers every supported Python template", () => { - const supported = Object.entries(RUNTIME_TEMPLATE_SHORTCUTS) - .filter(([, template]) => template.language === "Python") - .map(([name]) => name) - .sort(); - expect(SCAN_CASES.map(({ template }) => template).sort()).toEqual(supported); - }); - - test.each(SCAN_CASES)( + test.concurrent.each(SCAN_CASES)( "$template", { timeout: 15 * 60 * 1000 }, async ({ template, projectName, container }) => { diff --git a/src/assets/templates/bedrock-managed-agents/Dockerfile b/src/assets/templates/bedrock-managed-agents/Dockerfile index 249fd02775..1fe2cf198c 100644 --- a/src/assets/templates/bedrock-managed-agents/Dockerfile +++ b/src/assets/templates/bedrock-managed-agents/Dockerfile @@ -1,7 +1,8 @@ FROM public.ecr.aws/lambda/microvms:al2023-minimal # OS setup -RUN dnf install -y tar gzip findutils shadow-utils ca-certificates \ +RUN dnf upgrade --releasever=latest --refresh -y \ + && dnf install -y tar gzip findutils shadow-utils ca-certificates \ && dnf clean all \ && useradd -m -u 1000 app diff --git a/src/core/project/templates/pythonSecurity.test.ts b/src/core/project/templates/pythonSecurity.test.ts index 5309a371be..a55d53aee9 100644 --- a/src/core/project/templates/pythonSecurity.test.ts +++ b/src/core/project/templates/pythonSecurity.test.ts @@ -51,3 +51,9 @@ test("the Strands container updates OS packages and removes uv after installing /RUN uv sync --frozen --no-dev && \/usr\/local\/bin\/python -m pip uninstall -y uv[\s\S]*USER bedrock_agentcore/, ); }); + +test("the Bedrock Managed Agents container updates OS packages", async () => { + const dockerfile = await source.read("templates/bedrock-managed-agents/Dockerfile"); + + expect(dockerfile).toMatch(/RUN dnf upgrade --releasever=latest --refresh -y[\s\S]*USER app/); +});