diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 90229ffd3..2889e77c0 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -10,7 +10,7 @@ Orchestrators (ci.yml, canary.yml, canary-v0.yml, prerelease.yml, release-prepar release.yml, slack-issue-notification.yml, slack-open-prs-notification.yml, workflow-metrics.yml) -Jobs (verify.yml, e2e-test.yml) +Jobs (verify.yml, e2e-test.yml, generated-artifact-security.yml) ``` **Orchestrators** respond to events and coordinate work. They define _when_ and @@ -29,6 +29,7 @@ An orchestrator calls jobs via `workflow_call`. ci.yml |-- verify.yml (one job per platform: static checks on Linux, then bundle, | package, compile, smoke test and unit tests) + |-- generated-artifact-security.yml (fresh Python lockfiles and ARM64 image scans) `-- e2e-test.yml (deploys and invokes real AgentCore resources) canary.yml (scheduled) @@ -43,12 +44,13 @@ canary-v0.yml (scheduled) release-prepare.yml (version bump, vended CDK pin, release PR) release-main-and-preview.yml (npm publish and GitHub release when a release PR merges) - `-- verify.yml + |-- verify.yml + `-- generated-artifact-security.yml ``` -`verify.yml` and `e2e-test.yml` each appear in two orchestrators. Push CI builds -the CLI from source for E2E, while the canary installs the resolved npm version. -Both use the same E2E test runner. +Each reusable job appears in multiple orchestrators. Push CI builds the CLI from +source for E2E, while the canary installs the resolved npm version. Both use the +same E2E test runner. ## Naming Convention diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 559412620..0c612ab08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,13 @@ concurrency: cancel-in-progress: true jobs: + generated-artifact-security: + uses: ./.github/workflows/generated-artifact-security.yml + permissions: + contents: read + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + verify: uses: ./.github/workflows/verify.yml permissions: diff --git a/.github/workflows/generated-artifact-security.yml b/.github/workflows/generated-artifact-security.yml new file mode 100644 index 000000000..61878a7d7 --- /dev/null +++ b/.github/workflows/generated-artifact-security.yml @@ -0,0 +1,63 @@ +# Generates every supported Python project through the released CLI path, resolves fresh lockfiles, +# and scans both those dependencies and generated ARM64 container images. +name: generated-artifact-security +on: + workflow_call: + inputs: + ref: + required: true + type: string + workflow_dispatch: + schedule: + - cron: "23 7 * * 1" + +permissions: + contents: read + +env: + AGENTCORE_TELEMETRY_DISABLED: "1" + +jobs: + scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + - uses: oven-sh/setup-bun@v2 + - uses: actions/setup-node@v6 + with: + node-version: "24" + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.14" + - run: bun install --frozen-lockfile + - run: bun run build + + - name: Enable ARM64 container builds + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + platforms: arm64 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Install Trivy + uses: aquasecurity/setup-trivy@81e514348e19b6112ce2a7e3ecbafe19c1e1f567 # v0.3.1 + with: + version: v0.76.1 + cache: true + + - name: Scan generated projects and images + env: + AGENTCORE_CLI_PATH: node ${{ github.workspace }}/dist/index.js + GENERATED_ARTIFACT_SECURITY_REPORT_DIR: ${{ github.workspace }}/out/generated-artifact-security/reports + run: bun run test:e2e:generated-artifact-security + + - name: Upload scan reports + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: generated-artifact-security-${{ github.sha }} + path: out/generated-artifact-security + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/release-main-and-preview.yml b/.github/workflows/release-main-and-preview.yml index 2d8a10166..7a3cb7be7 100644 --- a/.github/workflows/release-main-and-preview.yml +++ b/.github/workflows/release-main-and-preview.yml @@ -51,8 +51,17 @@ jobs: ref: ${{ github.sha }} macos: false + generated-artifact-security: + needs: check-release + if: needs.check-release.outputs.is_release == 'true' + uses: ./.github/workflows/generated-artifact-security.yml + permissions: + contents: read + with: + ref: ${{ github.sha }} + publish: - needs: verify + needs: [verify, generated-artifact-security] # npm provenance is only issued from GitHub-hosted runners. runs-on: ubuntu-latest environment: diff --git a/e2eTest/README.md b/e2eTest/README.md index ea7f54dd9..25dea6eba 100644 --- a/e2eTest/README.md +++ b/e2eTest/README.md @@ -15,6 +15,13 @@ To run tagged tests: bun run test:e2e -- --tagsFilter='runtime || canary' ``` +The generated-project vulnerability E2E is Linux-only and additionally requires Docker with +ARM64 build support and Trivy: + +```sh +bun run test:e2e:generated-artifact-security +``` + Set `AGENTCORE_CLI_PATH` to use a different executable. The value is split on whitespace and is not run through a shell, so paths containing spaces, quoting, and shell syntax are not supported: diff --git a/e2eTest/project/generated-artifact-security.test.ts b/e2eTest/project/generated-artifact-security.test.ts new file mode 100644 index 000000000..408b8dd7b --- /dev/null +++ b/e2eTest/project/generated-artifact-security.test.ts @@ -0,0 +1,233 @@ +import { createHash } from "node:crypto"; +import { spawn } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, test } from "vitest"; +import { parse } from "yaml"; +import z from "zod"; +import { + RUNTIME_TEMPLATE_SHORTCUT_NAMES, + RUNTIME_TEMPLATE_SHORTCUTS, + type RuntimeTemplateShortcutName, +} from "../../src/handlers/project/shortcuts"; +import { CliRunner, parseResult, type RunResult } from "../helpers/run"; + +type ScanCase = { + template: RuntimeTemplateShortcutName; + projectName: string; + container: boolean; +}; + +type VulnerabilityException = { + id?: unknown; + statement?: unknown; + expired_at?: unknown; +}; + +const SCAN_CASES: readonly ScanCase[] = RUNTIME_TEMPLATE_SHORTCUT_NAMES.flatMap( + (template, index) => { + const metadata = RUNTIME_TEMPLATE_SHORTCUTS[template]; + return metadata.language === "Python" + ? [ + { + template, + projectName: `SecurityScan${index}`, + container: metadata.build === "Container", + }, + ] + : []; + }, +); + +const ProjectCreatedSchema = z.object({ + project: z.object({ path: z.string() }), +}); + +const repositoryRoot = resolve("."); +const reportRoot = resolve( + process.env.GENERATED_ARTIFACT_SECURITY_REPORT_DIR ?? "out/generated-artifact-security/reports", +); +const exceptionsPath = join( + repositoryRoot, + "security", + "generated-artifact-vulnerability-exceptions.yaml", +); + +function run(command: readonly string[], cwd: string): Promise { + return new Promise((resolveResult, reject) => { + const [executable, ...args] = command; + if (!executable) throw new Error("cannot run an empty command"); + const child = spawn(executable, args, { + cwd, + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (exitCode) => resolveResult({ stdout, stderr, exitCode: exitCode ?? -1 })); + }); +} + +async function runSuccessfully(command: readonly string[], cwd: string): Promise { + const result = await run(command, cwd); + expect( + result.exitCode, + `${command.join(" ")} failed\nstdout: ${result.stdout}\nstderr: ${result.stderr}`, + ).toBe(0); + return result; +} + +async function validateExceptions(): Promise { + const document = parse(await readFile(exceptionsPath, "utf8")) as { + vulnerabilities?: unknown; + } | null; + const vulnerabilities = document?.vulnerabilities; + expect(Array.isArray(vulnerabilities)).toBe(true); + if (!Array.isArray(vulnerabilities)) throw new Error("exceptions must be a list"); + + const today = new Date().toISOString().slice(0, 10); + for (const exception of vulnerabilities as VulnerabilityException[]) { + expect(exception.id, "every exception must identify a vulnerability").toBeTypeOf("string"); + expect(exception.statement, `${String(exception.id)} must document its rationale`).toBeTypeOf( + "string", + ); + expect( + (exception.statement as string).trim().length, + `${String(exception.id)} must document a non-empty rationale`, + ).toBeGreaterThan(0); + expect(exception.expired_at, `${String(exception.id)} must have an expiration`).toMatch( + /^\d{4}-\d{2}-\d{2}$/, + ); + expect( + (exception.expired_at as string) >= today, + `${String(exception.id)} expired on ${String(exception.expired_at)}`, + ).toBe(true); + } +} + +describe.runIf(process.platform === "linux")("generated artifact vulnerability scanning", () => { + const cli = new CliRunner(); + + test.concurrent.each(SCAN_CASES)( + "$template", + { timeout: 15 * 60 * 1000 }, + async ({ template, projectName, container }) => { + await validateExceptions(); + await mkdir(reportRoot, { recursive: true }); + const parent = await mkdtemp(join(tmpdir(), "agentcore-security-e2e-")); + try { + const created = parseResult( + ProjectCreatedSchema, + await cli.run( + [ + "create", + "--name", + projectName, + "--template", + template, + "--skip-install", + "--skip-git", + "--region", + "us-west-2", + "--json", + ], + parent, + ), + ); + const runtimeRoot = join(created.project.path, "app", "agent"); + await runSuccessfully(["uv", "lock", "--refresh"], runtimeRoot); + + const dependencyReport = join(reportRoot, `${template}-dependencies.json`); + await runSuccessfully( + [ + "trivy", + "fs", + "--scanners", + "vuln", + "--severity", + "HIGH,CRITICAL", + "--ignore-unfixed", + "--show-suppressed", + "--exit-code", + "1", + "--format", + "json", + "--output", + dependencyReport, + "--ignorefile", + exceptionsPath, + runtimeRoot, + ], + repositoryRoot, + ); + + const metadata: Record = { + template, + architecture: container ? "linux/arm64" : undefined, + lockfileSha256: createHash("sha256") + .update(await readFile(join(runtimeRoot, "uv.lock"))) + .digest("hex"), + dependencyReport, + }; + if (container) { + const imageName = `agentcore-generated-scan:${template}`; + await runSuccessfully( + [ + "docker", + "buildx", + "build", + "--platform", + "linux/arm64", + "--pull", + "--no-cache", + "--load", + "--tag", + imageName, + runtimeRoot, + ], + repositoryRoot, + ); + const image = await runSuccessfully( + ["docker", "image", "inspect", "--format", "{{.Id}}", imageName], + repositoryRoot, + ); + const imageReport = join(reportRoot, `${template}-image.json`); + await runSuccessfully( + [ + "trivy", + "image", + "--scanners", + "vuln", + "--severity", + "HIGH,CRITICAL", + "--ignore-unfixed", + "--show-suppressed", + "--exit-code", + "1", + "--format", + "json", + "--output", + imageReport, + "--ignorefile", + exceptionsPath, + imageName, + ], + repositoryRoot, + ); + metadata.imageDigest = image.stdout.trim(); + metadata.imageReport = imageReport; + } + await writeFile( + join(reportRoot, `${template}-metadata.json`), + `${JSON.stringify(metadata, null, 2)}\n`, + ); + } finally { + await rm(parent, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/e2eTest/setup.ts b/e2eTest/setup.ts index 1772d4e2d..f79d38b96 100644 --- a/e2eTest/setup.ts +++ b/e2eTest/setup.ts @@ -43,10 +43,12 @@ export async function cleanupStaleStacks(cfn: CloudFormationClient): Promise { + const dockerfile = await source.read("templates/bedrock-managed-agents/Dockerfile"); + + expect(dockerfile).toMatch(/RUN dnf upgrade --releasever=latest --refresh -y[\s\S]*USER app/); +});