-
-
{item.suite.name}
-
+
+
+ {item.suite.name}
+
+ {issues > 0 && (
+
+ Needs attention
+
+ )}
+ {!editable && (
+
+ Defined in code
+
+ )}
+
+ {item.suite.description && (
+
+ {item.suite.description}
+
+ )}
{issues > 0 && (
-
- Needs attention
-
+
+ {item.entry_statuses.flatMap((entryStatus, entryIndex) => (
+ entryStatus.issues.map((issue) => (
+
+
+ Entry {entryIndex + 1} ({entryStatus.preset_name}): {issue.message}
+
+
+ ))
+ ))}
+
)}
- {item.suite.description && (
-
- {item.suite.description}
-
- )}
- {issues > 0 && (
-
- {item.entry_statuses.flatMap((entryStatus, entryIndex) => (
- entryStatus.issues.map((issue) => (
-
-
- Entry {entryIndex + 1} ({entryStatus.preset_name}): {issue.message}
-
-
- ))
- ))}
-
- )}
@@ -229,7 +266,9 @@ export default function ScenarioSuiteLibrary() {
- {issues === 0
+ {item.suite.entries.length === 0
+ ? Empty suite
+ : issues === 0
? No advisory issues
: {issues} advisory issue{issues === 1 ? '' : 's'} }
@@ -240,6 +279,16 @@ export default function ScenarioSuiteLibrary() {
+ navigate(suiteEditorRoutePath(item.suite.name))}
data-testid={`edit-suite-${item.suite.name}`}
>
- Edit
+ {editable ? 'Edit' : 'View'}
)}
+ {launching && (
+ setLaunching(null)}
+ onSuiteChanged={() => {
+ const changedName = launching.suite.name
+ setLaunching(null)
+ refresh()
+ setActionError(
+ `Suite '${changedName}' or one of its presets changed while you were launching it. `
+ + 'Review it and launch again.',
+ )
+ }}
+ />
+ )}
+
{pendingDelete && (
{
)
})
+ it('links child runs back to their transient suite execution', () => {
+ renderPage(
+ `/scanner-history/${SCENARIO_RESULT_ID}`,
+ {
+ suiteExecutionId: 'execution-1',
+ suiteName: 'nightly_suite',
+ },
+ )
+
+ expect(screen.getByRole('link', { name: 'Back to nightly_suite' })).toHaveAttribute(
+ 'href',
+ '/registry/scenario-suites/executions/execution-1',
+ )
+ })
+
it('renders one expandable parent for attacks that share a display group', async () => {
const user = userEvent.setup()
const secondAttempt = {
diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.tsx
index 4376a5b301..0c96338a58 100644
--- a/frontend/src/components/Scenarios/ScenarioRunPage.tsx
+++ b/frontend/src/components/Scenarios/ScenarioRunPage.tsx
@@ -44,6 +44,7 @@ import {
formatTimestamp,
objectivePreview,
} from '@/components/AttackResults/attackAttemptFormatting'
+import { suiteExecutionRoutePath } from '@/components/ScenarioSuites/suiteRoutes'
import { useScenarioRunProgress } from '@/hooks/useScenarioRunProgress'
import { useScenarioRunResume } from '@/hooks/useScenarioRunResume'
import { useScenarioQueue } from '@/hooks/useScenarioQueue'
@@ -132,13 +133,19 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu
fromScenarioHistory?: boolean
scenarioHistorySearch?: string
scenarioName?: string
+ suiteExecutionId?: string
+ suiteName?: string
} | null
- const backPath = navigationState?.fromScenarioHistory
+ const backPath = navigationState?.suiteExecutionId
+ ? suiteExecutionRoutePath(navigationState.suiteExecutionId)
+ : navigationState?.fromScenarioHistory
? `/history/scanner${navigationState.scenarioHistorySearch ?? ''}`
: navigationState?.scenarioName
? `/scanner/${encodeURIComponent(navigationState.scenarioName)}`
: '/scanner'
- const backLabel = navigationState?.fromScenarioHistory
+ const backLabel = navigationState?.suiteExecutionId
+ ? `Back to ${navigationState.suiteName ?? 'suite execution'}`
+ : navigationState?.fromScenarioHistory
? 'Back to scanner history'
: navigationState?.scenarioName
? 'Back to scenario'
diff --git a/frontend/src/hooks/useScenarioSuiteExecution.test.tsx b/frontend/src/hooks/useScenarioSuiteExecution.test.tsx
new file mode 100644
index 0000000000..5a0510e04e
--- /dev/null
+++ b/frontend/src/hooks/useScenarioSuiteExecution.test.tsx
@@ -0,0 +1,161 @@
+import { act, render, screen, waitFor } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+
+import { scenarioSuitesApi } from '@/services/api'
+import type {
+ ScenarioSuiteExecutionStatus,
+ ScenarioSuiteExecutionSummary,
+} from '@/types'
+
+import {
+ SCENARIO_SUITE_EXECUTION_POLL_INTERVAL_MS,
+ useScenarioSuiteExecution,
+} from './useScenarioSuiteExecution'
+
+jest.mock('@/services/api', () => ({
+ scenarioSuitesApi: {
+ getExecution: jest.fn(),
+ },
+}))
+
+const mockGetExecution = scenarioSuitesApi.getExecution as jest.Mock
+
+function makeExecution(
+ status: ScenarioSuiteExecutionStatus = 'IN_PROGRESS',
+): ScenarioSuiteExecutionSummary {
+ return {
+ suite_execution_id: 'execution-1',
+ suite_name: 'nightly_suite',
+ suite_origin: 'stored',
+ suite_version: 'suite-v1',
+ status,
+ created_at: '2026-10-23T12:00:00Z',
+ updated_at: '2026-10-23T12:01:00Z',
+ entries: [],
+ }
+}
+
+function Harness({ executionId = 'execution-1' }: { readonly executionId?: string }) {
+ const { state, retry, applyExecution } = useScenarioSuiteExecution(executionId)
+ return (
+ <>
+ {state.loadStatus}
+ {state.execution?.status ?? 'none'}
+ {state.error ?? ''}
+ {String(state.stale)}
+ Retry
+ applyExecution(makeExecution('CANCELLED'))}>
+ Apply cancelled
+
+
+ )
+}
+
+describe('useScenarioSuiteExecution', () => {
+ beforeEach(() => {
+ jest.useFakeTimers()
+ jest.clearAllMocks()
+ })
+
+ afterEach(() => {
+ jest.useRealTimers()
+ })
+
+ it('polls until the execution reaches a terminal state', async () => {
+ mockGetExecution
+ .mockResolvedValueOnce(makeExecution('IN_PROGRESS'))
+ .mockResolvedValueOnce(makeExecution('COMPLETED'))
+
+ render( = new Set([
+ 'COMPLETED',
+ 'FAILED',
+ 'CANCELLED',
+])
+
+export function isTerminalSuiteExecutionStatus(status: ScenarioSuiteExecutionStatus): boolean {
+ return TERMINAL_STATUSES.has(status)
+}
+
+export function useScenarioSuiteExecution(
+ executionId: string,
+): UseScenarioSuiteExecutionResult {
+ const executionIdValid = executionId.trim() !== ''
+ const [state, setState] = useState(() => (
+ executionIdValid
+ ? {
+ loadStatus: 'loading',
+ execution: null,
+ error: null,
+ stale: false,
+ }
+ : {
+ loadStatus: 'not-found',
+ execution: null,
+ error: 'The scenario suite execution ID in this URL is missing or invalid.',
+ stale: false,
+ }
+ ))
+ const [retryEpoch, setRetryEpoch] = useState(0)
+ const timerRef = useRef | null>(null)
+ const controllerRef = useRef(null)
+ const latestExecutionRef = useRef(null)
+ const pollingStoppedRef = useRef(false)
+ const retriedRef = useRef(false)
+
+ useEffect(() => {
+ let active = true
+ pollingStoppedRef.current = false
+ retriedRef.current = false
+
+ const clearPollTimer = (): void => {
+ if (timerRef.current !== null) {
+ clearTimeout(timerRef.current)
+ timerRef.current = null
+ }
+ }
+
+ const schedulePoll = (): void => {
+ clearPollTimer()
+ timerRef.current = setTimeout(() => {
+ timerRef.current = null
+ void fetchExecutionAsync()
+ }, SCENARIO_SUITE_EXECUTION_POLL_INTERVAL_MS)
+ }
+
+ const fetchExecutionAsync = async (): Promise => {
+ if (!active || pollingStoppedRef.current) {
+ return
+ }
+ const controller = new AbortController()
+ controllerRef.current = controller
+ try {
+ const execution = await scenarioSuitesApi.getExecution(executionId, controller.signal)
+ if (!active || pollingStoppedRef.current) {
+ return
+ }
+ latestExecutionRef.current = execution
+ retriedRef.current = false
+ setState({ loadStatus: 'ready', execution, error: null, stale: false })
+ if (isTerminalSuiteExecutionStatus(execution.status)) {
+ pollingStoppedRef.current = true
+ return
+ }
+ schedulePoll()
+ } catch (error: unknown) {
+ if (!active || controller.signal.aborted) {
+ return
+ }
+ const apiError = toApiError(error)
+ if (apiError.status !== 404 && !retriedRef.current) {
+ retriedRef.current = true
+ schedulePoll()
+ return
+ }
+ const previous = latestExecutionRef.current
+ setState({
+ loadStatus: apiError.status === 404 && previous === null
+ ? 'not-found'
+ : previous === null
+ ? 'error'
+ : 'ready',
+ execution: previous,
+ error: apiError.detail,
+ stale: previous !== null,
+ })
+ if (
+ apiError.status !== 404
+ && previous !== null
+ && !isTerminalSuiteExecutionStatus(previous.status)
+ ) {
+ schedulePoll()
+ }
+ }
+ }
+
+ if (executionIdValid) {
+ void fetchExecutionAsync()
+ }
+
+ return () => {
+ active = false
+ clearPollTimer()
+ controllerRef.current?.abort()
+ controllerRef.current = null
+ }
+ }, [executionId, executionIdValid, retryEpoch])
+
+ const retry = useCallback((): void => {
+ pollingStoppedRef.current = false
+ retriedRef.current = false
+ setState((previous) => ({
+ ...previous,
+ loadStatus: previous.execution ? 'ready' : 'loading',
+ error: null,
+ stale: false,
+ }))
+ setRetryEpoch((epoch) => epoch + 1)
+ }, [])
+
+ const applyExecution = useCallback((execution: ScenarioSuiteExecutionSummary): void => {
+ latestExecutionRef.current = execution
+ setState({ loadStatus: 'ready', execution, error: null, stale: false })
+ if (isTerminalSuiteExecutionStatus(execution.status)) {
+ pollingStoppedRef.current = true
+ if (timerRef.current !== null) {
+ clearTimeout(timerRef.current)
+ timerRef.current = null
+ }
+ controllerRef.current?.abort()
+ controllerRef.current = null
+ }
+ }, [])
+
+ return { state, retry, applyExecution }
+}
diff --git a/frontend/src/services/api.test.ts b/frontend/src/services/api.test.ts
index e12d0c483b..b7d5a0e384 100644
--- a/frontend/src/services/api.test.ts
+++ b/frontend/src/services/api.test.ts
@@ -135,12 +135,23 @@ describe("api service", () => {
});
describe("scenarioSuitesApi", () => {
- it("supports collection and encoded item CRUD with optimistic concurrency", async () => {
+ it("supports definition CRUD and transient execution APIs with encoded identifiers", async () => {
const suite = { name: "nightly_suite", schema_version: 1, entries: [] };
- const response = { suite, version: "v1", entry_statuses: [] };
+ const response = { suite, origin: "stored", version: "v1", entry_statuses: [] };
+ const launch = {
+ expected_version: "v1",
+ entries: [],
+ target_name: "objective",
+ };
+ const execution = { suite_execution_id: "execution/id" };
const controller = new AbortController();
- (apiClient.get as jest.Mock).mockResolvedValue({ data: response });
- (apiClient.post as jest.Mock).mockResolvedValue({ data: response });
+ (apiClient.get as jest.Mock)
+ .mockResolvedValueOnce({ data: response })
+ .mockResolvedValueOnce({ data: response })
+ .mockResolvedValueOnce({ data: execution });
+ (apiClient.post as jest.Mock)
+ .mockResolvedValueOnce({ data: response })
+ .mockResolvedValue({ data: execution });
(apiClient.put as jest.Mock).mockResolvedValue({ data: response });
(apiClient.delete as jest.Mock).mockResolvedValue({ data: undefined });
@@ -157,6 +168,18 @@ describe("api service", () => {
});
await scenarioSuitesApi.remove("a/b");
expect(apiClient.delete).toHaveBeenCalledWith("/scenario-suites/a%2Fb");
+ await expect(scenarioSuitesApi.launch("a/b", launch)).resolves.toEqual(execution);
+ expect(apiClient.post).toHaveBeenCalledWith("/scenario-suites/a%2Fb/launch", launch);
+ await expect(scenarioSuitesApi.getExecution("execution/id", controller.signal)).resolves.toEqual(execution);
+ expect(apiClient.get).toHaveBeenCalledWith("/scenario-suites/executions/execution%2Fid", {
+ signal: controller.signal,
+ });
+ await expect(scenarioSuitesApi.cancelExecution("execution/id", controller.signal)).resolves.toEqual(execution);
+ expect(apiClient.post).toHaveBeenCalledWith(
+ "/scenario-suites/executions/execution%2Fid/cancel",
+ undefined,
+ { signal: controller.signal },
+ );
});
});
diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts
index 87ab32d7ac..20b9a01de2 100644
--- a/frontend/src/services/api.ts
+++ b/frontend/src/services/api.ts
@@ -52,6 +52,8 @@ import type {
ScenarioPresetListResponse,
ScenarioPresetResponse,
ScenarioSuite,
+ LaunchScenarioSuiteRequest,
+ ScenarioSuiteExecutionSummary,
ScenarioSuiteListResponse,
ScenarioSuiteResponse,
UpdateScenarioPresetRequest,
@@ -829,6 +831,40 @@ export const scenarioSuitesApi = {
remove: async (name: string): Promise => {
await apiClient.delete(`/scenario-suites/${encodeURIComponent(name)}`)
},
+
+ launch: async (
+ name: string,
+ request: LaunchScenarioSuiteRequest,
+ ): Promise => {
+ const response = await apiClient.post(
+ `/scenario-suites/${encodeURIComponent(name)}/launch`,
+ request,
+ )
+ return response.data
+ },
+
+ getExecution: async (
+ executionId: string,
+ signal?: AbortSignal,
+ ): Promise => {
+ const response = await apiClient.get(
+ `/scenario-suites/executions/${encodeURIComponent(executionId)}`,
+ { signal },
+ )
+ return response.data
+ },
+
+ cancelExecution: async (
+ executionId: string,
+ signal?: AbortSignal,
+ ): Promise => {
+ const response = await apiClient.post(
+ `/scenario-suites/executions/${encodeURIComponent(executionId)}/cancel`,
+ undefined,
+ { signal },
+ )
+ return response.data
+ },
}
const MAX_SEED_SEARCH_LENGTH = 1_000
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts
index aba995ca0c..a9d5a52ef9 100644
--- a/frontend/src/types/index.ts
+++ b/frontend/src/types/index.ts
@@ -1428,8 +1428,13 @@ export interface ScenarioSuiteEntryStatus {
export interface ScenarioSuiteResponse {
suite: ScenarioSuite
- /** Opaque document version required to update the suite. */
- version: string
+ /**
+ * Whether a stored document or registering code contributed this suite. A
+ * registered suite cannot be updated or deleted through the API.
+ */
+ origin: 'stored' | 'registered'
+ /** Opaque document version required to update a stored suite. Null when registered in code. */
+ version: string | null
/** Ordered exactly like `suite.entries`. */
entry_statuses: ScenarioSuiteEntryStatus[]
}
@@ -1444,6 +1449,73 @@ export interface UpdateScenarioSuiteRequest {
expected_version: string
}
+/** The preset state the operator reviewed for one suite entry. */
+export interface ScenarioSuiteLaunchEntrySnapshot {
+ entry_id: string
+ preset_name: string
+ origin: 'stored' | 'registered' | null
+ preset_version: string | null
+}
+
+/** Launch-owned fields and optimistic preconditions for one suite execution. */
+export interface LaunchScenarioSuiteRequest {
+ expected_version: string | null
+ entries: ScenarioSuiteLaunchEntrySnapshot[]
+ target_name: string
+ adversarial_target_name?: string | null
+ initializers?: string[] | null
+ initializer_args?: Record> | null
+ max_concurrency?: number | null
+ max_retries?: number | null
+ operation_id?: string | null
+ labels?: Record | null
+}
+
+export type ScenarioSuiteExecutionStatus =
+ | 'QUEUING'
+ | 'IN_PROGRESS'
+ | 'COMPLETED'
+ | 'FAILED'
+ | 'CANCELLED'
+
+export type ScenarioSuiteExecutionEntryState =
+ | 'PENDING'
+ | 'SUBMITTING'
+ | 'SUBMISSION_FAILED'
+ | 'CREATED'
+ | 'QUEUED'
+ | 'IN_PROGRESS'
+ | 'COMPLETED'
+ | 'FAILED'
+ | 'CANCELLED'
+
+/** Current process-local state for one ordered suite entry. */
+export interface ScenarioSuiteExecutionEntrySummary {
+ entry_id: string
+ entry_position: number
+ preset_name: string
+ preset_origin: 'stored' | 'registered'
+ preset_version: string | null
+ state: ScenarioSuiteExecutionEntryState
+ run: ScenarioRunSummary | null
+ /** User-safe failure text. The UI still maps it to state-owned copy before display. */
+ error: string | null
+ /** Internal exception class for diagnostics; never display it to operators. */
+ error_type: string | null
+}
+
+/** Transient progress and grouped child runs for one suite launch. */
+export interface ScenarioSuiteExecutionSummary {
+ suite_execution_id: string
+ suite_name: string
+ suite_origin: 'stored' | 'registered'
+ suite_version: string | null
+ status: ScenarioSuiteExecutionStatus
+ created_at: string
+ updated_at: string
+ entries: ScenarioSuiteExecutionEntrySummary[]
+}
+
/** The launch-owned fields a preset deliberately omits. */
export interface ResolveScenarioPresetRequest {
expected_version?: string | null
diff --git a/pyrit/backend/models/__init__.py b/pyrit/backend/models/__init__.py
index 9b573fdc47..711e4270b9 100644
--- a/pyrit/backend/models/__init__.py
+++ b/pyrit/backend/models/__init__.py
@@ -71,7 +71,13 @@
UpdateScenarioPresetRequest,
)
from pyrit.backend.models.scenario_suites import (
+ LaunchScenarioSuiteRequest,
ScenarioSuiteEntryStatus,
+ ScenarioSuiteExecutionEntryState,
+ ScenarioSuiteExecutionEntrySummary,
+ ScenarioSuiteExecutionStatus,
+ ScenarioSuiteExecutionSummary,
+ ScenarioSuiteLaunchEntrySnapshot,
ScenarioSuiteListResponse,
ScenarioSuiteResponse,
UpdateScenarioSuiteRequest,
@@ -132,6 +138,12 @@
"ScenarioPresetListResponse": "pyrit.backend.models.scenario_presets",
"ScenarioPresetResponse": "pyrit.backend.models.scenario_presets",
"UpdateScenarioPresetRequest": "pyrit.backend.models.scenario_presets",
+ "LaunchScenarioSuiteRequest": "pyrit.backend.models.scenario_suites",
+ "ScenarioSuiteExecutionEntryState": "pyrit.backend.models.scenario_suites",
+ "ScenarioSuiteExecutionEntrySummary": "pyrit.backend.models.scenario_suites",
+ "ScenarioSuiteExecutionStatus": "pyrit.backend.models.scenario_suites",
+ "ScenarioSuiteExecutionSummary": "pyrit.backend.models.scenario_suites",
+ "ScenarioSuiteLaunchEntrySnapshot": "pyrit.backend.models.scenario_suites",
"ScenarioSuiteEntryStatus": "pyrit.backend.models.scenario_suites",
"ScenarioSuiteListResponse": "pyrit.backend.models.scenario_suites",
"ScenarioSuiteResponse": "pyrit.backend.models.scenario_suites",
diff --git a/pyrit/backend/models/scenario_suites.py b/pyrit/backend/models/scenario_suites.py
index 4ffa70d2c0..91cd9c6c81 100644
--- a/pyrit/backend/models/scenario_suites.py
+++ b/pyrit/backend/models/scenario_suites.py
@@ -1,17 +1,27 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
-"""REST envelopes for scenario suite definition endpoints."""
+"""REST envelopes for scenario suite definition and execution endpoints."""
-from typing import Literal
+from datetime import datetime
+from enum import Enum
+from typing import Any, Literal
from uuid import UUID
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
+from pyrit.models.catalog.scenario import ScenarioRunSummary
from pyrit.models.catalog.scenario_preset import PresetIssue
from pyrit.models.catalog.scenario_suite import ScenarioSuite
+from pyrit.models.request_limits import MAX_ITEMS
__all__ = [
+ "LaunchScenarioSuiteRequest",
+ "ScenarioSuiteExecutionEntryState",
+ "ScenarioSuiteExecutionEntrySummary",
+ "ScenarioSuiteExecutionStatus",
+ "ScenarioSuiteExecutionSummary",
+ "ScenarioSuiteLaunchEntrySnapshot",
"ScenarioSuiteEntryStatus",
"ScenarioSuiteListResponse",
"ScenarioSuiteResponse",
@@ -20,7 +30,7 @@
class ScenarioSuiteEntryStatus(BaseModel):
- """Current deployment-specific status for one stored suite entry."""
+ """Current deployment-specific status for one suite entry."""
entry_id: UUID = Field(..., description="Stable identifier of the suite entry this status describes")
preset_name: str = Field(..., description="Referenced scenario preset name")
@@ -39,10 +49,23 @@ class ScenarioSuiteEntryStatus(BaseModel):
class ScenarioSuiteResponse(BaseModel):
- """A stored scenario suite, its document version, and current entry statuses."""
-
- suite: ScenarioSuite = Field(..., description="The stored scenario suite definition")
- version: str = Field(..., description="Opaque version required to update the suite")
+ """A scenario suite, its origin, optional document version, and current entry statuses."""
+
+ suite: ScenarioSuite = Field(..., description="The scenario suite definition")
+ origin: Literal["stored", "registered"] = Field(
+ "stored",
+ description=(
+ "Whether a stored document or registering code contributed this suite. A registered suite "
+ "is held in memory by the code that registered it and cannot be updated or deleted here."
+ ),
+ )
+ version: str | None = Field(
+ ...,
+ description=(
+ "Opaque version required to update a stored suite. None for a registered suite, "
+ "which has no document to guard."
+ ),
+ )
entry_statuses: list[ScenarioSuiteEntryStatus] = Field(
default_factory=list,
description="Preset origin, version, and advisory issues in the suite's entry order",
@@ -50,10 +73,10 @@ class ScenarioSuiteResponse(BaseModel):
class ScenarioSuiteListResponse(BaseModel):
- """The configured suite storage source and every readable suite."""
+ """The configured suite storage source and every readable suite from either origin."""
source: str = Field(..., description="Credential-free configured suite source")
- items: list[ScenarioSuiteResponse] = Field(..., description="Stored suites, sorted by name")
+ items: list[ScenarioSuiteResponse] = Field(..., description="Stored and registered suites, sorted by name")
class UpdateScenarioSuiteRequest(BaseModel):
@@ -61,3 +84,133 @@ class UpdateScenarioSuiteRequest(BaseModel):
suite: ScenarioSuite = Field(..., description="The replacement suite")
expected_version: str = Field(..., description="Version returned when the suite being edited was read")
+
+
+class ScenarioSuiteLaunchEntrySnapshot(BaseModel):
+ """The preset state the operator reviewed for one suite entry."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ entry_id: UUID = Field(..., description="Stable suite-entry identity")
+ preset_name: str = Field(..., description="Preset name shown for this entry")
+ origin: Literal["stored", "registered"] | None = Field(
+ ...,
+ description="Reviewed preset origin, or None when the entry was unresolved",
+ )
+ preset_version: str | None = Field(
+ ...,
+ description="Reviewed stored preset version, or None for registered or unresolved presets",
+ )
+
+
+class LaunchScenarioSuiteRequest(BaseModel):
+ """Launch-owned fields and optimistic preconditions for one suite execution."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ expected_version: str | None = Field(
+ None,
+ description="Reviewed stored suite version; None only for a suite registered in code",
+ )
+ entries: list[ScenarioSuiteLaunchEntrySnapshot] = Field(
+ ...,
+ min_length=1,
+ max_length=MAX_ITEMS,
+ description="Reviewed preset snapshots in the suite's declared order",
+ )
+ target_name: str = Field(..., min_length=1, description="Registered objective target used by every entry")
+ adversarial_target_name: str | None = Field(
+ None,
+ min_length=1,
+ description="Registered adversarial target used by entries that require one",
+ )
+ initializers: list[str] | None = Field(
+ None,
+ max_length=MAX_ITEMS,
+ description="Initializer names applied to every child scenario run",
+ )
+ initializer_args: dict[str, dict[str, Any]] | None = Field(
+ None,
+ max_length=MAX_ITEMS,
+ description="Per-initializer parameter overrides keyed by initializer name",
+ )
+ max_concurrency: int | None = Field(
+ None,
+ ge=1,
+ le=100,
+ description="Maximum concurrency applied independently to every child run",
+ )
+ max_retries: int | None = Field(
+ None,
+ ge=0,
+ le=20,
+ description="Maximum retries applied independently to every child run",
+ )
+ operation_id: UUID | None = Field(None, description="Saved Operation attributed to every child result")
+ labels: dict[str, str] | None = Field(None, description="Labels attached to every child result")
+
+
+class ScenarioSuiteExecutionStatus(str, Enum):
+ """Process-local lifecycle of one accepted suite execution."""
+
+ QUEUING = "QUEUING"
+ IN_PROGRESS = "IN_PROGRESS"
+ COMPLETED = "COMPLETED"
+ FAILED = "FAILED"
+ CANCELLED = "CANCELLED"
+
+
+class ScenarioSuiteExecutionEntryState(str, Enum):
+ """Submission or scenario-run state for one suite entry."""
+
+ PENDING = "PENDING"
+ SUBMITTING = "SUBMITTING"
+ SUBMISSION_FAILED = "SUBMISSION_FAILED"
+ CREATED = "CREATED"
+ QUEUED = "QUEUED"
+ IN_PROGRESS = "IN_PROGRESS"
+ COMPLETED = "COMPLETED"
+ FAILED = "FAILED"
+ CANCELLED = "CANCELLED"
+
+
+class ScenarioSuiteExecutionEntrySummary(BaseModel):
+ """Current process-local state for one ordered suite entry."""
+
+ entry_id: UUID = Field(..., description="Stable suite-entry identity")
+ entry_position: int = Field(..., ge=1, description="One-based position in the resolved suite")
+ preset_name: str = Field(..., description="Resolved preset name")
+ preset_origin: Literal["stored", "registered"] = Field(..., description="Resolved preset origin")
+ preset_version: str | None = Field(
+ None,
+ description="Resolved stored preset version, or None for a registered preset",
+ )
+ state: ScenarioSuiteExecutionEntryState = Field(..., description="Submission or child-run state")
+ run: ScenarioRunSummary | None = Field(None, description="Current child run summary after successful submission")
+ error: str | None = Field(
+ None,
+ description="User-safe submission or status failure message when no child run can be shown",
+ )
+ error_type: str | None = Field(
+ None,
+ description="Internal exception class name for diagnostics; clients should not display it",
+ )
+
+
+class ScenarioSuiteExecutionSummary(BaseModel):
+ """Transient progress and grouped child results for one suite launch."""
+
+ suite_execution_id: UUID = Field(..., description="Server-owned identity for this suite launch")
+ suite_name: str = Field(..., description="Resolved suite name")
+ suite_origin: Literal["stored", "registered"] = Field(..., description="Resolved suite origin")
+ suite_version: str | None = Field(
+ None,
+ description="Resolved stored suite version, or None for a suite registered in code",
+ )
+ status: ScenarioSuiteExecutionStatus = Field(..., description="Aggregate process-local execution state")
+ created_at: datetime = Field(..., description="When the suite execution was accepted")
+ updated_at: datetime = Field(..., description="Latest submission or child-run update")
+ entries: list[ScenarioSuiteExecutionEntrySummary] = Field(
+ ...,
+ description="Entry states in the suite's declared order",
+ )
diff --git a/pyrit/backend/routes/scenario_suites.py b/pyrit/backend/routes/scenario_suites.py
index 4557c790fa..fc95076360 100644
--- a/pyrit/backend/routes/scenario_suites.py
+++ b/pyrit/backend/routes/scenario_suites.py
@@ -4,20 +4,30 @@
"""Scenario suite definition API routes."""
from typing import Annotated
+from uuid import UUID
from azure.core.exceptions import AzureError
-from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi import APIRouter, Depends, HTTPException, Request, status
from pyrit.backend.middleware.auth import current_user_name, require_admin
from pyrit.backend.models.common import ProblemDetail
from pyrit.backend.models.scenario_suites import (
+ LaunchScenarioSuiteRequest,
+ ScenarioSuiteExecutionSummary,
ScenarioSuiteListResponse,
ScenarioSuiteResponse,
UpdateScenarioSuiteRequest,
)
+from pyrit.backend.services.scenario_suite_execution_service import (
+ ScenarioSuiteExecutionCapacityError,
+ ScenarioSuiteExecutionNotFoundError,
+ ScenarioSuiteLaunchConflictError,
+ ScenarioSuiteNotRunnableError,
+ get_scenario_suite_execution_service,
+)
from pyrit.backend.services.scenario_suite_service import get_scenario_suite_service
from pyrit.models.catalog import ScenarioSuite
-from pyrit.registry import ScenarioSuiteConflictError
+from pyrit.registry import ScenarioSuiteConflictError, ScenarioSuiteNameRegisteredError
router = APIRouter(prefix="/scenario-suites", tags=["scenario-suites"])
@@ -50,7 +60,7 @@ async def _load_suite_or_404_async(name: str) -> ScenarioSuiteResponse:
Read one suite, translating storage failures into HTTP responses.
Returns:
- ScenarioSuiteResponse: The stored suite and current entry statuses.
+ ScenarioSuiteResponse: The suite and current entry statuses.
Raises:
HTTPException: 404 if absent, 400 for an illegal name, or 503 if storage fails.
@@ -70,10 +80,10 @@ async def _load_suite_or_404_async(name: str) -> ScenarioSuiteResponse:
@router.get("", response_model=ScenarioSuiteListResponse)
async def list_scenario_suites() -> ScenarioSuiteListResponse: # pyrit-async-suffix-exempt
"""
- List every readable scenario suite from the configured storage source.
+ List every readable stored or registered scenario suite.
Returns:
- ScenarioSuiteListResponse: Stored suites with current entry statuses.
+ ScenarioSuiteListResponse: Suites from both origins with current entry statuses.
"""
try:
return await get_scenario_suite_service().list_suites_async()
@@ -81,6 +91,103 @@ async def list_scenario_suites() -> ScenarioSuiteListResponse: # pyrit-async-su
raise _storage_unavailable() from exc
+@router.get(
+ "/executions/{execution_id}",
+ response_model=ScenarioSuiteExecutionSummary,
+ responses={404: {"model": ProblemDetail, "description": "Transient suite execution not found"}},
+)
+async def get_scenario_suite_execution( # pyrit-async-suffix-exempt
+ execution_id: UUID,
+) -> ScenarioSuiteExecutionSummary:
+ """
+ Get process-local progress and grouped child runs for one suite execution.
+
+ Returns:
+ ScenarioSuiteExecutionSummary: Current transient suite progress.
+ """
+ execution = await get_scenario_suite_execution_service().get_execution_async(execution_id=execution_id)
+ if execution is None:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail=f"Scenario suite execution '{execution_id}' is no longer available",
+ )
+ return execution
+
+
+@router.post(
+ "/executions/{execution_id}/cancel",
+ response_model=ScenarioSuiteExecutionSummary,
+ responses={404: {"model": ProblemDetail, "description": "Transient suite execution not found"}},
+)
+async def cancel_scenario_suite_execution( # pyrit-async-suffix-exempt
+ execution_id: UUID,
+) -> ScenarioSuiteExecutionSummary:
+ """
+ Best-effort cancel pending submissions and child scenario runs.
+
+ Returns:
+ ScenarioSuiteExecutionSummary: Updated transient suite progress.
+ """
+ execution = await get_scenario_suite_execution_service().cancel_execution_async(execution_id=execution_id)
+ if execution is None:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail=f"Scenario suite execution '{execution_id}' is no longer available",
+ )
+ return execution
+
+
+@router.post(
+ "/{name}/launch",
+ response_model=ScenarioSuiteExecutionSummary,
+ status_code=status.HTTP_202_ACCEPTED,
+ responses={
+ 400: {"model": ProblemDetail, "description": "The suite has unresolved or invalid entries"},
+ 404: {"model": ProblemDetail, "description": "Scenario suite not found"},
+ 409: {"model": ProblemDetail, "description": "The suite or a referenced preset changed"},
+ 429: {"model": ProblemDetail, "description": "Active suite execution capacity is full"},
+ 503: {"model": ProblemDetail, "description": "Runtime or definition storage unavailable"},
+ },
+)
+async def launch_scenario_suite( # pyrit-async-suffix-exempt
+ name: str,
+ body: LaunchScenarioSuiteRequest,
+ http_request: Request,
+) -> ScenarioSuiteExecutionSummary:
+ """
+ Accept one transient suite execution after registry-backed preflight.
+
+ Returns:
+ ScenarioSuiteExecutionSummary: Server-owned execution identity and
+ initial ordered entry states.
+ """
+ runtime = getattr(http_request.app.state, "runtime_lifecycle", None)
+ if runtime is not None and runtime.state != "ready":
+ raise HTTPException(
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+ detail="Scenario suite launch was interrupted by runtime reinitialization.",
+ )
+ try:
+ return await get_scenario_suite_execution_service().launch_async(name=name, request=body)
+ except ScenarioSuiteExecutionNotFoundError as exc:
+ raise _not_found(name) from exc
+ except ScenarioSuiteLaunchConflictError as exc:
+ raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from None
+ except ScenarioSuiteConflictError as exc:
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail=f"Scenario suite '{name}' changed after it was reviewed; refresh it before launching.",
+ ) from exc
+ except ScenarioSuiteNotRunnableError as exc:
+ raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from None
+ except ScenarioSuiteExecutionCapacityError as exc:
+ raise HTTPException(status_code=status.HTTP_429_TOO_MANY_REQUESTS, detail=str(exc)) from None
+ except AzureError as exc:
+ raise _storage_unavailable() from exc
+ except RuntimeError as exc:
+ raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(exc)) from None
+
+
@router.get(
"/{name}",
response_model=ScenarioSuiteResponse,
@@ -91,7 +198,7 @@ async def get_scenario_suite(name: str) -> ScenarioSuiteResponse: # pyrit-async
Get one scenario suite and the version required to update it.
Returns:
- ScenarioSuiteResponse: The stored suite and current entry statuses.
+ ScenarioSuiteResponse: The suite and current entry statuses.
"""
return await _load_suite_or_404_async(name)
@@ -101,7 +208,12 @@ async def get_scenario_suite(name: str) -> ScenarioSuiteResponse: # pyrit-async
response_model=ScenarioSuiteResponse,
status_code=status.HTTP_201_CREATED,
dependencies=[Depends(require_admin)],
- responses={409: {"model": ProblemDetail, "description": "A suite is already stored under this name"}},
+ responses={
+ 409: {
+ "model": ProblemDetail,
+ "description": "A suite is already stored under this name, or the name is registered in code",
+ }
+ },
)
async def create_scenario_suite( # pyrit-async-suffix-exempt
suite: ScenarioSuite,
@@ -118,6 +230,11 @@ async def create_scenario_suite( # pyrit-async-suffix-exempt
try:
return await get_scenario_suite_service().save_suite_async(suite=suite, expected_version=None)
+ except ScenarioSuiteNameRegisteredError as exc:
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail=f"Scenario suite '{suite.name}' is registered in code and cannot be stored",
+ ) from exc
except ScenarioSuiteConflictError as exc:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
@@ -133,7 +250,10 @@ async def create_scenario_suite( # pyrit-async-suffix-exempt
dependencies=[Depends(require_admin)],
responses={
400: {"model": ProblemDetail, "description": "Body name does not match the path"},
- 409: {"model": ProblemDetail, "description": "The suite changed after it was read"},
+ 409: {
+ "model": ProblemDetail,
+ "description": "The suite changed after it was read, or the name is registered in code",
+ },
},
)
async def update_scenario_suite( # pyrit-async-suffix-exempt
@@ -156,10 +276,15 @@ async def update_scenario_suite( # pyrit-async-suffix-exempt
service = get_scenario_suite_service()
suite = body.suite
if suite.author is None:
- stored = await service.load_suite_async(name=name)
- if stored is not None:
- suite = suite.model_copy(update={"author": stored.suite.author})
+ existing = await service.load_suite_async(name=name)
+ if existing is not None:
+ suite = suite.model_copy(update={"author": existing.suite.author})
return await service.save_suite_async(suite=suite, expected_version=body.expected_version)
+ except ScenarioSuiteNameRegisteredError as exc:
+ raise HTTPException(
+ status_code=status.HTTP_409_CONFLICT,
+ detail=f"Scenario suite '{name}' is registered in code and cannot be stored",
+ ) from exc
except ScenarioSuiteConflictError as exc:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
diff --git a/pyrit/backend/services/__init__.py b/pyrit/backend/services/__init__.py
index fbf17703bf..b1ec37bf87 100644
--- a/pyrit/backend/services/__init__.py
+++ b/pyrit/backend/services/__init__.py
@@ -20,6 +20,10 @@
from pyrit.backend.services.scenario_preset_service import ScenarioPresetService, get_scenario_preset_service
from pyrit.backend.services.scenario_run_service import ScenarioRunService, get_scenario_run_service
from pyrit.backend.services.scenario_service import ScenarioService, get_scenario_service
+ from pyrit.backend.services.scenario_suite_execution_service import (
+ ScenarioSuiteExecutionService,
+ get_scenario_suite_execution_service,
+ )
from pyrit.backend.services.scenario_suite_service import ScenarioSuiteService, get_scenario_suite_service
from pyrit.backend.services.target_service import TargetService, get_target_service
@@ -38,6 +42,8 @@
"get_scenario_preset_service": "pyrit.backend.services.scenario_preset_service",
"ScenarioSuiteService": "pyrit.backend.services.scenario_suite_service",
"get_scenario_suite_service": "pyrit.backend.services.scenario_suite_service",
+ "ScenarioSuiteExecutionService": "pyrit.backend.services.scenario_suite_execution_service",
+ "get_scenario_suite_execution_service": "pyrit.backend.services.scenario_suite_execution_service",
"ScenarioRunService": "pyrit.backend.services.scenario_run_service",
"get_scenario_run_service": "pyrit.backend.services.scenario_run_service",
"TargetService": "pyrit.backend.services.target_service",
diff --git a/pyrit/backend/services/runtime_lifecycle.py b/pyrit/backend/services/runtime_lifecycle.py
index f5bd7384d1..9767c56749 100644
--- a/pyrit/backend/services/runtime_lifecycle.py
+++ b/pyrit/backend/services/runtime_lifecycle.py
@@ -16,7 +16,7 @@
from pyrit.backend.services.configuration_file_service import ConfigurationFileService
from pyrit.backend.services.environment_file_service import EnvironmentFileService
from pyrit.backend.services.scenario_run_service import get_scenario_run_service, peek_scenario_run_service
-from pyrit.backend.services.scenario_suite_service import get_scenario_suite_service
+from pyrit.backend.services.scenario_suite_execution_service import peek_scenario_suite_execution_service
from pyrit.backend.services.service_lifecycle import (
close_services_async,
has_active_analytics_work,
@@ -25,7 +25,7 @@
)
from pyrit.common.path import CONFIGURATION_DIRECTORY_PATH
from pyrit.memory import CentralMemory
-from pyrit.registry import InitializerRegistry, ScenarioPresetRegistry
+from pyrit.registry import InitializerRegistry, ScenarioPresetRegistry, ScenarioSuiteRegistry
from pyrit.setup.configuration_loader import ConfigurationLoader
from pyrit.setup.environment_loading import resolve_environment_async
from pyrit.setup.initialization import validate_reinitialization_memory
@@ -99,11 +99,13 @@ async def _management_async(self, config: ConfigurationLoader) -> None:
registry = await asyncio.to_thread(InitializerRegistry.get_registry_singleton)
registry.configure_custom_scripts_source(config.custom_initializers_source)
preset_registry = ScenarioPresetRegistry.get_registry_singleton()
- # Constructing the storage creates the default preset directory when no source is
- # configured, and the scan reads every stored document, so both stay off the event loop.
+ # Constructing either storage creates its default directory when no source is
+ # configured, and each scan reads every stored document, so both stay off the event loop.
await asyncio.to_thread(preset_registry.configure_source, config.scenario_presets_source)
await asyncio.to_thread(preset_registry.scan_stored_presets)
- await asyncio.to_thread(get_scenario_suite_service().configure_source, config.scenario_suites_source)
+ suite_registry = ScenarioSuiteRegistry.get_registry_singleton()
+ await asyncio.to_thread(suite_registry.configure_source, config.scenario_suites_source)
+ await asyncio.to_thread(suite_registry.scan_stored_suites)
def _publish(self, config: ConfigurationLoader) -> None:
self.app.state.configured_initializers = [
@@ -242,8 +244,10 @@ async def _apply_async(self, *, version: str) -> None:
def _has_active_work(self) -> bool:
"""Return whether any admitted or background runtime operation remains."""
service = peek_scenario_run_service()
+ suite_execution_service = peek_scenario_suite_execution_service()
return bool(
(service and service.has_active_work())
+ or (suite_execution_service and suite_execution_service.has_active_work())
or self.operations
or outstanding_estimates()
or has_active_manual_sends()
@@ -267,6 +271,9 @@ async def shutdown_async(self) -> None:
async def _shutdown_runtime_async(self) -> None:
"""Finish retained requests without cancelling their offloaded writes."""
service = peek_scenario_run_service()
+ suite_execution_service = peek_scenario_suite_execution_service()
+ if suite_execution_service:
+ suite_execution_service.stop_admission()
if service:
service.stop_admission()
pending = self.operations | self.management_operations
@@ -275,6 +282,13 @@ async def _shutdown_runtime_async(self) -> None:
results = await asyncio.gather(*pending, return_exceptions=True)
errors = [result for result in results if isinstance(result, BaseException)]
self.state = "stopping"
+ try:
+ suite_execution_service = peek_scenario_suite_execution_service()
+ if suite_execution_service:
+ await suite_execution_service.shutdown_async()
+ except (Exception, asyncio.CancelledError) as error:
+ error.add_note("Runtime shutdown phase: scenario suite execution shutdown.")
+ errors.append(error)
try:
service = peek_scenario_run_service()
if service:
diff --git a/pyrit/backend/services/scenario_run_service.py b/pyrit/backend/services/scenario_run_service.py
index 2936be6d7a..4d158ccc2a 100644
--- a/pyrit/backend/services/scenario_run_service.py
+++ b/pyrit/backend/services/scenario_run_service.py
@@ -17,6 +17,7 @@
from bisect import bisect_right
from collections import OrderedDict, deque
from collections.abc import AsyncGenerator, Mapping, Sequence
+from contextvars import ContextVar
from dataclasses import dataclass, replace
from datetime import UTC, datetime
from typing import Any
@@ -101,6 +102,18 @@
_SHUTDOWN_INTERRUPTION_REASON = "The backend process shut down before this scenario run completed."
_USER_CANCELLATION_REASON = "Run was cancelled by user"
_LAUNCH_REQUEST_METADATA_KEY = "scenario_launch_request_v1"
+_RESERVED_INITIAL_METADATA_KEYS = frozenset(
+ {
+ _SCHEDULER_METADATA_KEY,
+ _LAUNCH_REQUEST_METADATA_KEY,
+ SCENARIO_RUN_PLAN_METADATA_KEY,
+ SCENARIO_RUN_STARTED_AT_METADATA_KEY,
+ }
+)
+_INITIAL_METADATA_CONTEXT: ContextVar[Mapping[str, Any] | None] = ContextVar(
+ "scenario_run_initial_metadata",
+ default=None,
+)
_LAUNCH_REQUEST_FIELDS = (
"scenario_name",
"target_name",
@@ -246,16 +259,28 @@ def stop_admission(self) -> None:
"""Reject new preparations and prevent queued runs from starting."""
self._stopping = True
- async def start_run_async(self, *, request: RunScenarioRequest) -> ScenarioRunSummary:
+ async def start_run_async(
+ self,
+ *,
+ request: RunScenarioRequest,
+ initial_metadata: Mapping[str, Any] | None = None,
+ ) -> ScenarioRunSummary:
"""
Initialize and schedule a new run or an explicitly configured continuation.
+ Args:
+ request: Scenario launch configuration.
+ initial_metadata: Internal caller-owned scenario metadata persisted
+ atomically with a fresh result. Reserved run-service keys cannot
+ be supplied.
+
Returns:
ScenarioRunSummary: Current scheduled run state.
"""
+ metadata = self._validate_initial_metadata(initial_metadata=initial_metadata)
async with self._reserve_resume_request_async(request.scenario_result_id), self._launch_lock:
await self._validate_resume_admission_async(scenario_result_id=request.scenario_result_id)
- return await self._start_run_locked_async(request=request)
+ return await self._start_run_locked_async(request=request, initial_metadata=metadata)
async def resume_run_async(self, *, scenario_result_id: str) -> ScenarioRunSummary:
"""
@@ -270,7 +295,7 @@ async def resume_run_async(self, *, scenario_result_id: str) -> ScenarioRunSumma
)
assert stored is not None
request = await asyncio.to_thread(self._restore_launch_request, stored=stored)
- return await self._start_run_locked_async(request=request)
+ return await self._start_run_locked_async(request=request, initial_metadata={})
@contextlib.asynccontextmanager
async def _reserve_resume_request_async(self, scenario_result_id: str | None) -> AsyncGenerator[None, None]:
@@ -396,7 +421,12 @@ def _restore_launch_request(self, *, stored: ScenarioResult) -> RunScenarioReque
deep=True,
)
- async def _start_run_locked_async(self, *, request: RunScenarioRequest) -> ScenarioRunSummary:
+ async def _start_run_locked_async(
+ self,
+ *,
+ request: RunScenarioRequest,
+ initial_metadata: Mapping[str, Any],
+ ) -> ScenarioRunSummary:
"""
Initialize and schedule a scenario run.
@@ -407,6 +437,8 @@ async def _start_run_locked_async(self, *, request: RunScenarioRequest) -> Scena
Args:
request: The run request with scenario name, target, and options.
+ initial_metadata: Internal caller-owned metadata persisted with a
+ fresh scenario result.
Returns:
ScenarioRunSummary with a stable ID and current active or queued state.
@@ -426,7 +458,11 @@ async def _start_run_locked_async(self, *, request: RunScenarioRequest) -> Scena
resumed_from_cancelled = await self._is_run_cancelled_async(scenario_result_id=request.scenario_result_id)
if request.scenario_result_id:
self._preparing_run_ids.add(request.scenario_result_id)
- prepare_task = asyncio.create_task(self._prepare_run_async(request=request))
+ metadata_token = _INITIAL_METADATA_CONTEXT.set(initial_metadata)
+ try:
+ prepare_task = asyncio.create_task(self._prepare_run_async(request=request))
+ finally:
+ _INITIAL_METADATA_CONTEXT.reset(metadata_token)
self._preparations.add(prepare_task)
prepared = await asyncio.shield(prepare_task)
return await self._schedule_prepared_run_async(
@@ -611,12 +647,19 @@ async def _validate_run_operation_async(self, *, request: RunScenarioRequest) ->
labels = {key: value for key, value in (request.labels or {}).items() if key != "operation"}
return request.model_copy(update={"labels": labels if request.labels is not None else None})
- async def _prepare_run_async(self, *, request: RunScenarioRequest) -> _PreparedRun:
+ async def _prepare_run_async(
+ self,
+ *,
+ request: RunScenarioRequest,
+ initial_metadata: Mapping[str, Any] | None = None,
+ ) -> _PreparedRun:
"""
Resolve and initialize the scenario for a run request.
Args:
request: The run request with scenario name, target, and options.
+ initial_metadata: Internal caller-owned metadata persisted with a
+ fresh scenario result.
Returns:
_PreparedRun: The initialized scenario and its scoped default.
@@ -663,9 +706,32 @@ async def _prepare_run_async(self, *, request: RunScenarioRequest) -> _PreparedR
memory_labels=request.labels,
)
init_kwargs["operation_id"] = request.operation_id
- scenario = await self._initialize_scenario_async(request=request, init_kwargs=init_kwargs)
+ scenario = await self._initialize_scenario_async(
+ request=request,
+ init_kwargs=init_kwargs,
+ initial_metadata=(
+ initial_metadata if initial_metadata is not None else (_INITIAL_METADATA_CONTEXT.get() or {})
+ ),
+ )
return _PreparedRun(scenario=scenario, adversarial_target=adversarial_target)
+ @staticmethod
+ def _validate_initial_metadata(*, initial_metadata: Mapping[str, Any] | None) -> dict[str, Any]:
+ """
+ Copy caller metadata after protecting run-service-owned fields.
+
+ Returns:
+ dict[str, Any]: A detached mapping safe to merge into scenario metadata.
+
+ Raises:
+ ValueError: If the caller attempts to replace a reserved metadata key.
+ """
+ metadata = dict(initial_metadata or {})
+ reserved = sorted(_RESERVED_INITIAL_METADATA_KEYS.intersection(metadata))
+ if reserved:
+ raise ValueError(f"Initial scenario metadata cannot set reserved keys: {', '.join(reserved)}")
+ return metadata
+
async def get_run_async(self, *, scenario_result_id: str) -> ScenarioRunSummary | None:
"""
Get the current status of a scenario run by querying the database.
@@ -1249,7 +1315,13 @@ async def _run_initializers_async(self, *, request: RunScenarioRequest) -> None:
raise ValueError(f"Initializer not found: {e}") from None
await instance.initialize_async()
- async def _initialize_scenario_async(self, *, request: RunScenarioRequest, init_kwargs: dict[str, Any]) -> Scenario:
+ async def _initialize_scenario_async(
+ self,
+ *,
+ request: RunScenarioRequest,
+ init_kwargs: dict[str, Any],
+ initial_metadata: Mapping[str, Any] | None = None,
+ ) -> Scenario:
"""
Build and initialize the scenario via the registry.
@@ -1265,6 +1337,8 @@ async def _initialize_scenario_async(self, *, request: RunScenarioRequest, init_
scenario_result_id).
init_kwargs: The resolved common parameters to pass to
scenario.initialize_async.
+ initial_metadata: Internal caller-owned metadata persisted with a
+ fresh scenario result.
Returns:
The fully initialized Scenario instance ready for run_async.
@@ -1331,6 +1405,7 @@ async def _initialize_scenario_async(self, *, request: RunScenarioRequest, init_
scenario_params=request.scenario_params or {},
scenario_result_id=request.scenario_result_id or None,
initial_metadata={
+ **(initial_metadata or {}),
_SCHEDULER_METADATA_KEY: _SCHEDULER_METADATA_VALUE,
_LAUNCH_REQUEST_METADATA_KEY: launch_request,
},
diff --git a/pyrit/backend/services/scenario_suite_execution_service.py b/pyrit/backend/services/scenario_suite_execution_service.py
new file mode 100644
index 0000000000..db4c98e5ae
--- /dev/null
+++ b/pyrit/backend/services/scenario_suite_execution_service.py
@@ -0,0 +1,701 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Transient coordinator for launching registry-resolved scenario suites."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any, Literal
+from uuid import UUID, uuid4
+
+from pyrit.backend.models.scenario_presets import ResolveScenarioPresetRequest
+from pyrit.backend.models.scenario_suites import (
+ LaunchScenarioSuiteRequest,
+ ScenarioSuiteExecutionEntryState,
+ ScenarioSuiteExecutionEntrySummary,
+ ScenarioSuiteExecutionStatus,
+ ScenarioSuiteExecutionSummary,
+)
+from pyrit.backend.services.scenario_preset_service import ScenarioPresetService
+from pyrit.backend.services.scenario_run_service import ScenarioRunService, get_scenario_run_service
+from pyrit.models import ScenarioSuiteExecutionMetadata
+from pyrit.registry import ScenarioPresetRegistry, ScenarioSuiteRegistry
+
+if TYPE_CHECKING:
+ from pyrit.models.catalog import RunScenarioRequest, ScenarioRunSummary, ScenarioSuiteRegistryEntry
+
+logger = logging.getLogger(__name__)
+
+_MAX_RETAINED_EXECUTIONS = 256
+_CAPACITY_MESSAGE = (
+ "Too many scenario suites are still active. Wait for an active suite to finish before launching another."
+)
+_TERMINAL_EXECUTION_STATUSES = frozenset(
+ {
+ ScenarioSuiteExecutionStatus.COMPLETED,
+ ScenarioSuiteExecutionStatus.FAILED,
+ ScenarioSuiteExecutionStatus.CANCELLED,
+ }
+)
+
+
+class ScenarioSuiteExecutionNotFoundError(LookupError):
+ """A requested suite definition or transient execution does not exist."""
+
+
+class ScenarioSuiteExecutionCapacityError(RuntimeError):
+ """Active suite executions fill the transient coordinator capacity."""
+
+
+class ScenarioSuiteLaunchConflictError(ValueError):
+ """A suite or referenced preset changed after the operator reviewed it."""
+
+
+class ScenarioSuiteNotRunnableError(ValueError):
+ """A suite cannot launch because one or more entries are unresolved."""
+
+
+@dataclass(frozen=True)
+class _ResolvedLaunchEntry:
+ """One preflighted suite entry ready for scenario submission."""
+
+ entry_id: UUID
+ entry_position: int
+ preset_name: str
+ preset_origin: Literal["stored", "registered"]
+ preset_version: str | None
+ request: RunScenarioRequest
+
+ def metadata(self, *, suite_execution_id: UUID, suite: ScenarioSuiteRegistryEntry) -> dict[str, Any]:
+ """Return the scenario metadata fields for this child run."""
+ return ScenarioSuiteExecutionMetadata(
+ suite_execution_id=suite_execution_id,
+ suite_name=suite.suite.name,
+ suite_version=suite.version,
+ suite_entry_id=self.entry_id,
+ entry_position=self.entry_position,
+ preset_name=self.preset_name,
+ preset_version=self.preset_version,
+ ).to_metadata_fields()
+
+
+@dataclass
+class _ExecutionEntryState:
+ """Mutable process-local submission state for one resolved entry."""
+
+ resolved: _ResolvedLaunchEntry
+ state: ScenarioSuiteExecutionEntryState = ScenarioSuiteExecutionEntryState.PENDING
+ scenario_result_id: str | None = None
+ error: str | None = None
+ error_type: str | None = None
+ submission_task: asyncio.Task[ScenarioRunSummary] | None = None
+
+
+@dataclass
+class _ExecutionState:
+ """Mutable process-local state for one accepted suite execution."""
+
+ execution_id: UUID
+ suite: ScenarioSuiteRegistryEntry
+ entries: list[_ExecutionEntryState]
+ created_at: datetime
+ updated_at: datetime
+ cancel_requested: bool = False
+ task: asyncio.Task[None] | None = None
+ lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+
+
+@dataclass(frozen=True)
+class _ExecutionEntrySnapshot:
+ """Detached entry state used while querying child runs."""
+
+ resolved: _ResolvedLaunchEntry
+ state: ScenarioSuiteExecutionEntryState
+ scenario_result_id: str | None
+ error: str | None
+ error_type: str | None
+
+
+@dataclass(frozen=True)
+class _ExecutionSnapshot:
+ """Detached execution state used while querying child runs."""
+
+ execution_id: UUID
+ suite: ScenarioSuiteRegistryEntry
+ entries: tuple[_ExecutionEntrySnapshot, ...]
+ created_at: datetime
+ updated_at: datetime
+
+
+class ScenarioSuiteExecutionService:
+ """
+ Preflight suites and coordinate ordinary scenario runs without a persisted parent.
+
+ Definitions remain owned by the suite and preset registries. This service owns only
+ the transient identity and submission progress for one launch; each successful child
+ remains an ordinary persisted ``ScenarioResult`` managed by ``ScenarioRunService``.
+ """
+
+ SUBMISSION_FAILURE_MESSAGE = "The scenario run could not be created. Check server logs before retrying."
+ CHILD_STATUS_FAILURE_MESSAGE = "The scenario run status could not be loaded. Refresh the page or check server logs."
+
+ def __init__(
+ self,
+ *,
+ suite_registry: ScenarioSuiteRegistry | None = None,
+ preset_registry: ScenarioPresetRegistry | None = None,
+ run_service: ScenarioRunService | None = None,
+ ) -> None:
+ """Initialize the coordinator with injectable registries and run service."""
+ self._suite_registry = suite_registry
+ self._preset_registry = preset_registry
+ self._run_service = run_service
+ self._executions: OrderedDict[UUID, _ExecutionState] = OrderedDict()
+ self._tasks: set[asyncio.Task[None]] = set()
+ self._admission_lock = asyncio.Lock()
+ self._stopping = False
+
+ def has_active_work(self) -> bool:
+ """Return whether accepted suite submissions are still running."""
+ return any(not task.done() for task in self._tasks)
+
+ def stop_admission(self) -> None:
+ """Reject new launches while allowing retained progress to remain readable."""
+ self._stopping = True
+
+ async def launch_async(
+ self,
+ *,
+ name: str,
+ request: LaunchScenarioSuiteRequest,
+ ) -> ScenarioSuiteExecutionSummary:
+ """
+ Preflight one suite and begin ordered child-run submission.
+
+ Returns:
+ ScenarioSuiteExecutionSummary: Accepted transient execution identity
+ and initial entry states.
+ """
+ if self._stopping:
+ raise RuntimeError("Scenario suite launch scheduling is stopping.")
+
+ suite, resolved_entries = await asyncio.to_thread(
+ self._resolve_launch,
+ name=name,
+ request=request,
+ )
+ if self._stopping:
+ raise RuntimeError("Scenario suite launch scheduling is stopping.")
+
+ async with self._admission_lock:
+ if self._stopping:
+ raise RuntimeError("Scenario suite launch scheduling is stopping.")
+ await self._ensure_execution_capacity_async()
+
+ now = datetime.now(UTC)
+ state = _ExecutionState(
+ execution_id=uuid4(),
+ suite=suite,
+ entries=[_ExecutionEntryState(resolved=entry) for entry in resolved_entries],
+ created_at=now,
+ updated_at=now,
+ )
+ self._executions[state.execution_id] = state
+ task = asyncio.create_task(self._submit_entries_async(state=state))
+ state.task = task
+ self._tasks.add(task)
+ task.add_done_callback(self._submission_done)
+ return await self._build_summary_async(state=state)
+
+ async def get_execution_async(self, *, execution_id: UUID) -> ScenarioSuiteExecutionSummary | None:
+ """
+ Read current process-local progress for one suite execution.
+
+ Returns:
+ ScenarioSuiteExecutionSummary | None: Current grouped progress, or
+ None when the transient execution is unavailable.
+ """
+ state = self._executions.get(execution_id)
+ return None if state is None else await self._build_summary_async(state=state)
+
+ async def cancel_execution_async(self, *, execution_id: UUID) -> ScenarioSuiteExecutionSummary | None:
+ """
+ Best-effort cancel pending submissions and already-created child runs.
+
+ Returns:
+ ScenarioSuiteExecutionSummary | None: Updated grouped progress, or
+ None when the transient execution is unavailable.
+ """
+ state = self._executions.get(execution_id)
+ if state is None:
+ return None
+
+ current = await self._build_summary_async(state=state)
+ if current.status in _TERMINAL_EXECUTION_STATUSES:
+ return current
+
+ await self._request_cancellation_async(state=state)
+ return await self._build_summary_async(state=state)
+
+ async def shutdown_async(self) -> None:
+ """Request child cancellation and finish in-flight admission handoffs."""
+ self.stop_admission()
+ await asyncio.gather(
+ *(self._request_cancellation_async(state=state) for state in list(self._executions.values())),
+ return_exceptions=True,
+ )
+ tasks = list(self._tasks)
+ if tasks:
+ await asyncio.gather(*tasks, return_exceptions=True)
+
+ async def close_async(self) -> None:
+ """Close a drained coordinator."""
+ if self.has_active_work():
+ raise RuntimeError("Scenario suite execution work has not drained.")
+
+ def _resolve_launch(
+ self,
+ *,
+ name: str,
+ request: LaunchScenarioSuiteRequest,
+ ) -> tuple[ScenarioSuiteRegistryEntry, list[_ResolvedLaunchEntry]]:
+ """
+ Resolve and validate the complete launch before any scenario is submitted.
+
+ Returns:
+ tuple[ScenarioSuiteRegistryEntry, list[_ResolvedLaunchEntry]]: The
+ exact suite and ordered, runnable child requests.
+ """
+ suite = self._get_suite_registry().resolve_suite(name=name, expected_version=request.expected_version)
+ if suite is None:
+ raise ScenarioSuiteExecutionNotFoundError(f"Scenario suite '{name}' not found.")
+ if suite.origin == "stored" and request.expected_version is None:
+ raise ScenarioSuiteLaunchConflictError(
+ f"Scenario suite '{name}' must be refreshed before launch because no reviewed version was supplied."
+ )
+ if len(suite.suite.entries) != len(request.entries):
+ raise ScenarioSuiteLaunchConflictError(
+ f"Scenario suite '{name}' changed after it was reviewed; refresh it before launching."
+ )
+
+ preset_registry = self._get_preset_registry()
+ launch_fields = ResolveScenarioPresetRequest(
+ expected_version=None,
+ target_name=request.target_name,
+ adversarial_target_name=request.adversarial_target_name,
+ initializers=request.initializers,
+ initializer_args=request.initializer_args,
+ max_concurrency=request.max_concurrency,
+ max_retries=request.max_retries,
+ operation_id=request.operation_id,
+ labels=request.labels,
+ )
+ presets = {
+ preset_name: preset_registry.get_preset(name=preset_name)
+ for preset_name in dict.fromkeys(entry.preset_name for entry in suite.suite.entries)
+ }
+
+ resolved: list[_ResolvedLaunchEntry] = []
+ for position, (suite_entry, reviewed) in enumerate(
+ zip(suite.suite.entries, request.entries, strict=True),
+ start=1,
+ ):
+ if suite_entry.entry_id != reviewed.entry_id or suite_entry.preset_name != reviewed.preset_name:
+ raise ScenarioSuiteLaunchConflictError(
+ f"Scenario suite '{name}' changed after it was reviewed; refresh it before launching."
+ )
+
+ preset = presets[suite_entry.preset_name]
+ if preset is None:
+ raise ScenarioSuiteNotRunnableError(
+ f"Entry {position} references unavailable scenario preset '{suite_entry.preset_name}'."
+ )
+ if preset.origin != reviewed.origin or preset.version != reviewed.preset_version:
+ raise ScenarioSuiteLaunchConflictError(
+ f"Scenario preset '{suite_entry.preset_name}' changed after the suite was reviewed; "
+ "refresh the suite before launching."
+ )
+
+ issues = preset_registry.check_preset(preset=preset.preset)
+ if issues:
+ details = "; ".join(issue.message for issue in issues)
+ raise ScenarioSuiteNotRunnableError(
+ f"Entry {position} scenario preset '{suite_entry.preset_name}' is not runnable: {details}"
+ )
+
+ resolved.append(
+ _ResolvedLaunchEntry(
+ entry_id=suite_entry.entry_id,
+ entry_position=position,
+ preset_name=suite_entry.preset_name,
+ preset_origin=preset.origin,
+ preset_version=preset.version,
+ request=ScenarioPresetService.resolve_run_request(
+ preset=preset.preset,
+ launch=launch_fields,
+ ),
+ )
+ )
+ return suite, resolved
+
+ async def _submit_entries_async(self, *, state: _ExecutionState) -> None:
+ """Submit every resolved entry in declared order through the FIFO run service."""
+ try:
+ for entry in state.entries:
+ async with state.lock:
+ if state.cancel_requested:
+ self._mark_unstarted_entries_cancelled(state=state)
+ state.updated_at = datetime.now(UTC)
+ return
+ entry.state = ScenarioSuiteExecutionEntryState.SUBMITTING
+ task = asyncio.create_task(
+ self._get_run_service().start_run_async(
+ request=entry.resolved.request,
+ initial_metadata=entry.resolved.metadata(
+ suite_execution_id=state.execution_id,
+ suite=state.suite,
+ ),
+ )
+ )
+ entry.submission_task = task
+ state.updated_at = datetime.now(UTC)
+
+ try:
+ result = await asyncio.shield(task)
+ except Exception as exc:
+ logger.warning(
+ "Scenario suite execution %s entry %s could not be submitted.",
+ state.execution_id,
+ entry.resolved.entry_id,
+ exc_info=True,
+ )
+ async with state.lock:
+ entry.state = ScenarioSuiteExecutionEntryState.SUBMISSION_FAILED
+ entry.error = self.SUBMISSION_FAILURE_MESSAGE
+ entry.error_type = type(exc).__name__
+ state.updated_at = datetime.now(UTC)
+ continue
+
+ await self._record_submission_result_async(state=state, entry=entry, result=result)
+ except asyncio.CancelledError:
+ async with state.lock:
+ state.cancel_requested = True
+ self._mark_unstarted_entries_cancelled(state=state)
+ in_flight = [
+ (entry, entry.submission_task)
+ for entry in state.entries
+ if entry.state == ScenarioSuiteExecutionEntryState.SUBMITTING
+ and entry.submission_task is not None
+ and entry.scenario_result_id is None
+ ]
+ state.updated_at = datetime.now(UTC)
+ for entry, task in in_flight:
+ try:
+ result = await asyncio.shield(task)
+ except asyncio.CancelledError:
+ async with state.lock:
+ if entry.scenario_result_id is None:
+ entry.state = ScenarioSuiteExecutionEntryState.CANCELLED
+ state.updated_at = datetime.now(UTC)
+ except Exception as exc:
+ async with state.lock:
+ entry.state = ScenarioSuiteExecutionEntryState.SUBMISSION_FAILED
+ entry.error = self.SUBMISSION_FAILURE_MESSAGE
+ entry.error_type = type(exc).__name__
+ state.updated_at = datetime.now(UTC)
+ else:
+ await self._record_submission_result_async(state=state, entry=entry, result=result)
+ raise
+
+ async def _build_summary_async(self, *, state: _ExecutionState) -> ScenarioSuiteExecutionSummary:
+ """
+ Build a coherent response while querying child runs outside the state lock.
+
+ Returns:
+ ScenarioSuiteExecutionSummary: Current grouped execution progress.
+ """
+ async with state.lock:
+ snapshot = _ExecutionSnapshot(
+ execution_id=state.execution_id,
+ suite=state.suite,
+ entries=tuple(
+ _ExecutionEntrySnapshot(
+ resolved=entry.resolved,
+ state=entry.state,
+ scenario_result_id=entry.scenario_result_id,
+ error=entry.error,
+ error_type=entry.error_type,
+ )
+ for entry in state.entries
+ ),
+ created_at=state.created_at,
+ updated_at=state.updated_at,
+ )
+ return await self._build_summary_from_snapshot_async(snapshot=snapshot)
+
+ async def _build_summary_from_snapshot_async(
+ self,
+ *,
+ snapshot: _ExecutionSnapshot,
+ ) -> ScenarioSuiteExecutionSummary:
+ """
+ Build a response from detached coordinator state and live child runs.
+
+ Returns:
+ ScenarioSuiteExecutionSummary: Current grouped execution progress.
+ """
+ run_entries = [entry for entry in snapshot.entries if entry.scenario_result_id is not None]
+ results = await asyncio.gather(
+ *(
+ self._get_run_service().get_run_async(scenario_result_id=entry.scenario_result_id or "")
+ for entry in run_entries
+ ),
+ return_exceptions=True,
+ )
+ runs_by_entry: dict[UUID, ScenarioRunSummary | BaseException | None] = {
+ entry.resolved.entry_id: result for entry, result in zip(run_entries, results, strict=True)
+ }
+
+ summaries: list[ScenarioSuiteExecutionEntrySummary] = []
+ updated_at = snapshot.updated_at
+ for entry in snapshot.entries:
+ has_run = entry.resolved.entry_id in runs_by_entry
+ run_or_error = runs_by_entry.get(entry.resolved.entry_id)
+ run = None
+ error = entry.error
+ error_type = entry.error_type
+ entry_state = entry.state
+ if isinstance(run_or_error, BaseException):
+ logger.warning(
+ "Scenario suite execution %s entry %s child run status could not be loaded.",
+ snapshot.execution_id,
+ entry.resolved.entry_id,
+ exc_info=(type(run_or_error), run_or_error, run_or_error.__traceback__),
+ )
+ error = self.CHILD_STATUS_FAILURE_MESSAGE
+ error_type = type(run_or_error).__name__
+ elif run_or_error is not None:
+ run = run_or_error
+ entry_state = ScenarioSuiteExecutionEntryState(run.status.value)
+ updated_at = max(updated_at, run.updated_at)
+ elif has_run:
+ entry_state = ScenarioSuiteExecutionEntryState.FAILED
+ error = "The submitted child scenario run is no longer available."
+ error_type = "ScenarioRunNotFoundError"
+
+ summaries.append(
+ ScenarioSuiteExecutionEntrySummary(
+ entry_id=entry.resolved.entry_id,
+ entry_position=entry.resolved.entry_position,
+ preset_name=entry.resolved.preset_name,
+ preset_origin=entry.resolved.preset_origin,
+ preset_version=entry.resolved.preset_version,
+ state=entry_state,
+ run=run,
+ error=error,
+ error_type=error_type,
+ )
+ )
+
+ return ScenarioSuiteExecutionSummary(
+ suite_execution_id=snapshot.execution_id,
+ suite_name=snapshot.suite.suite.name,
+ suite_origin=snapshot.suite.origin,
+ suite_version=snapshot.suite.version,
+ status=self._execution_status(entries=summaries),
+ created_at=snapshot.created_at,
+ updated_at=updated_at,
+ entries=summaries,
+ )
+
+ async def _request_cancellation_async(self, *, state: _ExecutionState) -> None:
+ """Cancel known children without interrupting an in-flight admission handoff."""
+ async with state.lock:
+ state.cancel_requested = True
+ self._mark_unstarted_entries_cancelled(state=state)
+ admitted_entries = [
+ (entry, entry.scenario_result_id) for entry in state.entries if entry.scenario_result_id is not None
+ ]
+ state.updated_at = datetime.now(UTC)
+
+ await asyncio.gather(
+ *(
+ self._cancel_child_run_async(
+ state=state,
+ entry=entry,
+ scenario_result_id=scenario_result_id,
+ )
+ for entry, scenario_result_id in admitted_entries
+ )
+ )
+
+ async def _record_submission_result_async(
+ self,
+ *,
+ state: _ExecutionState,
+ entry: _ExecutionEntryState,
+ result: ScenarioRunSummary,
+ ) -> None:
+ """Record an admitted child and honor cancellation requested during admission."""
+ async with state.lock:
+ entry.scenario_result_id = result.scenario_result_id
+ entry.state = ScenarioSuiteExecutionEntryState(result.status.value)
+ entry.error = result.error
+ entry.error_type = result.error_type
+ should_cancel = state.cancel_requested
+ state.updated_at = datetime.now(UTC)
+
+ if should_cancel:
+ await self._cancel_child_run_async(
+ state=state,
+ entry=entry,
+ scenario_result_id=result.scenario_result_id,
+ )
+
+ async def _cancel_child_run_async(
+ self,
+ *,
+ state: _ExecutionState,
+ entry: _ExecutionEntryState,
+ scenario_result_id: str,
+ ) -> None:
+ """Cancel one admitted child and retain the scheduler's authoritative state."""
+ try:
+ result = await self._get_run_service().cancel_run_async(scenario_result_id=scenario_result_id)
+ except Exception:
+ logger.warning(
+ "Scenario suite execution %s child run %s could not be cancelled.",
+ state.execution_id,
+ scenario_result_id,
+ exc_info=True,
+ )
+ return
+ if result is None:
+ return
+
+ async with state.lock:
+ entry.state = ScenarioSuiteExecutionEntryState(result.status.value)
+ entry.error = result.error
+ entry.error_type = result.error_type
+ state.updated_at = datetime.now(UTC)
+
+ @staticmethod
+ def _mark_unstarted_entries_cancelled(*, state: _ExecutionState) -> None:
+ """Mark entries that have not begun admission as cancelled."""
+ for entry in state.entries:
+ if entry.state == ScenarioSuiteExecutionEntryState.PENDING:
+ entry.state = ScenarioSuiteExecutionEntryState.CANCELLED
+
+ async def _ensure_execution_capacity_async(self) -> None:
+ """Evict only terminal executions or reject admission when active work fills capacity."""
+ while len(self._executions) >= _MAX_RETAINED_EXECUTIONS:
+ evicted = False
+ for execution_id, state in list(self._executions.items()):
+ if state.task is None or not state.task.done():
+ continue
+ summary = await self._build_summary_async(state=state)
+ if summary.status not in _TERMINAL_EXECUTION_STATUSES:
+ continue
+ if self._executions.get(execution_id) is state:
+ self._executions.pop(execution_id)
+ evicted = True
+ break
+ if not evicted:
+ raise ScenarioSuiteExecutionCapacityError(_CAPACITY_MESSAGE)
+
+ @staticmethod
+ def _execution_status(
+ *,
+ entries: list[ScenarioSuiteExecutionEntrySummary],
+ ) -> ScenarioSuiteExecutionStatus:
+ """
+ Reduce ordered entry states to one process-local execution status.
+
+ Returns:
+ ScenarioSuiteExecutionStatus: Aggregate execution state.
+ """
+ states = {entry.state for entry in entries}
+ if states.intersection(
+ {
+ ScenarioSuiteExecutionEntryState.PENDING,
+ ScenarioSuiteExecutionEntryState.SUBMITTING,
+ }
+ ):
+ return ScenarioSuiteExecutionStatus.QUEUING
+ if states.intersection(
+ {
+ ScenarioSuiteExecutionEntryState.CREATED,
+ ScenarioSuiteExecutionEntryState.QUEUED,
+ ScenarioSuiteExecutionEntryState.IN_PROGRESS,
+ }
+ ):
+ return ScenarioSuiteExecutionStatus.IN_PROGRESS
+ if ScenarioSuiteExecutionEntryState.CANCELLED in states:
+ return ScenarioSuiteExecutionStatus.CANCELLED
+ if states.intersection(
+ {
+ ScenarioSuiteExecutionEntryState.SUBMISSION_FAILED,
+ ScenarioSuiteExecutionEntryState.FAILED,
+ }
+ ):
+ return ScenarioSuiteExecutionStatus.FAILED
+ return ScenarioSuiteExecutionStatus.COMPLETED
+
+ def _submission_done(self, task: asyncio.Task[None]) -> None:
+ """Retrieve unexpected task failures and release active-task ownership."""
+ self._tasks.discard(task)
+ if task.cancelled():
+ return
+ error = task.exception()
+ if error is not None:
+ logger.error(
+ "Scenario suite submission coordinator failed.",
+ exc_info=(type(error), error, error.__traceback__),
+ )
+
+ def _get_suite_registry(self) -> ScenarioSuiteRegistry:
+ """Return the injected suite registry or process singleton."""
+ return (
+ self._suite_registry if self._suite_registry is not None else ScenarioSuiteRegistry.get_registry_singleton()
+ )
+
+ def _get_preset_registry(self) -> ScenarioPresetRegistry:
+ """Return the injected preset registry or process singleton."""
+ return (
+ self._preset_registry
+ if self._preset_registry is not None
+ else ScenarioPresetRegistry.get_registry_singleton()
+ )
+
+ def _get_run_service(self) -> ScenarioRunService:
+ """Return the injected run service or process singleton."""
+ return self._run_service if self._run_service is not None else get_scenario_run_service()
+
+
+_service_instance: ScenarioSuiteExecutionService | None = None
+
+
+def peek_scenario_suite_execution_service() -> ScenarioSuiteExecutionService | None:
+ """Return the existing coordinator without constructing one."""
+ return _service_instance
+
+
+async def reset_scenario_suite_execution_service_async() -> None:
+ """Close and discard the drained coordinator singleton."""
+ global _service_instance
+ if _service_instance is not None:
+ await _service_instance.close_async()
+ _service_instance = None
+
+
+def get_scenario_suite_execution_service() -> ScenarioSuiteExecutionService:
+ """Return the process-local suite execution coordinator."""
+ global _service_instance
+ if _service_instance is None:
+ _service_instance = ScenarioSuiteExecutionService()
+ return _service_instance
diff --git a/pyrit/backend/services/scenario_suite_service.py b/pyrit/backend/services/scenario_suite_service.py
index dad057445f..3538d8bcbc 100644
--- a/pyrit/backend/services/scenario_suite_service.py
+++ b/pyrit/backend/services/scenario_suite_service.py
@@ -16,8 +16,13 @@
ScenarioSuiteListResponse,
ScenarioSuiteResponse,
)
-from pyrit.models.catalog import PresetIssue, ScenarioSuite, ScenarioSuiteEntry, StoredScenarioSuite
-from pyrit.registry import ScenarioPresetRegistry, ScenarioSuiteStorage
+from pyrit.models.catalog import (
+ PresetIssue,
+ ScenarioSuite,
+ ScenarioSuiteEntry,
+ ScenarioSuiteRegistryEntry,
+)
+from pyrit.registry import ScenarioPresetRegistry, ScenarioSuiteRegistry
logger = logging.getLogger(__name__)
@@ -32,30 +37,21 @@ class _PresetProjection:
class ScenarioSuiteService:
- """Wire adapter over ``ScenarioSuiteStorage`` with live preset-status projection."""
+ """Wire adapter over ``ScenarioSuiteRegistry`` with live preset-status projection."""
- def __init__(self, *, storage: ScenarioSuiteStorage | None = None) -> None:
+ def __init__(self, *, registry: ScenarioSuiteRegistry | None = None) -> None:
"""
Initialize the service.
Args:
- storage (ScenarioSuiteStorage | None): Storage to use, constructed lazily by default.
+ registry (ScenarioSuiteRegistry | None): Registry to use, or the singleton by default.
"""
- self._storage = storage
+ self._registry = registry
@property
def source(self) -> str:
"""Credential-free storage source suites are read from and written to."""
- return self._get_storage().display_source
-
- def configure_source(self, source: str | None) -> None:
- """
- Point the service at a local directory or Azure Blob source.
-
- Args:
- source (str | None): Configured source, or None for the default directory.
- """
- self._storage = ScenarioSuiteStorage(source=source)
+ return self._get_registry().source
async def list_suites_async(self) -> ScenarioSuiteListResponse:
"""
@@ -74,17 +70,17 @@ async def get_suite_async(self, *, name: str) -> ScenarioSuiteResponse | None:
Returns:
ScenarioSuiteResponse | None: The suite and current entry statuses, or None if absent.
"""
- stored = await self.load_suite_async(name=name)
- return None if stored is None else await asyncio.to_thread(self._to_response, stored=stored)
+ entry = await self.load_suite_async(name=name)
+ return None if entry is None else await asyncio.to_thread(self._to_response, entry=entry)
- async def load_suite_async(self, *, name: str) -> StoredScenarioSuite | None:
+ async def load_suite_async(self, *, name: str) -> ScenarioSuiteRegistryEntry | None:
"""
- Read one stored suite without projecting referenced presets.
+ Read one suite without projecting referenced presets.
Returns:
- StoredScenarioSuite | None: The stored suite, or None if absent.
+ ScenarioSuiteRegistryEntry | None: The suite and its origin, or None if absent.
"""
- return await asyncio.to_thread(self._get_storage().load_suite, name)
+ return await asyncio.to_thread(self._get_registry().get_suite, name=name)
async def save_suite_async(
self,
@@ -99,11 +95,12 @@ async def save_suite_async(
ScenarioSuiteResponse: The persisted suite, its new version, and current entry statuses.
"""
stored = await asyncio.to_thread(
- self._get_storage().save_suite,
+ self._get_registry().save_suite,
suite=suite,
expected_version=expected_version,
)
- return await asyncio.to_thread(self._to_response, stored=stored)
+ entry = ScenarioSuiteRegistryEntry(suite=stored.suite, origin="stored", version=stored.version)
+ return await asyncio.to_thread(self._to_response, entry=entry)
async def delete_suite_async(self, *, name: str) -> bool:
"""
@@ -112,18 +109,16 @@ async def delete_suite_async(self, *, name: str) -> bool:
Returns:
bool: True if a suite was deleted, False if it was already absent.
"""
- return await asyncio.to_thread(self._get_storage().delete_suite, name)
+ return await asyncio.to_thread(self._get_registry().delete_suite, name=name)
- def _get_storage(self) -> ScenarioSuiteStorage:
+ def _get_registry(self) -> ScenarioSuiteRegistry:
"""
- Return storage, constructing the default backend on first use.
+ Return the injected registry or the process singleton.
Returns:
- ScenarioSuiteStorage: The configured storage backend.
+ ScenarioSuiteRegistry: The suite registry.
"""
- if self._storage is None:
- self._storage = ScenarioSuiteStorage()
- return self._storage
+ return self._registry if self._registry is not None else ScenarioSuiteRegistry.get_registry_singleton()
def _read_all_responses(self) -> list[ScenarioSuiteResponse]:
"""
@@ -132,27 +127,28 @@ def _read_all_responses(self) -> list[ScenarioSuiteResponse]:
Returns:
list[ScenarioSuiteResponse]: Every readable suite, sorted by name.
"""
- stored = self._get_storage().list_suites()
+ entries = self._get_registry().list_suites()
projections: dict[str, _PresetProjection] = {}
- return [self._to_response(stored=stored[name], projections=projections) for name in sorted(stored)]
+ return [self._to_response(entry=entry, projections=projections) for entry in entries]
@staticmethod
def _to_response(
*,
- stored: StoredScenarioSuite,
+ entry: ScenarioSuiteRegistryEntry,
projections: dict[str, _PresetProjection] | None = None,
) -> ScenarioSuiteResponse:
"""
- Attach current preset metadata to one stored suite.
+ Attach current preset metadata to one suite.
Returns:
ScenarioSuiteResponse: The suite response.
"""
return ScenarioSuiteResponse(
- suite=stored.suite,
- version=stored.version,
+ suite=entry.suite,
+ origin=entry.origin,
+ version=entry.version,
entry_statuses=ScenarioSuiteService._resolve_entry_statuses(
- entries=stored.suite.entries,
+ entries=entry.suite.entries,
projections=projections,
),
)
diff --git a/pyrit/backend/services/service_lifecycle.py b/pyrit/backend/services/service_lifecycle.py
index 6e48e2c9fc..3ad17273da 100644
--- a/pyrit/backend/services/service_lifecycle.py
+++ b/pyrit/backend/services/service_lifecycle.py
@@ -14,6 +14,9 @@
from pyrit.backend.services.message_send_service import get_message_send_service
from pyrit.backend.services.scenario_run_service import reset_scenario_run_service_async
from pyrit.backend.services.scenario_service import get_scenario_service
+from pyrit.backend.services.scenario_suite_execution_service import (
+ reset_scenario_suite_execution_service_async,
+)
from pyrit.backend.services.scorer_service import get_scorer_service
from pyrit.backend.services.target_service import get_target_service
from pyrit.backend.services.technique_service import get_technique_service
@@ -64,6 +67,7 @@ async def _close_cached_services_async() -> None:
await get_scenario_service().close_async()
if get_converter_service.cache_info().currsize:
await get_converter_service().close_async()
+ await reset_scenario_suite_execution_service_async()
await reset_scenario_run_service_async()
finally:
for factory in (
diff --git a/pyrit/models/__init__.py b/pyrit/models/__init__.py
index cc6dcc462f..2db5d59b0e 100644
--- a/pyrit/models/__init__.py
+++ b/pyrit/models/__init__.py
@@ -195,6 +195,11 @@
ScenarioSeedGroupProgress,
ScenarioTechniqueProgress,
)
+ from pyrit.models.scenario_suite_execution import (
+ SCENARIO_SUITE_EXECUTION_METADATA_KEY,
+ SCENARIO_SUITE_EXECUTION_METADATA_VERSION,
+ ScenarioSuiteExecutionMetadata,
+ )
from pyrit.models.score import (
Acquisition,
AnswerMatches,
@@ -469,6 +474,9 @@
"SCENARIO_RUN_PLAN_METADATA_KEY": "pyrit.models.scenario_progress",
"SCENARIO_RUN_PLAN_VERSION": "pyrit.models.scenario_progress",
"SCENARIO_RUN_STARTED_AT_METADATA_KEY": "pyrit.models.scenario_progress",
+ "SCENARIO_SUITE_EXECUTION_METADATA_KEY": "pyrit.models.scenario_suite_execution",
+ "SCENARIO_SUITE_EXECUTION_METADATA_VERSION": "pyrit.models.scenario_suite_execution",
+ "ScenarioSuiteExecutionMetadata": "pyrit.models.scenario_suite_execution",
"ScenarioAttackResultDelta": "pyrit.models.scenario_progress",
"ScenarioAtomicGroupProgress": "pyrit.models.scenario_progress",
"ScenarioAttackTechniqueDetails": "pyrit.models.scenario_progress",
diff --git a/pyrit/models/catalog/__init__.py b/pyrit/models/catalog/__init__.py
index 6c532e7719..ea4b1f9aa3 100644
--- a/pyrit/models/catalog/__init__.py
+++ b/pyrit/models/catalog/__init__.py
@@ -41,7 +41,12 @@
ScenarioTechniqueSummary,
)
from pyrit.models.catalog.scenario_preset import PresetEntry, PresetIssue, ScenarioPreset, StoredPreset
- from pyrit.models.catalog.scenario_suite import ScenarioSuite, ScenarioSuiteEntry, StoredScenarioSuite
+ from pyrit.models.catalog.scenario_suite import (
+ ScenarioSuite,
+ ScenarioSuiteEntry,
+ ScenarioSuiteRegistryEntry,
+ StoredScenarioSuite,
+ )
from pyrit.models.catalog.scorer import ScorerInstance
from pyrit.models.catalog.target import TargetInstance
@@ -60,6 +65,7 @@
"ScenarioPreset": "pyrit.models.catalog.scenario_preset",
"ScenarioSuite": "pyrit.models.catalog.scenario_suite",
"ScenarioSuiteEntry": "pyrit.models.catalog.scenario_suite",
+ "ScenarioSuiteRegistryEntry": "pyrit.models.catalog.scenario_suite",
"ScenarioProducerCategoryCounts": "pyrit.models.catalog.scenario",
"ScenarioProducerCounts": "pyrit.models.catalog.scenario",
"ScenarioRunListItem": "pyrit.models.catalog.scenario",
diff --git a/pyrit/models/catalog/scenario_suite.py b/pyrit/models/catalog/scenario_suite.py
index 1f41ae6c57..991ca4cfbd 100644
--- a/pyrit/models/catalog/scenario_suite.py
+++ b/pyrit/models/catalog/scenario_suite.py
@@ -3,16 +3,19 @@
"""Scenario suite definition models."""
-from typing import Self
+from typing import Literal, Self
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from pyrit.models.identifiers.class_name_utils import validate_registry_name
+from pyrit.models.identifiers.component_identifier import ComponentIdentifier, Identifiable
from pyrit.models.request_limits import MAX_ITEMS
SUITE_SCHEMA_VERSION = 1
+_NON_BEHAVIORAL_FIELDS = frozenset({"name", "schema_version", "description", "author", "entries"})
+
class ScenarioSuiteEntry(BaseModel):
"""One ordered reference to a scenario preset."""
@@ -38,7 +41,7 @@ def _validate_preset_name(cls, value: str) -> str:
return value
-class ScenarioSuite(BaseModel):
+class ScenarioSuite(BaseModel, Identifiable):
"""
A named, ordered collection of scenario preset references.
@@ -113,9 +116,48 @@ def _validate_unique_entry_ids(self) -> Self:
seen.add(entry.entry_id)
return self
+ def _build_identifier(self) -> ComponentIdentifier:
+ """
+ Build the behavioral identity of this suite.
+
+ Entry identifiers support stable editing but do not change what the suite
+ runs, so only ordered preset names contribute to identity.
+
+ Returns:
+ ComponentIdentifier: Identity covering the suite's behavioral fields.
+ """
+ params = self.model_dump(mode="json", exclude=set(_NON_BEHAVIORAL_FIELDS), exclude_none=True)
+ params["entries"] = [entry.preset_name for entry in self.entries]
+ return ComponentIdentifier(
+ class_name=type(self).__name__,
+ class_module=type(self).__module__,
+ params=params,
+ )
+
+ def get_identifier(self) -> ComponentIdentifier:
+ """
+ Return the behavioral identity derived from the current suite definition.
+
+ Returns:
+ ComponentIdentifier: Identity derived from the ordered preset references.
+ """
+ return self._build_identifier()
+
class StoredScenarioSuite(BaseModel):
"""A scenario suite together with the version of its stored document."""
suite: ScenarioSuite = Field(..., description="The stored scenario suite")
version: str = Field(..., description="Opaque version of the document this suite was read from")
+
+
+class ScenarioSuiteRegistryEntry(BaseModel):
+ """A scenario suite together with the layer that contributed it."""
+
+ suite: ScenarioSuite = Field(..., description="The scenario suite")
+ origin: Literal["stored", "registered"] = Field(
+ ..., description="Whether a stored document or registering code contributed this suite"
+ )
+ version: str | None = Field(
+ None, description="Opaque version of the stored document, or None for a registered suite"
+ )
diff --git a/pyrit/models/scenario_suite_execution.py b/pyrit/models/scenario_suite_execution.py
new file mode 100644
index 0000000000..9312a6dc5a
--- /dev/null
+++ b/pyrit/models/scenario_suite_execution.py
@@ -0,0 +1,53 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Versioned metadata linking ordinary scenario results to one suite execution."""
+
+from typing import Any, Literal, Self
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field
+
+SCENARIO_SUITE_EXECUTION_METADATA_KEY = "scenario_suite_execution_v1"
+SCENARIO_SUITE_EXECUTION_METADATA_VERSION = 1
+
+
+class ScenarioSuiteExecutionMetadata(BaseModel):
+ """Identity stamped on every scenario result successfully submitted by a suite launch."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ version: Literal[1] = SCENARIO_SUITE_EXECUTION_METADATA_VERSION
+ suite_execution_id: UUID = Field(..., description="Server-owned identity for one suite launch")
+ suite_name: str = Field(..., description="Resolved suite name")
+ suite_version: str | None = Field(
+ None,
+ description="Resolved stored suite version, or None for a suite registered in code",
+ )
+ suite_entry_id: UUID = Field(..., description="Stable identity of the launched suite entry")
+ entry_position: int = Field(..., ge=1, description="One-based position in the resolved suite")
+ preset_name: str = Field(..., description="Resolved preset name")
+ preset_version: str | None = Field(
+ None,
+ description="Resolved stored preset version, or None for a preset registered in code",
+ )
+
+ def to_metadata_fields(self) -> dict[str, Any]:
+ """
+ Return the namespaced scenario metadata entry owned by this model.
+
+ Returns:
+ dict[str, Any]: JSON-compatible fields for ``ScenarioResult.metadata``.
+ """
+ return {SCENARIO_SUITE_EXECUTION_METADATA_KEY: self.model_dump(mode="json")}
+
+ @classmethod
+ def from_metadata(cls, metadata: dict[str, Any]) -> Self | None:
+ """
+ Read suite execution metadata from a scenario result.
+
+ Returns:
+ Self | None: The validated envelope, or None for a direct or legacy run.
+ """
+ raw = metadata.get(SCENARIO_SUITE_EXECUTION_METADATA_KEY)
+ return None if raw is None else cls.model_validate(raw)
diff --git a/pyrit/registry/__init__.py b/pyrit/registry/__init__.py
index 499805bed4..70aa7afca3 100644
--- a/pyrit/registry/__init__.py
+++ b/pyrit/registry/__init__.py
@@ -22,6 +22,9 @@
ScenarioPresetNameRegisteredError,
ScenarioPresetRegistry,
ScenarioRegistry,
+ ScenarioSuiteMetadata,
+ ScenarioSuiteNameRegisteredError,
+ ScenarioSuiteRegistry,
ScorerMetadata,
ScorerRegistry,
TargetMetadata,
@@ -64,6 +67,9 @@
"ScenarioPresetRegistry": "pyrit.registry.components",
"ScenarioPresetStorage": "pyrit.registry.scenario_preset_storage",
"ScenarioSuiteConflictError": "pyrit.registry.scenario_suite_storage",
+ "ScenarioSuiteMetadata": "pyrit.registry.components",
+ "ScenarioSuiteNameRegisteredError": "pyrit.registry.components",
+ "ScenarioSuiteRegistry": "pyrit.registry.components",
"ScenarioSuiteStorage": "pyrit.registry.scenario_suite_storage",
"ScenarioRegistry": "pyrit.registry.components",
"ScorerRegistry": "pyrit.registry.components",
diff --git a/pyrit/registry/components/__init__.py b/pyrit/registry/components/__init__.py
index 131d310f29..80475c07a5 100644
--- a/pyrit/registry/components/__init__.py
+++ b/pyrit/registry/components/__init__.py
@@ -30,6 +30,11 @@
ScenarioPresetRegistry,
)
from pyrit.registry.components.scenario_registry import ScenarioMetadata, ScenarioRegistry
+ from pyrit.registry.components.scenario_suite_registry import (
+ ScenarioSuiteMetadata,
+ ScenarioSuiteNameRegisteredError,
+ ScenarioSuiteRegistry,
+ )
from pyrit.registry.components.scorer_registry import ScorerMetadata, ScorerRegistry
from pyrit.registry.components.target_registry import TargetMetadata, TargetRegistry
@@ -48,6 +53,9 @@
"ScenarioPresetRegistry": "pyrit.registry.components.scenario_preset_registry",
"ScenarioPresetMetadata": "pyrit.registry.components.scenario_preset_registry",
"ScenarioPresetNameRegisteredError": "pyrit.registry.components.scenario_preset_registry",
+ "ScenarioSuiteRegistry": "pyrit.registry.components.scenario_suite_registry",
+ "ScenarioSuiteMetadata": "pyrit.registry.components.scenario_suite_registry",
+ "ScenarioSuiteNameRegisteredError": "pyrit.registry.components.scenario_suite_registry",
"TargetRegistry": "pyrit.registry.components.target_registry",
"TargetMetadata": "pyrit.registry.components.target_registry",
}
diff --git a/pyrit/registry/components/scenario_suite_registry.py b/pyrit/registry/components/scenario_suite_registry.py
new file mode 100644
index 0000000000..b1c78aa0c9
--- /dev/null
+++ b/pyrit/registry/components/scenario_suite_registry.py
@@ -0,0 +1,276 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Registry owning scenario suite storage and version-checked resolution."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+from pyrit.models.catalog.scenario_suite import (
+ ScenarioSuite,
+ ScenarioSuiteRegistryEntry,
+ StoredScenarioSuite,
+)
+from pyrit.registry.instance_registry import DefaultInstanceRegistry, InstanceRegistry
+from pyrit.registry.registry import Registry
+from pyrit.registry.registry_metadata import RegistryMetadata
+from pyrit.registry.scenario_suite_storage import ScenarioSuiteConflictError, ScenarioSuiteStorage
+
+logger = logging.getLogger(__name__)
+
+
+class ScenarioSuiteNameRegisteredError(ValueError):
+ """A suite name is owned by code in this deployment and cannot be stored."""
+
+ def __init__(self, name: str) -> None:
+ """
+ Initialize the error with the name code already owns.
+
+ Args:
+ name (str): The suite name a document tried to claim.
+ """
+ self.name = name
+ super().__init__(
+ f"Scenario suite '{name}' is registered by code in this deployment and cannot be "
+ f"saved as a document. Unlike a version conflict, re-reading will not clear this."
+ )
+
+
+@dataclass(frozen=True)
+class ScenarioSuiteMetadata(RegistryMetadata):
+ """Metadata for the intentionally empty buildable suite class catalog."""
+
+
+class ScenarioSuiteRegistry(Registry["ScenarioSuite", ScenarioSuiteMetadata]):
+ """
+ Own registered and stored scenario suite resolution.
+
+ Stored suites are operator-owned JSON documents that can be edited and deleted.
+ Registered suites are contributed in process by code and are read-only. A stored
+ document shadows a registered suite with the same name so operator data takes
+ precedence, while new writes to a code-owned name are refused.
+
+ Stored suites are read through on every call. Their versions describe the current
+ document bytes, so caching them would weaken optimistic concurrency.
+ """
+
+ def __init__(self, *, storage: ScenarioSuiteStorage | None = None) -> None:
+ """
+ Initialize the registry without touching storage.
+
+ Args:
+ storage (ScenarioSuiteStorage | None): Storage to read and write suites
+ through. Defaults to the standard directory, constructed on first use.
+ """
+ super().__init__()
+ self._storage = storage
+ self.instances: InstanceRegistry[ScenarioSuite] = DefaultInstanceRegistry(instance_type=ScenarioSuite)
+
+ def _discover(self) -> None:
+ """Register no classes because suites are definitions rather than buildable classes."""
+
+ def _metadata_class(self) -> type[ScenarioSuiteMetadata]:
+ """Return the metadata class for the empty buildable catalog."""
+ return ScenarioSuiteMetadata
+
+ @property
+ def source(self) -> str:
+ """Credential-free storage source suites are read from and written to."""
+ return self._get_storage().display_source
+
+ def configure_source(self, source: str | None) -> None:
+ """
+ Point the registry at a local directory or Azure Blob source.
+
+ Args:
+ source (str | None): Configured source, or None for the default directory.
+ """
+ self._storage = ScenarioSuiteStorage(source=source)
+
+ def scan_stored_suites(self) -> int:
+ """
+ Read every stored suite once so unusable documents are reported at startup.
+
+ Returns:
+ int: The number of suites that parsed cleanly.
+ """
+ try:
+ suites = self._get_storage().list_suites()
+ except Exception:
+ logger.exception("Could not read scenario suites at startup; the suite library may be unavailable.")
+ return 0
+
+ logger.info(f"Loaded {len(suites)} scenario suite(s) from {self.source}.")
+ return len(suites)
+
+ def register_suite(self, *, suite: ScenarioSuite, tags: dict[str, str] | list[str] | None = None) -> None:
+ """
+ Contribute one read-only suite from code for the lifetime of this process.
+
+ Args:
+ suite (ScenarioSuite): The suite, registered under its own name.
+ tags (dict[str, str] | list[str] | None): Tags for discovery through ``instances``.
+
+ Raises:
+ ValueError: If a suite is already registered under this name.
+ """
+ try:
+ self.instances.register(suite, name=suite.name, tags=tags)
+ except ValueError as exc:
+ raise ValueError(f"Scenario suite '{suite.name}' is already registered.") from exc
+
+ if self._stored_version(name=suite.name) is not None:
+ logger.warning(
+ f"Scenario suite '{suite.name}' was registered but a stored document of the same "
+ f"name already exists; the stored document takes precedence."
+ )
+
+ def list_suites(self) -> list[ScenarioSuiteRegistryEntry]:
+ """
+ Read every suite this deployment offers.
+
+ Returns:
+ list[ScenarioSuiteRegistryEntry]: Stored and registered suites sorted by
+ name, with a stored document shadowing a registration of the same name.
+ """
+ entries = {
+ name: ScenarioSuiteRegistryEntry(suite=stored.suite, origin="stored", version=stored.version)
+ for name, stored in self._get_storage().list_suites().items()
+ }
+ for entry in self.instances.get_all_instances():
+ entries.setdefault(
+ entry.name,
+ ScenarioSuiteRegistryEntry(suite=entry.instance, origin="registered"),
+ )
+ return [entries[name] for name in sorted(entries)]
+
+ def get_suite(self, *, name: str) -> ScenarioSuiteRegistryEntry | None:
+ """
+ Read one suite together with its origin and optional document version.
+
+ Args:
+ name (str): The suite name.
+
+ Returns:
+ ScenarioSuiteRegistryEntry | None: The suite, or None if absent or malformed.
+ """
+ stored = self._get_storage().load_suite(name)
+ if stored is not None:
+ return ScenarioSuiteRegistryEntry(suite=stored.suite, origin="stored", version=stored.version)
+
+ registered = self.instances.get(name)
+ return ScenarioSuiteRegistryEntry(suite=registered, origin="registered") if registered is not None else None
+
+ def get_suite_version(self, *, name: str) -> str | None:
+ """
+ Read the version of one stored document without parsing it.
+
+ Args:
+ name (str): The suite name.
+
+ Returns:
+ str | None: The stored document version, or None when no document exists.
+ """
+ return self._get_storage().get_suite_version(name)
+
+ def save_suite(self, *, suite: ScenarioSuite, expected_version: str | None) -> StoredScenarioSuite:
+ """
+ Persist one suite behind an optimistic-concurrency check.
+
+ Args:
+ suite (ScenarioSuite): The suite to persist.
+ expected_version (str | None): None for create, or the version being replaced.
+
+ Returns:
+ StoredScenarioSuite: The persisted suite and its new version.
+
+ Raises:
+ ScenarioSuiteConflictError: If the stored version does not match ``expected_version``.
+ ScenarioSuiteNameRegisteredError: If code already registered this name.
+ ValueError: If the suite name is not legal.
+ """
+ try:
+ self.instances.validate_name_available(suite.name)
+ except ValueError as exc:
+ raise ScenarioSuiteNameRegisteredError(suite.name) from exc
+
+ stored = self._get_storage().save_suite(suite=suite, expected_version=expected_version)
+ logger.info(f"Saved scenario suite '{suite.name}'.")
+ return stored
+
+ def delete_suite(self, *, name: str) -> bool:
+ """
+ Delete one stored suite without affecting a registered suite.
+
+ Args:
+ name (str): The suite name.
+
+ Returns:
+ bool: True if a stored document was deleted, otherwise False.
+ """
+ deleted = self._get_storage().delete_suite(name)
+ if deleted:
+ logger.info(f"Deleted scenario suite '{name}'.")
+ return deleted
+
+ def resolve_suite(self, *, name: str, expected_version: str | None) -> ScenarioSuiteRegistryEntry | None:
+ """
+ Read one suite for execution, confirming the version the caller reviewed.
+
+ Args:
+ name (str): The suite name.
+ expected_version (str | None): Reviewed stored version, or None to accept
+ whichever suite is currently available.
+
+ Returns:
+ ScenarioSuiteRegistryEntry | None: The resolved suite, or None if absent.
+
+ Raises:
+ ScenarioSuiteConflictError: If the available version differs from the
+ caller's expected version.
+ """
+ stored = self._get_storage().load_suite(name)
+ if stored is not None:
+ if expected_version is not None and expected_version != stored.version:
+ raise ScenarioSuiteConflictError(
+ name=name,
+ expected_version=expected_version,
+ actual_version=stored.version,
+ )
+ return ScenarioSuiteRegistryEntry(suite=stored.suite, origin="stored", version=stored.version)
+
+ registered = self.instances.get(name)
+ if registered is None:
+ return None
+ if expected_version is not None:
+ raise ScenarioSuiteConflictError(name=name, expected_version=expected_version, actual_version=None)
+ return ScenarioSuiteRegistryEntry(suite=registered, origin="registered")
+
+ def _get_storage(self) -> ScenarioSuiteStorage:
+ """
+ Return storage, constructing the default backend on first use.
+
+ Returns:
+ ScenarioSuiteStorage: The configured storage backend.
+ """
+ if self._storage is None:
+ self._storage = ScenarioSuiteStorage()
+ return self._storage
+
+ def _stored_version(self, *, name: str) -> str | None:
+ """
+ Read a stored document version for collision logging.
+
+ Args:
+ name (str): The suite name.
+
+ Returns:
+ str | None: The version, or None when absent or unavailable.
+ """
+ try:
+ return self._get_storage().get_suite_version(name)
+ except Exception:
+ logger.debug(f"Could not check storage for an existing scenario suite '{name}'.", exc_info=True)
+ return None
diff --git a/pyrit/setup/initialization.py b/pyrit/setup/initialization.py
index dbb6fbe902..be385594f2 100644
--- a/pyrit/setup/initialization.py
+++ b/pyrit/setup/initialization.py
@@ -71,6 +71,7 @@ def reset_setup_registries() -> None:
ConverterRegistry,
InitializerRegistry,
ScenarioRegistry,
+ ScenarioSuiteRegistry,
ScorerRegistry,
TargetRegistry,
)
@@ -84,6 +85,7 @@ def reset_setup_registries() -> None:
ConverterRegistry,
InitializerRegistry,
ScenarioRegistry,
+ ScenarioSuiteRegistry,
ScorerRegistry,
TargetRegistry,
):
diff --git a/tests/unit/backend/test_runtime_lifecycle.py b/tests/unit/backend/test_runtime_lifecycle.py
index 1f699d84ee..5bc4187327 100644
--- a/tests/unit/backend/test_runtime_lifecycle.py
+++ b/tests/unit/backend/test_runtime_lifecycle.py
@@ -162,16 +162,17 @@ async def test_management_configures_scenario_suite_storage_source_async(tmp_pat
)
initializer_registry = MagicMock()
preset_registry = MagicMock()
- suite_service = MagicMock()
+ suite_registry = MagicMock()
with (
patch.object(lifecycle_module.InitializerRegistry, "get_registry_singleton", return_value=initializer_registry),
patch.object(lifecycle_module.ScenarioPresetRegistry, "get_registry_singleton", return_value=preset_registry),
- patch.object(lifecycle_module, "get_scenario_suite_service", return_value=suite_service),
+ patch.object(lifecycle_module.ScenarioSuiteRegistry, "get_registry_singleton", return_value=suite_registry),
):
await lifecycle._management_async(config)
- suite_service.configure_source.assert_called_once_with(suite_source)
+ suite_registry.configure_source.assert_called_once_with(suite_source)
+ suite_registry.scan_stored_suites.assert_called_once_with()
@pytest.mark.parametrize(
@@ -314,6 +315,18 @@ async def test_active_work_rejects_apply_without_stopping_or_mutating(runtime: R
assert not hasattr(service, "request_stop") or not service.request_stop.called
+async def test_active_suite_submission_rejects_apply_without_mutating(runtime: RuntimeLifecycle) -> None:
+ """Test process-local suite coordination participates in the idle barrier."""
+ service = MagicMock()
+ service.has_active_work.return_value = True
+ with patch.object(lifecycle_module, "peek_scenario_suite_execution_service", return_value=service):
+ await apply_async(runtime)
+
+ assert runtime.outcome == "busy"
+ assert runtime.state == "ready"
+ lifecycle_module.close_services_async.assert_not_awaited()
+
+
@pytest.mark.parametrize("count", [1, 3])
async def test_admitted_manual_send_blocks_reinitialization_without_an_http_request_async(
runtime: RuntimeLifecycle,
diff --git a/tests/unit/backend/test_scenario_resume.py b/tests/unit/backend/test_scenario_resume.py
index d234a450a5..55c52ef4c9 100644
--- a/tests/unit/backend/test_scenario_resume.py
+++ b/tests/unit/backend/test_scenario_resume.py
@@ -7,7 +7,7 @@
import pathlib
import threading
import uuid
-from collections.abc import AsyncIterator, Awaitable, Sequence
+from collections.abc import AsyncIterator, Awaitable, Mapping, Sequence
from typing import Any, ClassVar
from unittest.mock import AsyncMock, MagicMock, patch
@@ -868,7 +868,8 @@ async def test_overlapping_resume_requests_only_initialize_once_async(
entered = asyncio.Event()
release = asyncio.Event()
- async def fail_launch_async(*, request: RunScenarioRequest) -> None:
+ async def fail_launch_async(*, request: RunScenarioRequest, initial_metadata: Mapping[str, Any]) -> None:
+ assert initial_metadata == {}
entered.set()
await release.wait()
raise ValueError("Synthetic initialization failure")
diff --git a/tests/unit/backend/test_scenario_run_service.py b/tests/unit/backend/test_scenario_run_service.py
index f29c7a64e1..809275db9c 100644
--- a/tests/unit/backend/test_scenario_run_service.py
+++ b/tests/unit/backend/test_scenario_run_service.py
@@ -314,6 +314,47 @@ async def test_prepare_and_restore_preserve_common_input_support_async(
await service.close_async()
+async def test_prepare_persists_internal_initial_metadata_with_run_service_fields(
+ patch_central_database: MagicMock,
+) -> None:
+ """Test suite attribution is created atomically with the scenario result."""
+ registry = MagicMock(spec=ScenarioRegistry)
+ registry.get_class.return_value = Scam
+ registry.create_and_initialize_async = AsyncMock(return_value=MagicMock(spec=Scenario))
+ service = ScenarioRunService()
+ suite_metadata = {"version": 1, "suite_execution_id": "execution-id"}
+ with (
+ patch.object(ScenarioRegistry, "get_registry_singleton", return_value=registry),
+ patch.object(service, "_run_initializers_async", new_callable=AsyncMock),
+ patch.object(service._configuration_resolver, "resolve_target"),
+ ):
+ await service._prepare_run_async(
+ request=RunScenarioRequest(scenario_name="test.support", target_name="my_target"),
+ initial_metadata={"scenario_suite_execution_v1": suite_metadata},
+ )
+
+ metadata = registry.create_and_initialize_async.call_args.kwargs["initial_metadata"]
+ assert metadata["scenario_suite_execution_v1"] == suite_metadata
+ assert metadata[_svc_mod._SCHEDULER_METADATA_KEY] == _svc_mod._SCHEDULER_METADATA_VALUE
+ assert _svc_mod._LAUNCH_REQUEST_METADATA_KEY in metadata
+ await service.close_async()
+
+
+async def test_start_rejects_caller_metadata_that_overwrites_run_service_fields(
+ patch_central_database: MagicMock,
+) -> None:
+ """Test internal launch callers cannot replace scheduler-owned metadata."""
+ service = ScenarioRunService()
+
+ with pytest.raises(ValueError, match="reserved keys"):
+ await service.start_run_async(
+ request=_make_request(),
+ initial_metadata={_svc_mod._LAUNCH_REQUEST_METADATA_KEY: {"forged": True}},
+ )
+
+ await service.close_async()
+
+
async def test_prepare_requires_target_for_scenarios_that_declare_it_async(
patch_central_database: MagicMock,
) -> None:
diff --git a/tests/unit/backend/test_scenario_suite_execution_service.py b/tests/unit/backend/test_scenario_suite_execution_service.py
new file mode 100644
index 0000000000..83d0e3eaf7
--- /dev/null
+++ b/tests/unit/backend/test_scenario_suite_execution_service.py
@@ -0,0 +1,622 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Tests for transient scenario suite execution coordination."""
+
+import asyncio
+from collections.abc import AsyncGenerator
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Literal
+from unittest.mock import AsyncMock, MagicMock
+from uuid import UUID
+
+import pytest
+
+import pyrit.backend.services.scenario_suite_execution_service as _execution_mod
+from pyrit.backend.models.scenario_suites import (
+ LaunchScenarioSuiteRequest,
+ ScenarioSuiteExecutionEntryState,
+ ScenarioSuiteExecutionStatus,
+ ScenarioSuiteLaunchEntrySnapshot,
+)
+from pyrit.backend.services.scenario_run_service import ScenarioRunService
+from pyrit.backend.services.scenario_suite_execution_service import (
+ ScenarioSuiteExecutionCapacityError,
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteLaunchConflictError,
+ ScenarioSuiteNotRunnableError,
+)
+from pyrit.models import SCENARIO_SUITE_EXECUTION_METADATA_KEY, ScenarioRunState
+from pyrit.models.catalog import (
+ PresetEntry,
+ PresetIssue,
+ RunScenarioRequest,
+ ScenarioPreset,
+ ScenarioRunSummary,
+ ScenarioSuite,
+ ScenarioSuiteEntry,
+)
+from pyrit.registry import ScenarioPresetRegistry, ScenarioSuiteRegistry, ScenarioSuiteStorage
+
+ENTRY_ONE = UUID("11111111-1111-4111-8111-111111111111")
+ENTRY_TWO = UUID("22222222-2222-4222-8222-222222222222")
+OPERATION_ID = UUID("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa")
+
+
+def _suite(*, name: str = "weekly") -> ScenarioSuite:
+ """Build a two-entry suite with stable identities."""
+ return ScenarioSuite(
+ name=name,
+ entries=[
+ ScenarioSuiteEntry(entry_id=ENTRY_ONE, preset_name="first"),
+ ScenarioSuiteEntry(entry_id=ENTRY_TWO, preset_name="second"),
+ ],
+ )
+
+
+def _preset_entry(
+ *,
+ name: str,
+ scenario_name: str,
+ origin: Literal["stored", "registered"] = "stored",
+ version: str | None = None,
+) -> PresetEntry:
+ """Build a registry entry for one runnable preset."""
+ return PresetEntry(
+ preset=ScenarioPreset(name=name, scenario_name=scenario_name),
+ origin=origin,
+ version=version,
+ )
+
+
+def _run_summary(
+ *,
+ run_id: str,
+ scenario_name: str,
+ state: ScenarioRunState = ScenarioRunState.QUEUED,
+) -> ScenarioRunSummary:
+ """Build a minimal child run summary."""
+ now = datetime(2025, 1, 1, tzinfo=UTC)
+ return ScenarioRunSummary(
+ scenario_result_id=run_id,
+ scenario_name=scenario_name,
+ status=state,
+ created_at=now,
+ updated_at=now,
+ )
+
+
+def _launch_request(
+ *,
+ suite_version: str | None,
+ first_origin: str | None = "stored",
+ first_version: str | None = "first-v1",
+ second_origin: str | None = "stored",
+ second_version: str | None = "second-v1",
+) -> LaunchScenarioSuiteRequest:
+ """Build the reviewed suite and preset snapshot sent by the UI."""
+ return LaunchScenarioSuiteRequest(
+ expected_version=suite_version,
+ entries=[
+ ScenarioSuiteLaunchEntrySnapshot(
+ entry_id=ENTRY_ONE,
+ preset_name="first",
+ origin=first_origin,
+ preset_version=first_version,
+ ),
+ ScenarioSuiteLaunchEntrySnapshot(
+ entry_id=ENTRY_TWO,
+ preset_name="second",
+ origin=second_origin,
+ preset_version=second_version,
+ ),
+ ],
+ target_name="objective-target",
+ adversarial_target_name="adversarial-target",
+ max_concurrency=7,
+ max_retries=3,
+ operation_id=OPERATION_ID,
+ labels={"team": "purple"},
+ )
+
+
+async def _wait_for_submissions(service: ScenarioSuiteExecutionService) -> None:
+ """Wait for immediate mock submissions without sleeping wall-clock time."""
+ for _ in range(100):
+ if not service.has_active_work():
+ return
+ await asyncio.sleep(0)
+ raise AssertionError("Suite submission coordinator did not finish")
+
+
+@pytest.fixture
+async def execution_context(
+ tmp_path: Path,
+) -> AsyncGenerator[
+ tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+ None,
+]:
+ """Create isolated registries and a controllable run service."""
+ suite_registry = ScenarioSuiteRegistry(storage=ScenarioSuiteStorage(source=str(tmp_path)))
+ preset_registry = MagicMock(spec=ScenarioPresetRegistry)
+ presets = {
+ "first": _preset_entry(
+ name="first",
+ scenario_name="scenario.first",
+ version="first-v1",
+ ),
+ "second": _preset_entry(
+ name="second",
+ scenario_name="scenario.second",
+ version="second-v1",
+ ),
+ }
+ preset_registry.get_preset.side_effect = lambda *, name: presets.get(name)
+ preset_registry.check_preset.return_value = []
+
+ calls: list[tuple[RunScenarioRequest, dict[str, Any]]] = []
+ runs: dict[str, ScenarioRunSummary] = {}
+
+ async def start_run_async(
+ *,
+ request: RunScenarioRequest,
+ initial_metadata: dict[str, Any],
+ ) -> ScenarioRunSummary:
+ calls.append((request, initial_metadata))
+ run_id = f"run-{len(calls)}"
+ summary = _run_summary(run_id=run_id, scenario_name=request.scenario_name)
+ runs[run_id] = summary
+ return summary
+
+ async def get_run_async(*, scenario_result_id: str) -> ScenarioRunSummary | None:
+ return runs.get(scenario_result_id)
+
+ async def cancel_run_async(*, scenario_result_id: str) -> ScenarioRunSummary | None:
+ current = runs.get(scenario_result_id)
+ if current is None:
+ return None
+ cancelled = current.model_copy(update={"status": ScenarioRunState.CANCELLED})
+ runs[scenario_result_id] = cancelled
+ return cancelled
+
+ run_service = MagicMock(spec=ScenarioRunService)
+ run_service.start_run_async = AsyncMock(side_effect=start_run_async)
+ run_service.get_run_async = AsyncMock(side_effect=get_run_async)
+ run_service.cancel_run_async = AsyncMock(side_effect=cancel_run_async)
+
+ service = ScenarioSuiteExecutionService(
+ suite_registry=suite_registry,
+ preset_registry=preset_registry,
+ run_service=run_service,
+ )
+ try:
+ yield service, suite_registry, preset_registry, run_service, calls, runs
+ finally:
+ await service.shutdown_async()
+
+
+async def test_launch_queues_entries_in_order_and_stamps_versioned_metadata(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test server-owned identity, launch propagation, ordering, and child metadata."""
+ service, suite_registry, _, _, calls, _ = execution_context
+ stored = suite_registry.save_suite(suite=_suite(), expected_version=None)
+
+ accepted = await service.launch_async(
+ name="weekly",
+ request=_launch_request(suite_version=stored.version),
+ )
+ await _wait_for_submissions(service)
+ summary = await service.get_execution_async(execution_id=accepted.suite_execution_id)
+
+ assert summary is not None
+ assert summary.status == ScenarioSuiteExecutionStatus.IN_PROGRESS
+ assert [entry.preset_name for entry in summary.entries] == ["first", "second"]
+ assert [entry.entry_position for entry in summary.entries] == [1, 2]
+ assert [request.scenario_name for request, _ in calls] == [
+ "scenario.first",
+ "scenario.second",
+ ]
+ for request, _ in calls:
+ assert request.target_name == "objective-target"
+ assert request.adversarial_target_name == "adversarial-target"
+ assert request.max_concurrency == 7
+ assert request.max_retries == 3
+ assert request.operation_id == OPERATION_ID
+ assert request.labels == {"team": "purple"}
+
+ first_metadata = calls[0][1][SCENARIO_SUITE_EXECUTION_METADATA_KEY]
+ second_metadata = calls[1][1][SCENARIO_SUITE_EXECUTION_METADATA_KEY]
+ assert first_metadata == {
+ "version": 1,
+ "suite_execution_id": str(accepted.suite_execution_id),
+ "suite_name": "weekly",
+ "suite_version": stored.version,
+ "suite_entry_id": str(ENTRY_ONE),
+ "entry_position": 1,
+ "preset_name": "first",
+ "preset_version": "first-v1",
+ }
+ assert second_metadata["suite_execution_id"] == str(accepted.suite_execution_id)
+ assert second_metadata["suite_entry_id"] == str(ENTRY_TWO)
+ assert second_metadata["entry_position"] == 2
+
+
+async def test_registered_suite_and_presets_launch_with_nullable_versions(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test code-defined definitions use the same launch path with null versions."""
+ service, suite_registry, preset_registry, _, calls, _ = execution_context
+ suite_registry.register_suite(suite=_suite())
+ preset_registry.get_preset.side_effect = lambda *, name: _preset_entry(
+ name=name,
+ scenario_name=f"scenario.{name}",
+ origin="registered",
+ version=None,
+ )
+
+ accepted = await service.launch_async(
+ name="weekly",
+ request=_launch_request(
+ suite_version=None,
+ first_origin="registered",
+ first_version=None,
+ second_origin="registered",
+ second_version=None,
+ ),
+ )
+ await _wait_for_submissions(service)
+
+ summary = await service.get_execution_async(execution_id=accepted.suite_execution_id)
+ assert summary is not None
+ assert summary.suite_origin == "registered"
+ assert summary.suite_version is None
+ assert [entry.preset_version for entry in summary.entries] == [None, None]
+ assert calls[0][1][SCENARIO_SUITE_EXECUTION_METADATA_KEY]["suite_version"] is None
+ assert calls[0][1][SCENARIO_SUITE_EXECUTION_METADATA_KEY]["preset_version"] is None
+
+
+async def test_stored_suite_requires_the_reviewed_version(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test a stored suite cannot silently launch without optimistic concurrency."""
+ service, suite_registry, _, run_service, _, _ = execution_context
+ suite_registry.save_suite(suite=_suite(), expected_version=None)
+
+ with pytest.raises(ScenarioSuiteLaunchConflictError, match="no reviewed version"):
+ await service.launch_async(name="weekly", request=_launch_request(suite_version=None))
+
+ run_service.start_run_async.assert_not_awaited()
+
+
+async def test_changed_preset_snapshot_rejects_the_whole_launch_before_submission(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test referenced preset drift is a hard preflight conflict."""
+ service, suite_registry, _, run_service, _, _ = execution_context
+ stored = suite_registry.save_suite(suite=_suite(), expected_version=None)
+
+ with pytest.raises(ScenarioSuiteLaunchConflictError, match="first"):
+ await service.launch_async(
+ name="weekly",
+ request=_launch_request(suite_version=stored.version, first_version="stale"),
+ )
+
+ run_service.start_run_async.assert_not_awaited()
+
+
+async def test_advisory_preset_issue_becomes_a_hard_launch_gate(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test suite editing remains permissive while execution rejects invalid entries."""
+ service, suite_registry, preset_registry, run_service, _, _ = execution_context
+ stored = suite_registry.save_suite(suite=_suite(), expected_version=None)
+ preset_registry.check_preset.side_effect = lambda *, preset: (
+ [PresetIssue(field="scenario_name", message="Scenario is unavailable.")] if preset.name == "second" else []
+ )
+
+ with pytest.raises(ScenarioSuiteNotRunnableError, match="Entry 2"):
+ await service.launch_async(
+ name="weekly",
+ request=_launch_request(suite_version=stored.version),
+ )
+
+ run_service.start_run_async.assert_not_awaited()
+
+
+async def test_submission_failure_does_not_prevent_later_entries_from_queueing(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test accepted entries retain independent ordinary-run failure semantics."""
+ service, suite_registry, _, run_service, calls, runs = execution_context
+ stored = suite_registry.save_suite(suite=_suite(), expected_version=None)
+
+ async def start_run_async(
+ *,
+ request: RunScenarioRequest,
+ initial_metadata: dict[str, Any],
+ ) -> ScenarioRunSummary:
+ calls.append((request, initial_metadata))
+ if len(calls) == 1:
+ raise ValueError(
+ "(sqlite3.OperationalError) database is locked "
+ "[SQL: INSERT INTO ScenarioResultEntries ...] "
+ "[parameters: ('secret-token',)]"
+ )
+ summary = _run_summary(
+ run_id="run-2",
+ scenario_name=request.scenario_name,
+ state=ScenarioRunState.COMPLETED,
+ )
+ runs[summary.scenario_result_id] = summary
+ return summary
+
+ run_service.start_run_async.side_effect = start_run_async
+
+ accepted = await service.launch_async(
+ name="weekly",
+ request=_launch_request(suite_version=stored.version),
+ )
+ await _wait_for_submissions(service)
+ summary = await service.get_execution_async(execution_id=accepted.suite_execution_id)
+
+ assert summary is not None
+ assert summary.status == ScenarioSuiteExecutionStatus.FAILED
+ assert [entry.state for entry in summary.entries] == [
+ ScenarioSuiteExecutionEntryState.SUBMISSION_FAILED,
+ ScenarioSuiteExecutionEntryState.COMPLETED,
+ ]
+ assert summary.entries[0].error == service.SUBMISSION_FAILURE_MESSAGE
+ assert summary.entries[0].error_type == "ValueError"
+ assert "database is locked" not in summary.model_dump_json()
+ assert "secret-token" not in summary.model_dump_json()
+ assert summary.entries[1].run is not None
+ assert len(calls) == 2
+
+
+async def test_child_status_failure_is_nonterminal_and_recovers(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test a temporary child status error stays nonterminal and clears after recovery."""
+ service, suite_registry, _, run_service, _, runs = execution_context
+ stored = suite_registry.save_suite(suite=_suite(), expected_version=None)
+
+ accepted = await service.launch_async(
+ name="weekly",
+ request=_launch_request(suite_version=stored.version),
+ )
+ await _wait_for_submissions(service)
+
+ failed_once = False
+
+ async def get_run_async(*, scenario_result_id: str) -> ScenarioRunSummary | None:
+ nonlocal failed_once
+ if scenario_result_id == "run-1" and not failed_once:
+ failed_once = True
+ raise RuntimeError("SELECT secret_column FROM private_table")
+ return runs.get(scenario_result_id)
+
+ runs["run-2"] = runs["run-2"].model_copy(update={"status": ScenarioRunState.COMPLETED})
+ run_service.get_run_async.side_effect = get_run_async
+ unavailable = await service.get_execution_async(execution_id=accepted.suite_execution_id)
+ recovered = await service.get_execution_async(execution_id=accepted.suite_execution_id)
+
+ assert unavailable is not None
+ assert unavailable.status == ScenarioSuiteExecutionStatus.IN_PROGRESS
+ assert unavailable.entries[0].state == ScenarioSuiteExecutionEntryState.QUEUED
+ assert unavailable.entries[0].error == service.CHILD_STATUS_FAILURE_MESSAGE
+ assert unavailable.entries[0].error_type == "RuntimeError"
+ assert "secret_column" not in unavailable.model_dump_json()
+ assert unavailable.entries[1].run is not None
+ assert recovered is not None
+ assert recovered.status == ScenarioSuiteExecutionStatus.IN_PROGRESS
+ assert recovered.entries[0].run is not None
+ assert recovered.entries[0].error is None
+ assert recovered.entries[0].error_type is None
+
+
+async def test_child_status_failure_does_not_suppress_cancellation(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test cancellation remains available while a child status read is failing."""
+ service, suite_registry, _, run_service, _, runs = execution_context
+ stored = suite_registry.save_suite(suite=_suite(), expected_version=None)
+
+ accepted = await service.launch_async(
+ name="weekly",
+ request=_launch_request(suite_version=stored.version),
+ )
+ await _wait_for_submissions(service)
+
+ fail_reads = True
+
+ async def get_run_async(*, scenario_result_id: str) -> ScenarioRunSummary | None:
+ if scenario_result_id == "run-1" and fail_reads:
+ raise TimeoutError("temporary read timeout")
+ return runs.get(scenario_result_id)
+
+ run_service.get_run_async.side_effect = get_run_async
+ cancelling = await service.cancel_execution_async(execution_id=accepted.suite_execution_id)
+
+ assert cancelling is not None
+ assert cancelling.status == ScenarioSuiteExecutionStatus.CANCELLED
+ assert run_service.cancel_run_async.await_count == 2
+
+ fail_reads = False
+ recovered = await service.get_execution_async(execution_id=accepted.suite_execution_id)
+
+ assert recovered is not None
+ assert recovered.status == ScenarioSuiteExecutionStatus.CANCELLED
+ assert [entry.state for entry in recovered.entries] == [
+ ScenarioSuiteExecutionEntryState.CANCELLED,
+ ScenarioSuiteExecutionEntryState.CANCELLED,
+ ]
+
+
+async def test_cancel_recovers_and_cancels_child_admitted_before_response(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test cancellation retains an admitted child ID until the response handoff completes."""
+ service, suite_registry, _, run_service, calls, runs = execution_context
+ stored = suite_registry.save_suite(suite=_suite(), expected_version=None)
+ admitted = asyncio.Event()
+ release_response = asyncio.Event()
+
+ async def start_run_async(
+ *,
+ request: RunScenarioRequest,
+ initial_metadata: dict[str, Any],
+ ) -> ScenarioRunSummary:
+ calls.append((request, initial_metadata))
+ summary = _run_summary(
+ run_id=f"run-{len(calls)}",
+ scenario_name=request.scenario_name,
+ )
+ runs[summary.scenario_result_id] = summary
+ admitted.set()
+ await release_response.wait()
+ return summary
+
+ run_service.start_run_async.side_effect = start_run_async
+
+ summary = None
+ try:
+ accepted = await service.launch_async(
+ name="weekly",
+ request=_launch_request(suite_version=stored.version),
+ )
+ await asyncio.wait_for(admitted.wait(), timeout=5)
+
+ cancelling = await service.cancel_execution_async(execution_id=accepted.suite_execution_id)
+
+ assert cancelling is not None
+ assert cancelling.status == ScenarioSuiteExecutionStatus.QUEUING
+ assert [entry.state for entry in cancelling.entries] == [
+ ScenarioSuiteExecutionEntryState.SUBMITTING,
+ ScenarioSuiteExecutionEntryState.CANCELLED,
+ ]
+ run_service.cancel_run_async.assert_not_awaited()
+
+ release_response.set()
+ await _wait_for_submissions(service)
+ summary = await service.get_execution_async(execution_id=accepted.suite_execution_id)
+ finally:
+ release_response.set()
+
+ assert summary is not None
+ assert summary.status == ScenarioSuiteExecutionStatus.CANCELLED
+ assert [entry.state for entry in summary.entries] == [
+ ScenarioSuiteExecutionEntryState.CANCELLED,
+ ScenarioSuiteExecutionEntryState.CANCELLED,
+ ]
+ run_service.cancel_run_async.assert_awaited_once_with(scenario_result_id="run-1")
+ assert len(calls) == 1
+
+
+async def test_retention_rejects_active_capacity_and_evicts_terminal_execution(
+ execution_context: tuple[
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteRegistry,
+ MagicMock,
+ MagicMock,
+ list[tuple[RunScenarioRequest, dict[str, Any]]],
+ dict[str, ScenarioRunSummary],
+ ],
+) -> None:
+ """Test the actual retention boundary never evicts queued or running children."""
+ service, suite_registry, _, _, _, runs = execution_context
+ stored = suite_registry.save_suite(suite=_suite(), expected_version=None)
+ request = _launch_request(suite_version=stored.version)
+
+ accepted = [
+ await service.launch_async(name="weekly", request=request)
+ for _ in range(_execution_mod._MAX_RETAINED_EXECUTIONS)
+ ]
+ await _wait_for_submissions(service)
+
+ with pytest.raises(ScenarioSuiteExecutionCapacityError, match="still active"):
+ await service.launch_async(name="weekly", request=request)
+
+ first = await service.get_execution_async(execution_id=accepted[0].suite_execution_id)
+ assert first is not None
+ assert first.status == ScenarioSuiteExecutionStatus.IN_PROGRESS
+
+ for entry in first.entries:
+ assert entry.run is not None
+ runs[entry.run.scenario_result_id] = entry.run.model_copy(update={"status": ScenarioRunState.COMPLETED})
+
+ replacement = await service.launch_async(name="weekly", request=request)
+
+ assert await service.get_execution_async(execution_id=accepted[0].suite_execution_id) is None
+ assert await service.get_execution_async(execution_id=replacement.suite_execution_id) is not None
diff --git a/tests/unit/backend/test_scenario_suite_routes.py b/tests/unit/backend/test_scenario_suite_routes.py
index 4f55c333b7..568fd1152b 100644
--- a/tests/unit/backend/test_scenario_suite_routes.py
+++ b/tests/unit/backend/test_scenario_suite_routes.py
@@ -4,7 +4,9 @@
"""Tests for scenario suite definition routes."""
from collections.abc import Iterator
+from datetime import UTC, datetime
from pathlib import Path
+from typing import Literal
from unittest.mock import AsyncMock, MagicMock, patch
from uuid import UUID
@@ -16,15 +18,33 @@
from pyrit.backend.middleware.auth import current_user_name, require_admin
from pyrit.backend.models.scenario_suites import (
ScenarioSuiteEntryStatus,
+ ScenarioSuiteExecutionEntryState,
+ ScenarioSuiteExecutionEntrySummary,
+ ScenarioSuiteExecutionStatus,
+ ScenarioSuiteExecutionSummary,
ScenarioSuiteListResponse,
ScenarioSuiteResponse,
)
+from pyrit.backend.services.scenario_suite_execution_service import (
+ ScenarioSuiteExecutionCapacityError,
+ ScenarioSuiteExecutionNotFoundError,
+ ScenarioSuiteExecutionService,
+ ScenarioSuiteLaunchConflictError,
+ ScenarioSuiteNotRunnableError,
+)
from pyrit.backend.services.scenario_suite_service import ScenarioSuiteService
-from pyrit.models.catalog import PresetIssue, ScenarioSuite, ScenarioSuiteEntry, StoredScenarioSuite
-from pyrit.registry import ScenarioPresetRegistry, ScenarioSuiteConflictError, ScenarioSuiteStorage
+from pyrit.models.catalog import PresetIssue, ScenarioSuite, ScenarioSuiteEntry, ScenarioSuiteRegistryEntry
+from pyrit.registry import (
+ ScenarioPresetRegistry,
+ ScenarioSuiteConflictError,
+ ScenarioSuiteNameRegisteredError,
+ ScenarioSuiteRegistry,
+ ScenarioSuiteStorage,
+)
SUITE_NAME = "weekly"
ENTRY_ID = UUID("11111111-1111-4111-8111-111111111111")
+EXECUTION_ID = UUID("eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee")
def _suite(*, name: str = SUITE_NAME, author: str | None = None) -> ScenarioSuite:
@@ -39,13 +59,15 @@ def _suite(*, name: str = SUITE_NAME, author: str | None = None) -> ScenarioSuit
def _response(
*,
suite: ScenarioSuite | None = None,
- version: str = "suite-v1",
+ origin: Literal["stored", "registered"] = "stored",
+ version: str | None = "suite-v1",
issues: list[PresetIssue] | None = None,
) -> ScenarioSuiteResponse:
"""Build a scenario suite response envelope."""
definition = suite or _suite()
return ScenarioSuiteResponse(
suite=definition,
+ origin=origin,
version=version,
entry_statuses=[
ScenarioSuiteEntryStatus(
@@ -59,6 +81,51 @@ def _response(
)
+def _execution_summary(
+ *,
+ status_value: ScenarioSuiteExecutionStatus = ScenarioSuiteExecutionStatus.QUEUING,
+) -> ScenarioSuiteExecutionSummary:
+ """Build one transient suite execution response."""
+ now = datetime(2025, 1, 1, tzinfo=UTC)
+ return ScenarioSuiteExecutionSummary(
+ suite_execution_id=EXECUTION_ID,
+ suite_name=SUITE_NAME,
+ suite_origin="stored",
+ suite_version="suite-v1",
+ status=status_value,
+ created_at=now,
+ updated_at=now,
+ entries=[
+ ScenarioSuiteExecutionEntrySummary(
+ entry_id=ENTRY_ID,
+ entry_position=1,
+ preset_name="nightly",
+ preset_origin="stored",
+ preset_version="preset-v1",
+ state=ScenarioSuiteExecutionEntryState.PENDING,
+ )
+ ],
+ )
+
+
+def _launch_body() -> dict[str, object]:
+ """Build the suite launch payload sent by the browser."""
+ return {
+ "expected_version": "suite-v1",
+ "entries": [
+ {
+ "entry_id": str(ENTRY_ID),
+ "preset_name": "nightly",
+ "origin": "stored",
+ "preset_version": "preset-v1",
+ }
+ ],
+ "target_name": "objective-target",
+ "max_concurrency": 5,
+ "max_retries": 2,
+ }
+
+
@pytest.fixture
def client(compatibility_headers: dict[str, str]) -> Iterator[TestClient]:
"""Create a client with administrator authorization satisfied."""
@@ -80,11 +147,26 @@ def service() -> Iterator[MagicMock]:
"""Patch the scenario suite service resolved by the routes."""
mock_service = MagicMock(spec=ScenarioSuiteService)
mock_service.get_suite_async.return_value = _response()
- mock_service.load_suite_async.return_value = StoredScenarioSuite(suite=_suite(), version="suite-v1")
+ mock_service.load_suite_async.return_value = ScenarioSuiteRegistryEntry(
+ suite=_suite(),
+ origin="stored",
+ version="suite-v1",
+ )
with patch("pyrit.backend.routes.scenario_suites.get_scenario_suite_service", return_value=mock_service):
yield mock_service
+@pytest.fixture
+def execution_service() -> Iterator[MagicMock]:
+ """Patch the transient suite execution coordinator resolved by the routes."""
+ mock_service = MagicMock(spec=ScenarioSuiteExecutionService)
+ with patch(
+ "pyrit.backend.routes.scenario_suites.get_scenario_suite_execution_service",
+ return_value=mock_service,
+ ):
+ yield mock_service
+
+
@pytest.fixture
def unavailable_preset_backend(
tmp_path: Path,
@@ -93,7 +175,7 @@ def unavailable_preset_backend(
registry = MagicMock(spec=ScenarioPresetRegistry)
registry.get_preset.side_effect = AzureError("expired preset container signature")
storage = ScenarioSuiteStorage(source=str(tmp_path))
- scenario_service = ScenarioSuiteService(storage=storage)
+ scenario_service = ScenarioSuiteService(registry=ScenarioSuiteRegistry(storage=storage))
with (
patch.object(ScenarioPresetRegistry, "get_registry_singleton", return_value=registry),
patch(
@@ -128,6 +210,97 @@ def test_list_is_open_and_returns_entry_issues(anonymous_client: TestClient, ser
assert response.json()["items"][0]["entry_statuses"][0]["issues"][0]["message"] == "Preset is missing."
+def test_launch_is_open_and_returns_server_execution_id(
+ anonymous_client: TestClient,
+ execution_service: MagicMock,
+) -> None:
+ """Test one launch action creates a server-owned transient execution."""
+ execution_service.launch_async = AsyncMock(return_value=_execution_summary())
+
+ response = anonymous_client.post(f"/api/scenario-suites/{SUITE_NAME}/launch", json=_launch_body())
+
+ assert response.status_code == 202
+ assert response.json()["suite_execution_id"] == str(EXECUTION_ID)
+ launch = execution_service.launch_async.call_args.kwargs["request"]
+ assert launch.expected_version == "suite-v1"
+ assert launch.entries[0].preset_version == "preset-v1"
+ assert launch.max_concurrency == 5
+
+
+def test_launch_is_rejected_during_runtime_reinitialization(
+ anonymous_client: TestClient,
+ execution_service: MagicMock,
+) -> None:
+ """Test runtime transitions reject new suite work before preflight."""
+ with patch.object(app.state, "runtime_lifecycle", MagicMock(state="initializing"), create=True):
+ response = anonymous_client.post(
+ f"/api/scenario-suites/{SUITE_NAME}/launch",
+ json=_launch_body(),
+ )
+
+ assert response.status_code == 503
+ execution_service.launch_async.assert_not_called()
+
+
+@pytest.mark.parametrize(
+ ("error", "expected_status"),
+ [
+ (ScenarioSuiteExecutionNotFoundError("missing"), 404),
+ (ScenarioSuiteLaunchConflictError("changed"), 409),
+ (ScenarioSuiteNotRunnableError("not runnable"), 400),
+ (ScenarioSuiteExecutionCapacityError("wait for an active suite"), 429),
+ (AzureError("storage unavailable"), 503),
+ (RuntimeError("scheduling is stopping"), 503),
+ ],
+)
+def test_launch_maps_preflight_failures(
+ anonymous_client: TestClient,
+ execution_service: MagicMock,
+ error: Exception,
+ expected_status: int,
+) -> None:
+ """Test actionable launch failures without exposing storage details."""
+ execution_service.launch_async = AsyncMock(side_effect=error)
+
+ response = anonymous_client.post(f"/api/scenario-suites/{SUITE_NAME}/launch", json=_launch_body())
+
+ assert response.status_code == expected_status
+ if isinstance(error, AzureError):
+ assert "storage unavailable" not in response.text
+ if isinstance(error, ScenarioSuiteExecutionCapacityError):
+ assert response.json()["detail"] == "wait for an active suite"
+
+
+def test_get_and_cancel_transient_execution(
+ anonymous_client: TestClient,
+ execution_service: MagicMock,
+) -> None:
+ """Test grouped progress reads and best-effort cancellation."""
+ execution_service.get_execution_async = AsyncMock(return_value=_execution_summary())
+ execution_service.cancel_execution_async = AsyncMock(
+ return_value=_execution_summary(status_value=ScenarioSuiteExecutionStatus.CANCELLED)
+ )
+
+ get_response = anonymous_client.get(f"/api/scenario-suites/executions/{EXECUTION_ID}")
+ cancel_response = anonymous_client.post(f"/api/scenario-suites/executions/{EXECUTION_ID}/cancel")
+
+ assert get_response.status_code == 200
+ assert cancel_response.status_code == 200
+ assert cancel_response.json()["status"] == "CANCELLED"
+
+
+def test_missing_transient_execution_returns_not_found(
+ anonymous_client: TestClient,
+ execution_service: MagicMock,
+) -> None:
+ """Test process-local progress expiry is explicit."""
+ execution_service.get_execution_async = AsyncMock(return_value=None)
+ execution_service.cancel_execution_async = AsyncMock(return_value=None)
+
+ assert anonymous_client.get(f"/api/scenario-suites/executions/{EXECUTION_ID}").status_code == 404
+ assert anonymous_client.post(f"/api/scenario-suites/executions/{EXECUTION_ID}/cancel").status_code == 404
+
+
def test_get_returns_suite_and_version(client: TestClient, service: MagicMock) -> None:
"""Test a single open read."""
service.get_suite_async = AsyncMock(return_value=_response())
@@ -135,6 +308,7 @@ def test_get_returns_suite_and_version(client: TestClient, service: MagicMock) -
response = client.get(f"/api/scenario-suites/{SUITE_NAME}")
assert response.status_code == 200
+ assert response.json()["origin"] == "stored"
assert response.json()["version"] == "suite-v1"
assert response.json()["suite"]["entries"][0]["entry_id"] == str(ENTRY_ID)
@@ -146,6 +320,17 @@ def test_get_reports_missing_suite(client: TestClient, service: MagicMock) -> No
assert client.get("/api/scenario-suites/missing").status_code == 404
+def test_get_registered_suite_returns_origin_and_null_version(client: TestClient, service: MagicMock) -> None:
+ """Test the read-only registered suite response contract."""
+ service.get_suite_async = AsyncMock(return_value=_response(origin="registered", version=None))
+
+ response = client.get(f"/api/scenario-suites/{SUITE_NAME}")
+
+ assert response.status_code == 200
+ assert response.json()["origin"] == "registered"
+ assert response.json()["version"] is None
+
+
def test_storage_failure_is_sanitized(client: TestClient, service: MagicMock) -> None:
"""Test that Azure SDK details do not reach API callers."""
service.list_suites_async = AsyncMock(side_effect=AzureError("container key=secret"))
@@ -246,6 +431,16 @@ def test_create_over_existing_name_is_conflict(client: TestClient, service: Magi
assert response.status_code == 409
+def test_create_over_registered_name_is_actionable_conflict(client: TestClient, service: MagicMock) -> None:
+ """Test code-owned names are not reported as retryable version conflicts."""
+ service.save_suite_async = AsyncMock(side_effect=ScenarioSuiteNameRegisteredError(SUITE_NAME))
+
+ response = client.post("/api/scenario-suites", json=_suite().model_dump(mode="json"))
+
+ assert response.status_code == 409
+ assert response.json()["detail"] == f"Scenario suite '{SUITE_NAME}' is registered in code and cannot be stored"
+
+
def test_update_passes_expected_version(client: TestClient, service: MagicMock) -> None:
"""Test optimistic concurrency on update."""
service.save_suite_async = AsyncMock(return_value=_response(version="suite-v2"))
@@ -262,7 +457,11 @@ def test_update_passes_expected_version(client: TestClient, service: MagicMock)
def test_update_preserves_stored_author(client: TestClient, service: MagicMock) -> None:
"""Test author preservation when the editor omits it."""
stored = _suite(author="Grace Hopper")
- service.load_suite_async.return_value = StoredScenarioSuite(suite=stored, version="suite-v1")
+ service.load_suite_async.return_value = ScenarioSuiteRegistryEntry(
+ suite=stored,
+ origin="stored",
+ version="suite-v1",
+ )
service.save_suite_async.return_value = _response(suite=stored, version="suite-v2")
response = client.put(
@@ -326,6 +525,23 @@ def test_update_reports_version_conflict(client: TestClient, service: MagicMock)
assert "re-read" in response.json()["detail"]
+def test_update_registered_name_is_actionable_conflict(client: TestClient, service: MagicMock) -> None:
+ """Test a direct update cannot mutate a code-defined suite."""
+ service.load_suite_async.return_value = ScenarioSuiteRegistryEntry(
+ suite=_suite(),
+ origin="registered",
+ )
+ service.save_suite_async = AsyncMock(side_effect=ScenarioSuiteNameRegisteredError(SUITE_NAME))
+
+ response = client.put(
+ f"/api/scenario-suites/{SUITE_NAME}",
+ json={"suite": _suite().model_dump(mode="json"), "expected_version": "suite-v1"},
+ )
+
+ assert response.status_code == 409
+ assert "registered in code" in response.json()["detail"]
+
+
def test_update_requires_admin(anonymous_client: TestClient, service: MagicMock) -> None:
"""Test update authorization."""
response = anonymous_client.put(
diff --git a/tests/unit/backend/test_scenario_suite_service.py b/tests/unit/backend/test_scenario_suite_service.py
index b235dc9436..cf2d05f539 100644
--- a/tests/unit/backend/test_scenario_suite_service.py
+++ b/tests/unit/backend/test_scenario_suite_service.py
@@ -13,7 +13,12 @@
from pyrit.backend.services.scenario_suite_service import ScenarioSuiteService, get_scenario_suite_service
from pyrit.models.catalog import PresetEntry, PresetIssue, ScenarioPreset, ScenarioSuite, ScenarioSuiteEntry
-from pyrit.registry import ScenarioPresetRegistry, ScenarioSuiteConflictError, ScenarioSuiteStorage
+from pyrit.registry import (
+ ScenarioPresetRegistry,
+ ScenarioSuiteConflictError,
+ ScenarioSuiteRegistry,
+ ScenarioSuiteStorage,
+)
ENTRY_ONE = UUID("11111111-1111-4111-8111-111111111111")
ENTRY_TWO = UUID("22222222-2222-4222-8222-222222222222")
@@ -48,8 +53,9 @@ def preset_registry() -> Iterator[MagicMock]:
@pytest.fixture
def service(tmp_path: Path, preset_registry: MagicMock) -> ScenarioSuiteService:
- """Create a service backed by isolated suite storage."""
- return ScenarioSuiteService(storage=ScenarioSuiteStorage(source=str(tmp_path)))
+ """Create a service backed by an isolated suite registry."""
+ registry = ScenarioSuiteRegistry(storage=ScenarioSuiteStorage(source=str(tmp_path)))
+ return ScenarioSuiteService(registry=registry)
async def test_crud_preserves_order_and_duplicate_preset_references(
@@ -61,6 +67,7 @@ async def test_crud_preserves_order_and_duplicate_preset_references(
assert loaded is not None
assert loaded.version == created.version
+ assert loaded.origin == "stored"
assert [status.entry_id for status in loaded.entry_statuses] == [ENTRY_ONE, ENTRY_TWO]
assert [status.preset_name for status in loaded.entry_statuses] == ["nightly", "nightly"]
@@ -96,6 +103,20 @@ async def test_list_reuses_preset_projection_across_suites(
preset_registry.check_preset.assert_called_once()
+async def test_registered_suite_is_listed_with_nullable_version(
+ service: ScenarioSuiteService,
+) -> None:
+ """Test code-defined suite projection through the backend contract."""
+ registry = service._get_registry()
+ registry.register_suite(suite=_suite(name="code_suite", preset_names=("nightly",)))
+
+ response = await service.list_suites_async()
+
+ assert [(item.suite.name, item.origin, item.version) for item in response.items] == [
+ ("code_suite", "registered", None),
+ ]
+
+
@pytest.mark.parametrize(
"error",
[AzureError("expired preset credentials"), OSError("preset directory unavailable")],
@@ -215,10 +236,11 @@ async def test_delete_reports_presence(service: ScenarioSuiteService) -> None:
def test_configure_source_replaces_storage(tmp_path: Path, preset_registry: MagicMock) -> None:
- """Test runtime source configuration."""
- service = ScenarioSuiteService()
+ """Test that the service reports its registry's configured source."""
+ registry = ScenarioSuiteRegistry()
+ service = ScenarioSuiteService(registry=registry)
- service.configure_source(str(tmp_path))
+ registry.configure_source(str(tmp_path))
assert service.source == str(tmp_path)
diff --git a/tests/unit/backend/test_technique_service.py b/tests/unit/backend/test_technique_service.py
index ce3cf9c570..ce892dfc95 100644
--- a/tests/unit/backend/test_technique_service.py
+++ b/tests/unit/backend/test_technique_service.py
@@ -6,7 +6,7 @@
import ast
import asyncio
import threading
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
from enum import Enum
from typing import Any
from unittest.mock import AsyncMock, patch
@@ -457,9 +457,14 @@ async def test_launch_survives_registration_between_resolution_and_construction_
service = ScenarioRunService()
initialize = service._initialize_scenario_async
- async def register_then_initialize_async(*, request: RunScenarioRequest, init_kwargs: dict[str, Any]) -> Scenario:
+ async def register_then_initialize_async(
+ *,
+ request: RunScenarioRequest,
+ init_kwargs: dict[str, Any],
+ initial_metadata: Mapping[str, Any] | None = None,
+ ) -> Scenario:
await TechniqueService().create_async(CreateTechniqueRequest(name="during_launch", type="PromptSendingAttack"))
- return await initialize(request=request, init_kwargs=init_kwargs)
+ return await initialize(request=request, init_kwargs=init_kwargs, initial_metadata=initial_metadata)
try:
with (
diff --git a/tests/unit/models/test_scenario_suite.py b/tests/unit/models/test_scenario_suite.py
index b9086e3041..e1bb38e065 100644
--- a/tests/unit/models/test_scenario_suite.py
+++ b/tests/unit/models/test_scenario_suite.py
@@ -7,7 +7,12 @@
import pytest
-from pyrit.models.catalog.scenario_suite import SUITE_SCHEMA_VERSION, ScenarioSuite, ScenarioSuiteEntry
+from pyrit.models.catalog.scenario_suite import (
+ SUITE_SCHEMA_VERSION,
+ ScenarioSuite,
+ ScenarioSuiteEntry,
+ ScenarioSuiteRegistryEntry,
+)
ENTRY_ONE = UUID("11111111-1111-4111-8111-111111111111")
ENTRY_TWO = UUID("22222222-2222-4222-8222-222222222222")
@@ -32,6 +37,26 @@ def test_suite_preserves_entry_order_exactly() -> None:
assert [entry.preset_name for entry in suite.entries] == ["second", "first"]
+def test_suite_identifier_uses_ordered_preset_names_not_editor_entry_ids() -> None:
+ """Test that stable editor IDs do not change the suite's behavioral identity."""
+ first = ScenarioSuite(
+ name="weekly",
+ entries=[
+ _entry(entry_id=ENTRY_ONE, preset_name="first"),
+ _entry(entry_id=ENTRY_TWO, preset_name="second"),
+ ],
+ )
+ second = ScenarioSuite(
+ name="renamed",
+ entries=[
+ _entry(entry_id=ENTRY_TWO, preset_name="first"),
+ _entry(entry_id=ENTRY_ONE, preset_name="second"),
+ ],
+ )
+
+ assert first.get_identifier() == second.get_identifier()
+
+
def test_suite_allows_duplicate_preset_names_with_distinct_entry_ids() -> None:
"""Test that one preset may appear multiple times in a suite."""
suite = ScenarioSuite(
@@ -96,3 +121,10 @@ def test_suite_rejects_unknown_fields() -> None:
"""Test that misspelled hand-authored fields fail loudly."""
with pytest.raises(ValueError, match="presets"):
ScenarioSuite(name="weekly", presets=[]) # type: ignore[call-arg]
+
+
+def test_registered_suite_entry_has_no_document_version() -> None:
+ """Test the registry envelope for a suite defined in code."""
+ entry = ScenarioSuiteRegistryEntry(suite=ScenarioSuite(name="weekly"), origin="registered")
+
+ assert entry.version is None
diff --git a/tests/unit/models/test_scenario_suite_execution.py b/tests/unit/models/test_scenario_suite_execution.py
new file mode 100644
index 0000000000..06ac0bee4c
--- /dev/null
+++ b/tests/unit/models/test_scenario_suite_execution.py
@@ -0,0 +1,34 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Tests for versioned scenario suite execution metadata."""
+
+from uuid import UUID
+
+from pyrit.models import (
+ SCENARIO_SUITE_EXECUTION_METADATA_KEY,
+ ScenarioSuiteExecutionMetadata,
+)
+
+
+def test_suite_execution_metadata_round_trips_through_scenario_metadata() -> None:
+ """Test the persisted envelope keeps server and definition identities intact."""
+ metadata = ScenarioSuiteExecutionMetadata(
+ suite_execution_id=UUID("eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee"),
+ suite_name="weekly",
+ suite_version="suite-v1",
+ suite_entry_id=UUID("11111111-1111-4111-8111-111111111111"),
+ entry_position=2,
+ preset_name="nightly",
+ preset_version="preset-v1",
+ )
+
+ fields = metadata.to_metadata_fields()
+
+ assert fields[SCENARIO_SUITE_EXECUTION_METADATA_KEY]["version"] == 1
+ assert ScenarioSuiteExecutionMetadata.from_metadata(fields) == metadata
+
+
+def test_direct_run_has_no_suite_execution_metadata() -> None:
+ """Test absence remains distinct from an invalid or empty suite envelope."""
+ assert ScenarioSuiteExecutionMetadata.from_metadata({"run_plan": {}}) is None
diff --git a/tests/unit/registry/test_scenario_suite_registry.py b/tests/unit/registry/test_scenario_suite_registry.py
new file mode 100644
index 0000000000..ef597a7838
--- /dev/null
+++ b/tests/unit/registry/test_scenario_suite_registry.py
@@ -0,0 +1,291 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Tests for the scenario suite registry."""
+
+import json
+import logging
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+from uuid import UUID
+
+import pytest
+
+from pyrit.models.catalog import ScenarioSuite, ScenarioSuiteEntry
+from pyrit.registry import (
+ ScenarioSuiteConflictError,
+ ScenarioSuiteNameRegisteredError,
+ ScenarioSuiteRegistry,
+ ScenarioSuiteStorage,
+)
+
+ENTRY_ONE = UUID("11111111-1111-4111-8111-111111111111")
+ENTRY_TWO = UUID("22222222-2222-4222-8222-222222222222")
+
+
+def _suite(
+ name: str = "weekly",
+ *,
+ description: str | None = None,
+ preset_names: tuple[str, ...] = ("nightly",),
+) -> ScenarioSuite:
+ """Build a suite with stable entry identifiers."""
+ entry_ids = (ENTRY_ONE, ENTRY_TWO)
+ return ScenarioSuite(
+ name=name,
+ description=description,
+ entries=[
+ ScenarioSuiteEntry(entry_id=entry_ids[index], preset_name=preset_name)
+ for index, preset_name in enumerate(preset_names)
+ ],
+ )
+
+
+@pytest.fixture
+def registry(tmp_path: Path) -> ScenarioSuiteRegistry:
+ """Create a registry backed by an isolated suite directory."""
+ return ScenarioSuiteRegistry(storage=ScenarioSuiteStorage(source=str(tmp_path)))
+
+
+def test_saved_suite_round_trips_as_stored(registry: ScenarioSuiteRegistry) -> None:
+ """Test stored origin and opaque version projection."""
+ created = registry.save_suite(suite=_suite(), expected_version=None)
+
+ entry = registry.get_suite(name="weekly")
+
+ assert entry is not None
+ assert entry.suite == created.suite
+ assert entry.origin == "stored"
+ assert entry.version == created.version
+
+
+def test_listing_merges_both_origins_in_name_order(registry: ScenarioSuiteRegistry) -> None:
+ """Test the combined suite library ordering."""
+ registry.save_suite(suite=_suite("stored_suite"), expected_version=None)
+ registry.register_suite(suite=_suite("code_suite"))
+
+ assert [(entry.suite.name, entry.origin) for entry in registry.list_suites()] == [
+ ("code_suite", "registered"),
+ ("stored_suite", "stored"),
+ ]
+
+
+def test_registered_suite_is_readable_and_has_no_version(registry: ScenarioSuiteRegistry) -> None:
+ """Test the read-only code-registration contract."""
+ registry.register_suite(suite=_suite())
+
+ entry = registry.get_suite(name="weekly")
+
+ assert entry is not None
+ assert entry.origin == "registered"
+ assert entry.version is None
+ assert registry.get_suite_version(name="weekly") is None
+
+
+def test_registered_suites_support_instance_tags(registry: ScenarioSuiteRegistry) -> None:
+ """Test registry metadata for future built-in suite discovery."""
+ registry.register_suite(suite=_suite(), tags=["starter"])
+
+ assert [entry.name for entry in registry.instances.get_by_tag(tag="starter")] == ["weekly"]
+ assert len(registry.instances.list_metadata()) == 1
+
+
+def test_registering_the_same_name_twice_is_refused(registry: ScenarioSuiteRegistry) -> None:
+ """Test one code owner per registered name."""
+ registry.register_suite(suite=_suite())
+
+ with pytest.raises(ValueError, match="weekly"):
+ registry.register_suite(suite=_suite())
+
+
+def test_stored_document_shadows_registered_suite(
+ registry: ScenarioSuiteRegistry,
+ tmp_path: Path,
+) -> None:
+ """Test operator precedence on both list and point reads."""
+ registry.register_suite(suite=_suite(description="code"))
+ (tmp_path / "weekly.json").write_text(
+ json.dumps({"description": "operator", "entries": []}),
+ encoding="utf-8",
+ )
+
+ listed = registry.list_suites()
+ read = registry.get_suite(name="weekly")
+
+ assert len(listed) == 1
+ assert listed[0].origin == "stored"
+ assert listed[0].suite.description == "operator"
+ assert read is not None
+ assert read.origin == "stored"
+ assert read.suite.description == "operator"
+
+
+def test_registering_a_shadowed_name_logs_the_precedence(
+ registry: ScenarioSuiteRegistry,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """Test that an existing collision is not silent."""
+ registry.save_suite(suite=_suite(), expected_version=None)
+
+ with caplog.at_level(logging.WARNING):
+ registry.register_suite(suite=_suite())
+
+ assert "takes precedence" in caplog.text
+
+
+def test_saving_a_registered_name_raises_dedicated_error(registry: ScenarioSuiteRegistry) -> None:
+ """Test actionable rejection distinct from optimistic concurrency."""
+ registry.register_suite(suite=_suite())
+
+ with pytest.raises(ScenarioSuiteNameRegisteredError) as caught:
+ registry.save_suite(suite=_suite(), expected_version=None)
+
+ assert caught.value.name == "weekly"
+ assert not isinstance(caught.value, ScenarioSuiteConflictError)
+
+
+def test_updating_a_shadowing_document_raises_registered_error(registry: ScenarioSuiteRegistry) -> None:
+ """Test that an existing operator shadow cannot turn a code-owned name into an edit path."""
+ created = registry.save_suite(suite=_suite(description="operator"), expected_version=None)
+ registry.register_suite(suite=_suite(description="code"))
+
+ with pytest.raises(ScenarioSuiteNameRegisteredError):
+ registry.save_suite(suite=_suite(description="updated"), expected_version=created.version)
+
+
+def test_update_with_stale_version_still_uses_storage_conflict(registry: ScenarioSuiteRegistry) -> None:
+ """Test that ordinary stored concurrency behavior is preserved."""
+ created = registry.save_suite(suite=_suite(), expected_version=None)
+ registry.save_suite(suite=_suite(description="winner"), expected_version=created.version)
+
+ with pytest.raises(ScenarioSuiteConflictError):
+ registry.save_suite(suite=_suite(description="stale"), expected_version=created.version)
+
+
+def test_delete_never_removes_a_registered_suite(registry: ScenarioSuiteRegistry) -> None:
+ """Test that deletion is limited to the persistence layer."""
+ registry.register_suite(suite=_suite())
+
+ assert registry.delete_suite(name="weekly") is False
+ assert registry.get_suite(name="weekly") is not None
+
+
+def test_delete_shadowed_document_reveals_registered_suite(registry: ScenarioSuiteRegistry) -> None:
+ """Test fallback to code after removing an operator override."""
+ registry.save_suite(suite=_suite(description="operator"), expected_version=None)
+ registry.register_suite(suite=_suite(description="code"))
+
+ assert registry.delete_suite(name="weekly") is True
+ remaining = registry.get_suite(name="weekly")
+ assert remaining is not None
+ assert remaining.origin == "registered"
+ assert remaining.suite.description == "code"
+
+
+def test_resolve_checks_stored_version(registry: ScenarioSuiteRegistry) -> None:
+ """Test future execution resolution against the reviewed document."""
+ created = registry.save_suite(suite=_suite(), expected_version=None)
+
+ resolved = registry.resolve_suite(name="weekly", expected_version=created.version)
+
+ assert resolved is not None
+ assert resolved.origin == "stored"
+ with pytest.raises(ScenarioSuiteConflictError):
+ registry.resolve_suite(name="weekly", expected_version="stale")
+
+
+def test_resolve_registered_suite_rejects_a_document_version(registry: ScenarioSuiteRegistry) -> None:
+ """Test transition from a reviewed document to a registered fallback."""
+ created = registry.save_suite(suite=_suite(), expected_version=None)
+ registry.delete_suite(name="weekly")
+ registry.register_suite(suite=_suite())
+
+ with pytest.raises(ScenarioSuiteConflictError):
+ registry.resolve_suite(name="weekly", expected_version=created.version)
+
+ resolved = registry.resolve_suite(name="weekly", expected_version=None)
+ assert resolved is not None
+ assert resolved.origin == "registered"
+
+
+def test_get_version_recovers_a_malformed_document(
+ registry: ScenarioSuiteRegistry,
+ tmp_path: Path,
+) -> None:
+ """Test overwriting unreadable operator data only with its current version."""
+ (tmp_path / "broken.json").write_text("{not json", encoding="utf-8")
+
+ version = registry.get_suite_version(name="broken")
+
+ assert version is not None
+ assert registry.get_suite(name="broken") is None
+ assert registry.save_suite(suite=_suite("broken"), expected_version=version).suite.name == "broken"
+
+
+def test_external_storage_changes_are_visible_immediately(
+ registry: ScenarioSuiteRegistry,
+ tmp_path: Path,
+) -> None:
+ """Test that the registry does not cache stored suites."""
+ assert registry.list_suites() == []
+ (tmp_path / "external.json").write_text(json.dumps({"entries": []}), encoding="utf-8")
+
+ assert [entry.suite.name for entry in registry.list_suites()] == ["external"]
+
+
+def test_startup_scan_counts_valid_suites_and_reports_invalid_ones(
+ registry: ScenarioSuiteRegistry,
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """Test startup visibility without failing on malformed operator data."""
+ registry.save_suite(suite=_suite(), expected_version=None)
+ (tmp_path / "broken.json").write_text("{not json", encoding="utf-8")
+
+ with caplog.at_level(logging.WARNING):
+ loaded = registry.scan_stored_suites()
+
+ assert loaded == 1
+ assert "broken" in caplog.text
+
+
+def test_unavailable_storage_does_not_fail_startup(caplog: pytest.LogCaptureFixture) -> None:
+ """Test startup resilience when the configured source is unreachable."""
+ storage = MagicMock(spec=ScenarioSuiteStorage)
+ storage.list_suites.side_effect = OSError("source unavailable")
+ registry = ScenarioSuiteRegistry(storage=storage)
+
+ with caplog.at_level(logging.ERROR):
+ assert registry.scan_stored_suites() == 0
+
+ assert "Could not read scenario suites" in caplog.text
+
+
+def test_configure_source_replaces_an_earlier_source(tmp_path: Path) -> None:
+ """Test registry ownership of suite persistence configuration."""
+ first = tmp_path / "first"
+ second = tmp_path / "second"
+ first.mkdir()
+ second.mkdir()
+ registry = ScenarioSuiteRegistry(storage=ScenarioSuiteStorage(source=str(first)))
+ registry.save_suite(suite=_suite(), expected_version=None)
+
+ registry.configure_source(str(second))
+
+ assert registry.source == str(second)
+ assert registry.list_suites() == []
+
+
+def test_constructing_registry_does_not_touch_storage() -> None:
+ """Test lazy default storage construction."""
+ with patch.object(ScenarioSuiteStorage, "__init__", side_effect=AssertionError("storage was constructed")):
+ ScenarioSuiteRegistry()
+
+
+def test_registry_singleton_is_shared() -> None:
+ """Test the common registry singleton lifecycle."""
+ ScenarioSuiteRegistry.reset_registry_singleton()
+ try:
+ assert ScenarioSuiteRegistry.get_registry_singleton() is ScenarioSuiteRegistry.get_registry_singleton()
+ finally:
+ ScenarioSuiteRegistry.reset_registry_singleton()
diff --git a/tests/unit/setup/test_reinitialization.py b/tests/unit/setup/test_reinitialization.py
index 40530580cf..d8f6129d57 100644
--- a/tests/unit/setup/test_reinitialization.py
+++ b/tests/unit/setup/test_reinitialization.py
@@ -15,7 +15,15 @@
from pyrit.common.apply_defaults import get_global_default_values
from pyrit.common.random_context import configure_random_seed, get_configured_random_seed
from pyrit.memory import AzureSQLMemory, CentralMemory, SQLiteMemory
-from pyrit.registry import AttackRegistry, InitializerRegistry, Registry, TargetRegistry
+from pyrit.models.catalog import ScenarioSuite
+from pyrit.registry import (
+ AttackRegistry,
+ InitializerRegistry,
+ Registry,
+ ScenarioSuiteRegistry,
+ ScenarioSuiteStorage,
+ TargetRegistry,
+)
from pyrit.setup.configuration_loader import ConfigurationLoader
from pyrit.setup.environment_loading import resolve_environment_async
from pyrit.setup.initialization import reset_setup_registries, validate_reinitialization_memory
@@ -60,6 +68,33 @@ def test_reset_setup_registries_discards_attack_catalog_without_resetting_memory
assert CentralMemory.get_memory_instance() is memory
+def test_reset_setup_registries_replaces_registered_suites_without_deleting_storage(tmp_path: Path) -> None:
+ storage = ScenarioSuiteStorage(source=str(tmp_path))
+ registry = ScenarioSuiteRegistry(storage=storage)
+ ScenarioSuiteRegistry.set_registry_singleton(registry)
+ stored = registry.save_suite(suite=ScenarioSuite(name="stored_suite"), expected_version=None)
+ registered = ScenarioSuite(name="registered_suite")
+ registry.register_suite(suite=registered)
+
+ reset_setup_registries()
+
+ replacement = ScenarioSuiteRegistry.get_registry_singleton()
+ replacement.configure_source(str(tmp_path))
+ assert replacement is not registry
+ assert replacement.get_suite(name=registered.name) is None
+ stored_after_reset = replacement.get_suite(name=stored.suite.name)
+ assert stored_after_reset is not None
+ assert stored_after_reset.version == stored.version
+
+ replacement.register_suite(suite=registered)
+ reset_setup_registries()
+
+ after_removal = ScenarioSuiteRegistry.get_registry_singleton()
+ after_removal.configure_source(str(tmp_path))
+ assert after_removal.get_suite(name=registered.name) is None
+ assert after_removal.get_suite(name=stored.suite.name) is not None
+
+
async def test_preflight_uses_isolated_registry_without_changing_live_state() -> None:
live_registry = InitializerRegistry.get_registry_singleton()
config = ConfigurationLoader(