From 48f89c6750a940eaf769afd843f44f2b7977f0ca Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Mon, 19 Oct 2026 23:35:42 -0700 Subject: [PATCH 1/2] TEST Use shared watchdog for backend correctness helpers Keep manual-send terminal polling non-cancelling and align the analytics singleton correctness test with the shared deadlock backstop. Guard these budgets and prove a timed-out observer leaves a send able to finish. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1b1dc1a4-802e-4475-9c27-f178cf67a8ab --- doc/contributing/4_running_tests.md | 6 +++ tests/unit/backend/conftest.py | 8 +++- tests/unit/backend/mocks.py | 5 ++- tests/unit/backend/test_analytics_routes.py | 11 ++++- .../unit/backend/test_message_send_service.py | 45 ++++++++++++++++++- 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/doc/contributing/4_running_tests.md b/doc/contributing/4_running_tests.md index db21b98cb1..0f30a9a8db 100644 --- a/doc/contributing/4_running_tests.md +++ b/doc/contributing/4_running_tests.md @@ -84,6 +84,12 @@ timeout coverage expires the real cancellation path only after the request start Event-controlled analytics expiry tests freeze both deadline clocks so floating-point rounding cannot turn a valid query budget into a failed assertion. +Backend manual-send completion and first-request analytics correctness tests also +use the shared watchdog. Manual sends are polled until a terminal snapshot, including +finalization. Expiring the test's wait does not cancel the send; its lifecycle owner +still drains pending work before mocks and memory close. Explicit zero-timeout tests +check this ownership without requiring the send to finish quickly. + For scenario correctness tests, wait for the owned task to finish, including persistence and scheduler handoff. Use events to control concurrent steps and a finite test watchdog to detect deadlocks, not short per-step latency limits. diff --git a/tests/unit/backend/conftest.py b/tests/unit/backend/conftest.py index 36f385d235..6d0ce4bde3 100644 --- a/tests/unit/backend/conftest.py +++ b/tests/unit/backend/conftest.py @@ -19,12 +19,18 @@ from pyrit.backend.services.message_send_service import get_message_send_service from pyrit.memory import SQLiteMemory from test_support.fixtures import authentication_policy, compatibility_headers, compatibility_id # noqa: F401 +from unit.async_utils import TEST_WATCHDOG_SECONDS @pytest.fixture async def analytics_service(sqlite_instance: SQLiteMemory) -> AsyncGenerator[AttackResultAnalytics, None]: """Use real isolated memory with finite correctness budgets, not production latency assertions.""" - execution = partial(AnalyticsExecution, queue_timeout=300.0, report_timeout=300.0, quick_timeout=300.0) + execution = partial( + AnalyticsExecution, + queue_timeout=TEST_WATCHDOG_SECONDS, + report_timeout=TEST_WATCHDOG_SECONDS, + quick_timeout=TEST_WATCHDOG_SECONDS, + ) assert get_analytics_service.cache_info().currsize == 0 with patch("pyrit.analytics.attack_result_analytics.AnalyticsExecution", side_effect=execution): try: diff --git a/tests/unit/backend/mocks.py b/tests/unit/backend/mocks.py index 48f65cf08a..285d705371 100644 --- a/tests/unit/backend/mocks.py +++ b/tests/unit/backend/mocks.py @@ -14,12 +14,13 @@ from pyrit.memory import MemoryInterface from pyrit.models import AtomicAttackIdentifier, AttackOutcome, AttackResult, ComponentIdentifier from pyrit.prompt_target import PromptTarget +from unit.async_utils import TEST_WATCHDOG_SECONDS async def _settle_send_async( - *, service: MessageSendService, status: MessageSendStatus, timeout_seconds: float = 10 + *, service: MessageSendService, status: MessageSendStatus, timeout_seconds: float = TEST_WATCHDOG_SECONDS ) -> MessageSendStatus: - """Wait for a terminal snapshot without cancelling the send on timeout.""" + """Wait for a terminal snapshot with a deadlock backstop, without cancelling the send.""" async with asyncio.timeout(timeout_seconds): while status.state not in (MessageSendState.COMPLETED, MessageSendState.FAILED, MessageSendState.INTERRUPTED): status = await service.get_status_async( diff --git a/tests/unit/backend/test_analytics_routes.py b/tests/unit/backend/test_analytics_routes.py index 02e1a766c5..a5d61d8ed9 100644 --- a/tests/unit/backend/test_analytics_routes.py +++ b/tests/unit/backend/test_analytics_routes.py @@ -43,6 +43,7 @@ AttackAnalyticsResultsQuery, AttackOutcome, ) +from unit.async_utils import TEST_WATCHDOG_SECONDS from unit.memory.test_attack_analytics import make_result @@ -123,7 +124,12 @@ def create() -> AttackResultAnalytics: assert get_analytics_service.cache_info().currsize == 0 user = AuthenticatedUser(oid="test", name="Test", email="test@example.invalid", groups=["test-group"]) - budgets = partial(AnalyticsExecution, report_timeout=30.0, quick_timeout=30.0, queue_timeout=30.0) + budgets = partial( + AnalyticsExecution, + report_timeout=TEST_WATCHDOG_SECONDS, + quick_timeout=TEST_WATCHDOG_SECONDS, + queue_timeout=TEST_WATCHDOG_SECONDS, + ) with ( patch("pyrit.backend.services.analytics_service.AttackResultAnalytics", side_effect=create), patch("pyrit.analytics.attack_result_analytics.AnalyticsExecution", side_effect=budgets), @@ -141,6 +147,9 @@ def create() -> AttackResultAnalytics: assert all(response.status_code == 200 for response in responses) assert len(owners) == 1 assert get_analytics_service() is owners[0] + execution = owners[0]._execution() + assert execution._queue_timeout == TEST_WATCHDOG_SECONDS + assert all(lane.timeout == TEST_WATCHDOG_SECONDS for lane in execution._lanes.values()) finally: await close_analytics_service_async() diff --git a/tests/unit/backend/test_message_send_service.py b/tests/unit/backend/test_message_send_service.py index 40dc447eb0..6762c46990 100644 --- a/tests/unit/backend/test_message_send_service.py +++ b/tests/unit/backend/test_message_send_service.py @@ -4,6 +4,7 @@ """Tests for the shared synchronous manual-message owner.""" import asyncio +import inspect import uuid from collections.abc import AsyncGenerator, AsyncIterator, Generator, Iterator, Sequence from contextlib import asynccontextmanager, contextmanager @@ -64,7 +65,7 @@ from pyrit.prompt_normalizer import ConverterConfiguration, PromptNormalizer from pyrit.prompt_target.common.target_capabilities import CapabilityName, TargetCapabilities from pyrit.prompt_target.common.target_configuration import TargetConfiguration -from unit.async_utils import wait_for_completion_async +from unit.async_utils import TEST_WATCHDOG_SECONDS, wait_for_completion_async from unit.backend.mocks import ( _make_matching_target_mock, _settle_send_async, @@ -2868,6 +2869,45 @@ async def test_settle_send_waits_past_finalizing_async(self, terminal_state: Mes for call in service.get_status_async.await_args_list ) + def test_settle_send_defaults_to_shared_watchdog(self) -> None: + parameter = inspect.signature(_settle_send_async).parameters["timeout_seconds"] + assert parameter.default == TEST_WATCHDOG_SECONDS + + async def test_settle_send_timeout_preserves_send_until_success_async( + self, + *, + message_send_service: MessageSendService, + send_dependencies: tuple[MagicMock, AsyncMock], + ) -> None: + started: asyncio.Future[None] = asyncio.get_running_loop().create_future() + release = asyncio.Event() + + async def hold_send_async(**_: Any) -> None: + started.set_result(None) + await release.wait() + + send_dependencies[1].side_effect = hold_send_async + async with message_send_lifecycle_async(message_send_service): + try: + status = await message_send_service.submit_async(attack_result_id="attack", request=_submission()) + await wait_for_completion_async(future=started) + task = message_send_service._sends[status.send_id].task + assert task is not None + + with pytest.raises(TimeoutError): + await _settle_send_async(service=message_send_service, status=status, timeout_seconds=0) + assert not task.done() + assert task.cancelling() == 0 + + release.set() + final = await _settle_send_async(service=message_send_service, status=status) + assert final.state == MessageSendState.COMPLETED + assert task.done() + assert not task.cancelled() + assert not message_send_service._scheduler._conversations + finally: + release.set() + @pytest.mark.parametrize("failure", ["assertion", "timeout"]) async def test_send_lifecycle_drains_pending_work_on_test_failure_async( self, @@ -2898,9 +2938,10 @@ async def hold_send_async(**_: Any) -> None: if failure == "assertion": raise AssertionError("controlled test failure") try: - await _settle_send_async(service=message_send_service, status=status, timeout_seconds=0.01) + await _settle_send_async(service=message_send_service, status=status, timeout_seconds=0) except TimeoutError: assert not task.done() + assert task.cancelling() == 0 assert not cancelled.is_set() raise From b2b257ee583c9ec2e4eb8e4995e6bbd62dfdc7fd Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Fri, 23 Oct 2026 07:09:21 -0700 Subject: [PATCH 2/2] TEST Allow manual-send watchdogs to expire before pytest timeout Raise async and repeated send class backstops above the shared completion watchdog, guard their effective timeout markers, and document the budgets. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1b1dc1a4-802e-4475-9c27-f178cf67a8ab --- doc/contributing/4_running_tests.md | 2 ++ tests/unit/backend/test_message_send_service.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/doc/contributing/4_running_tests.md b/doc/contributing/4_running_tests.md index eac7391915..d01d0680da 100644 --- a/doc/contributing/4_running_tests.md +++ b/doc/contributing/4_running_tests.md @@ -96,6 +96,8 @@ use the shared watchdog. Manual sends are polled until a terminal snapshot, incl finalization. Expiring the test's wait does not cancel the send; its lifecycle owner still drains pending work before mocks and memory close. Explicit zero-timeout tests check this ownership without requiring the send to finish quickly. +The async and repeated manual-send test classes use 600-second whole-test backstops +so the 300-second completion watchdog can expire and allow cleanup before pytest stops the test. The scorer-validation cancellation test keeps its five-second observer around scorer draining and native SQLite connection cleanup. ERROR-result persistence is diff --git a/tests/unit/backend/test_message_send_service.py b/tests/unit/backend/test_message_send_service.py index c4e47873ec..ba0f82fa44 100644 --- a/tests/unit/backend/test_message_send_service.py +++ b/tests/unit/backend/test_message_send_service.py @@ -2787,7 +2787,7 @@ def _submission( ) -@pytest.mark.timeout(20) +@pytest.mark.timeout(2 * TEST_WATCHDOG_SECONDS) @pytest.mark.usefixtures("patch_central_database") class TestAsyncMessageSend: async def test_repetitions_cannot_bypass_original_capabilities_through_an_alias_async( @@ -2884,9 +2884,12 @@ async def test_settle_send_waits_past_finalizing_async(self, terminal_state: Mes for call in service.get_status_async.await_args_list ) - def test_settle_send_defaults_to_shared_watchdog(self) -> None: + def test_settle_send_defaults_to_shared_watchdog(self, request: pytest.FixtureRequest) -> None: parameter = inspect.signature(_settle_send_async).parameters["timeout_seconds"] assert parameter.default == TEST_WATCHDOG_SECONDS + timeout = request.node.get_closest_marker("timeout") + assert timeout is not None + assert timeout.args[0] > TEST_WATCHDOG_SECONDS async def test_settle_send_timeout_preserves_send_until_success_async( self, @@ -3578,9 +3581,14 @@ async def hold_write_async(**_: Any) -> None: ).state == MessageSendState.INTERRUPTED -@pytest.mark.timeout(20) +@pytest.mark.timeout(2 * TEST_WATCHDOG_SECONDS) @pytest.mark.usefixtures("patch_central_database") class TestRepeatedMessageSend: + def test_whole_test_timeout_exceeds_send_watchdog(self, request: pytest.FixtureRequest) -> None: + timeout = request.node.get_closest_marker("timeout") + assert timeout is not None + assert timeout.args[0] > TEST_WATCHDOG_SECONDS + async def test_missing_validated_target_fails_before_branch_registration_async( self, *,