diff --git a/docs/examples/evaluation_insight_results.md b/docs/examples/evaluation_insight_results.md new file mode 100644 index 00000000..2df1bc59 --- /dev/null +++ b/docs/examples/evaluation_insight_results.md @@ -0,0 +1,102 @@ +# Evaluation insight results + +Some evaluators go beyond scoring and group related failures across the evaluated +sessions. `Builtin.FailureAnalysis` is one. Add it to `evaluator_ids` and the +completed result carries its insights in `insight_results`, alongside the scores +from any other evaluators in the same run: + +```python +from bedrock_agentcore.evaluation import ( + BatchEvaluationRunConfig, + BatchEvaluationRunner, + BatchEvaluatorConfig, + CloudWatchDataSourceConfig, + FileDatasetProvider, +) + +config = BatchEvaluationRunConfig( + batch_evaluation_name="my-failure-analysis", + evaluator_config=BatchEvaluatorConfig( + evaluator_ids=["Builtin.FailureAnalysis", "Builtin.GoalSuccessRate"], + ), + data_source=CloudWatchDataSourceConfig( + service_names=["MyAgent.DEFAULT"], + log_group_names=["/aws/bedrock-agentcore/runtimes/MyAgent-abc123-DEFAULT"], + ), +) + +runner = BatchEvaluationRunner(region="us-west-2") +result = runner.run_dataset_evaluation( + agent_invoker=agent_invoker, + dataset=FileDatasetProvider("dataset.json").get_dataset(), + config=config, +) + +for insight in result.insight_results or []: + print(f"{insight.name} ({insight.affected_session_count} sessions)") + for cause in insight.root_causes or []: + print(f" {cause.name}: {cause.suggested_fix}") +``` + +`run_dataset_evaluation` polls until the job reaches a terminal state, so the +insights are already on the result it returns. + +An empty list and `None` mean different things. An empty list means the analysis +ran and found no failures, while `None` means no analysis evaluator was requested +or its results could not be read, in which case `error_details` names the +evaluator. Branch on the two separately rather than treating both as "no +insights": + +```python +if result.insight_results is None: + print("no analysis ran, or its results were unavailable:", result.error_details) +elif not result.insight_results: + print("analysis ran and found no failures") +``` + +The models take the response as it arrives on the wire and expose it as +snake_case attributes. Fields the response omits come back as `None`: + +```python +from bedrock_agentcore.evaluation import InsightResult + +insight = InsightResult.model_validate( + { + "type": "FAILURE", + "name": "Tool calls fail on missing permissions", + "description": "Tool calls that change service state fail for lack of permissions.", + "createdAt": "2026-10-15T21:21:11Z", + "affectedSessionCount": 2, + "rootCauses": [ + { + "name": "Execution role lacks the actions the tools call", + "explanation": "The role is missing the actions the restart tool calls.", + "suggestedFix": "Grant the role those actions.", + "sampleSessions": [ + { + "sessionId": "session-0001", + "findings": [ + { + "description": "restart_service failed with PermissionError.", + "targetId": "1a027c6a3f598862", + "targetType": "SPAN_ID", + } + ], + } + ], + } + ], + } +) + +assert insight.affected_session_count == 2 +assert insight.created_at.year == 2026 + +finding = insight.root_causes[0].sample_sessions[0].findings[0] +assert finding.target_type == "SPAN_ID" +assert finding.confidence is None +assert insight.root_causes[0].sample_sessions[0].last_seen_at is None +``` + +Sample sessions normally sit on each root cause. An insight carries its own +`sample_sessions` when it has no root causes. diff --git a/pyproject.toml b/pyproject.toml index 4a634f89..de0c99d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "boto3>=1.43.103", - "botocore>=1.43.103", + "boto3>=1.43.115", + "botocore>=1.43.115", "pydantic>=2.0.0,<2.41.3", "urllib3>=1.26.0", "starlette>=0.46.2", diff --git a/src/bedrock_agentcore/evaluation/__init__.py b/src/bedrock_agentcore/evaluation/__init__.py index b827feed..40bf85a5 100644 --- a/src/bedrock_agentcore/evaluation/__init__.py +++ b/src/bedrock_agentcore/evaluation/__init__.py @@ -19,7 +19,11 @@ EvaluatorStatistics, EvaluatorSummary, FailedScenario, + InsightResult, + InsightRootCause, OnlineEvaluationDataSourceConfig, + SampleSession, + SessionFinding, ) from bedrock_agentcore.evaluation.runner.batch.batch_evaluation_runner import ( BatchEvaluationRunner, @@ -81,6 +85,10 @@ "EvaluatorStatistics", "EvaluatorSummary", "FailedScenario", + "InsightResult", + "InsightRootCause", + "SampleSession", + "SessionFinding", "AgentInvokerInput", "AgentInvokerOutput", "CloudWatchAgentSpanCollector", diff --git a/src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_models.py b/src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_models.py index d8138ed9..18f4452a 100644 --- a/src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_models.py +++ b/src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_models.py @@ -245,6 +245,103 @@ class CloudWatchOutputDataConfig(BaseModel): log_stream_name: str +class SessionFinding(BaseModel): + """A failure found in a sampled session that placed it under an insight or root cause. + + .. warning:: + This feature is in preview and may change in future releases. + + Attributes: + description: Evidence text describing the failure. + target_id: Identifier of the span, trace, or span name the finding points at, + interpreted according to ``target_type``. + target_type: What ``target_id`` refers to, such as ``"SPAN_ID"``. + category: Failure category assigned by the analysis. + confidence: Confidence in the finding, between 0 and 1. + """ + + model_config = ConfigDict(alias_generator=alias_generators.to_camel, populate_by_name=True) + + description: Optional[str] = None + target_id: Optional[str] = None + target_type: Optional[str] = None + category: Optional[str] = None + confidence: Optional[float] = None + + +class SampleSession(BaseModel): + """A sampled session illustrating an insight or root cause. + + .. warning:: + This feature is in preview and may change in future releases. + + Attributes: + session_id: Identifier of the sampled session. + last_seen_at: Latest timestamp observed for the session in the batch input. + findings: Failures that placed this session under the insight or root cause. + """ + + model_config = ConfigDict(alias_generator=alias_generators.to_camel, populate_by_name=True) + + session_id: Optional[str] = None + last_seen_at: Optional[datetime] = None + findings: Optional[List[SessionFinding]] = None + + +class InsightRootCause(BaseModel): + """A root cause grouped under an insight. + + .. warning:: + This feature is in preview and may change in future releases. + + Attributes: + name: Short label for the root cause. + explanation: Why the grouped failures happen. + suggested_fix: Recommended change to address the root cause. + affected_session_count: Number of distinct sessions affected. A session can + appear under more than one root cause, so these counts can add up to more + than the insight's own count. + sample_sessions: Sessions illustrating this root cause. + """ + + model_config = ConfigDict(alias_generator=alias_generators.to_camel, populate_by_name=True) + + name: Optional[str] = None + explanation: Optional[str] = None + suggested_fix: Optional[str] = None + affected_session_count: Optional[int] = None + sample_sessions: Optional[List[SampleSession]] = None + + +class InsightResult(BaseModel): + """A job-level insight grouping related failures across the batch. + + .. warning:: + This feature is in preview and may change in future releases. + + Attributes: + type: Kind of insight, such as ``"FAILURE"``. + name: Short label for the insight. + description: What the grouped failures have in common. + created_at: When the insight was generated. + affected_session_count: Number of distinct sessions affected. + sample_sessions: Sessions illustrating the insight. Populated only when the + insight has no root causes, since otherwise the samples sit under each + root cause. + root_causes: Root causes grouped under this insight. + """ + + model_config = ConfigDict(alias_generator=alias_generators.to_camel, populate_by_name=True) + + type: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + created_at: Optional[datetime] = None + affected_session_count: Optional[int] = None + sample_sessions: Optional[List[SampleSession]] = None + root_causes: Optional[List[InsightRootCause]] = None + + class BatchEvaluationResult(BaseModel): """Result returned by :py:meth:`BatchEvaluationRunner.run`. @@ -263,6 +360,11 @@ class BatchEvaluationResult(BaseModel): the service. ``None`` if the API did not return it. evaluation_results: Aggregated per-evaluator statistics. Present when the job completed successfully; ``None`` otherwise. + insight_results: Job-level insights produced by an analysis evaluator such + as ``Builtin.FailureAnalysis``. An empty list means the analysis ran and + found no failures. ``None`` means no analysis evaluator was requested, or + its results could not be read, in which case ``error_details`` names the + evaluator. error_details: Service-reported error messages when the job failed. agent_invocation_failures: Scenarios that failed during the agent invocation phase (before the evaluation job was started). A @@ -285,6 +387,7 @@ class BatchEvaluationResult(BaseModel): created_at: datetime updated_at: Optional[datetime] = None evaluation_results: Optional[BatchEvaluationSummary] = None + insight_results: Optional[List[InsightResult]] = None error_details: Optional[List[str]] = None agent_invocation_failures: List[FailedScenario] = Field(default_factory=list) output_data_config: Optional[CloudWatchOutputDataConfig] = None diff --git a/src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_runner.py b/src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_runner.py index e16df71e..7f83a407 100644 --- a/src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_runner.py +++ b/src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_runner.py @@ -20,6 +20,7 @@ BatchEvaluationSummary, CloudWatchOutputDataConfig, FailedScenario, + InsightResult, ) from bedrock_agentcore.evaluation.runner.dataset_types import Dataset, PredefinedScenario, Scenario, SimulatedScenario from bedrock_agentcore.evaluation.runner.invoker_types import AgentInvokerFn @@ -429,6 +430,13 @@ def run_dataset_evaluation( if "evaluationResults" in response: evaluation_results = BatchEvaluationSummary.model_validate(response["evaluationResults"]) + # Absent and empty mean different things: the field is omitted when no + # analysis evaluator ran or its results were unreadable, and empty when the + # analysis ran and found no failures. Keep that distinction in the result. + insight_results = None + if "insightResults" in response: + insight_results = [InsightResult.model_validate(insight) for insight in response["insightResults"]] + output_data_config = None if "outputConfig" in response: odc = response["outputConfig"].get("cloudWatchConfig") @@ -448,6 +456,7 @@ def run_dataset_evaluation( description=response.get("description"), agent_invocation_failures=failed_scenarios, evaluation_results=evaluation_results, + insight_results=insight_results, error_details=response.get("errorDetails"), output_data_config=output_data_config, kms_key_arn=response.get("kmsKeyArn"), diff --git a/tests/bedrock_agentcore/evaluation/runner/batch/test_batch_evaluation_runner.py b/tests/bedrock_agentcore/evaluation/runner/batch/test_batch_evaluation_runner.py index 28680e74..a13158a9 100644 --- a/tests/bedrock_agentcore/evaluation/runner/batch/test_batch_evaluation_runner.py +++ b/tests/bedrock_agentcore/evaluation/runner/batch/test_batch_evaluation_runner.py @@ -108,7 +108,12 @@ def _make_start_response(batch_evaluation_id="eval-001"): } -def _make_completed_response(batch_evaluation_id="eval-001", with_output_data=False, with_eval_results=False): +def _make_completed_response( + batch_evaluation_id="eval-001", + with_output_data=False, + with_eval_results=False, + insight_results=None, +): response = { "batchEvaluationId": batch_evaluation_id, "batchEvaluationArn": f"arn:aws:bedrock-agentcore:us-west-2:123:batch-evaluation/{batch_evaluation_id}", @@ -140,9 +145,70 @@ def _make_completed_response(batch_evaluation_id="eval-001", with_output_data=Fa } ], } + if insight_results is not None: + response["insightResults"] = insight_results return response +_INSIGHT_RESULTS = [ + { + "type": "FAILURE", + "name": "IAM Permission Denial Errors", + "description": "The agent could not complete service operations it lacked permission for.", + "createdAt": _T1, + "affectedSessionCount": 1, + "rootCauses": [ + { + "name": "IAM Permission Gaps Blocking Agent Service Operations", + "explanation": "The execution role is missing the actions the tool calls require.", + "suggestedFix": "Grant the execution role the actions the restart tool calls.", + "affectedSessionCount": 1, + "sampleSessions": [ + { + "sessionId": "ops-restart-denied-1", + "lastSeenAt": _T1, + "findings": [ + { + "targetId": "1a027c6a3f598862", + "targetType": "SPAN_ID", + "description": "restart_service raised PermissionError.", + "category": "execution-error-authentication", + "confidence": 0.9, + } + ], + } + ], + } + ], + }, + { + # An insight carries its own sample sessions only when it has no root + # causes, so this entry covers that branch of the response. + "type": "FAILURE", + "name": "Query Timeout Errors", + "description": "Checkout-API queries exceeded the tool timeout.", + "createdAt": _T0, + "affectedSessionCount": 2, + "sampleSessions": [ + { + "sessionId": "ops-query-timeout-1", + "lastSeenAt": _T0, + "findings": [ + { + "targetId": "674d83289183f34e", + "targetType": "SPAN_ID", + "description": "query_metrics timed out after 30s.", + "category": "execution-error-timeout", + "confidence": 0.75, + } + ], + }, + {"sessionId": "ops-query-timeout-2"}, + ], + }, +] + + # --------------------------------------------------------------------------- # Helpers: patch runner internals # --------------------------------------------------------------------------- @@ -338,6 +404,86 @@ def test_run_parses_evaluation_results(): assert summary.statistics.average_score == 0.9 +def _run_with_insight_results(insight_results): + runner = _make_runner() + runner.data_plane_client.start_batch_evaluation.return_value = _make_start_response() + runner.data_plane_client.get_batch_evaluation.return_value = _make_completed_response( + insight_results=insight_results + ) + + with patch.object( + runner, + "_execute_scenarios_parallel", + return_value=( + [MagicMock(scenario_id="s1", session_id="s1-session-abc", start_time=_T0, end_time=_T1, ground_truth=None)], + [], + ), + ): + return runner.run_dataset_evaluation(config=_make_config(), dataset=_DATASET, agent_invoker=_make_invoker()) + + +def test_run_parses_insight_results(): + result = _run_with_insight_results(_INSIGHT_RESULTS) + + assert result.insight_results is not None + assert len(result.insight_results) == 2 + insight = result.insight_results[0] + assert insight.type == "FAILURE" + assert insight.name == "IAM Permission Denial Errors" + assert insight.description == "The agent could not complete service operations it lacked permission for." + assert insight.created_at == _T1 + assert insight.affected_session_count == 1 + + root_cause = insight.root_causes[0] + assert root_cause.name == "IAM Permission Gaps Blocking Agent Service Operations" + assert root_cause.explanation == "The execution role is missing the actions the tool calls require." + assert root_cause.suggested_fix == "Grant the execution role the actions the restart tool calls." + assert root_cause.affected_session_count == 1 + + sample = root_cause.sample_sessions[0] + assert sample.session_id == "ops-restart-denied-1" + assert sample.last_seen_at == _T1 + finding = sample.findings[0] + assert finding.target_type == "SPAN_ID" + assert finding.target_id == "1a027c6a3f598862" + assert finding.description == "restart_service raised PermissionError." + assert finding.category == "execution-error-authentication" + assert finding.confidence == 0.9 + + +def test_run_parses_insight_sample_sessions_without_root_causes(): + insight = _run_with_insight_results(_INSIGHT_RESULTS).insight_results[1] + + assert insight.root_causes is None + assert insight.created_at == _T0 + assert [sample.session_id for sample in insight.sample_sessions] == [ + "ops-query-timeout-1", + "ops-query-timeout-2", + ] + assert insight.sample_sessions[0].findings[0].confidence == 0.75 + # Every member is optional, so a session the service sends bare still parses. + assert insight.sample_sessions[1].last_seen_at is None + assert insight.sample_sessions[1].findings is None + + +def test_insight_models_round_trip_to_wire_names(): + insight = _run_with_insight_results(_INSIGHT_RESULTS).insight_results[0] + + dumped = insight.model_dump(by_alias=True, exclude_none=True) + assert set(dumped) == set(_INSIGHT_RESULTS[0]) + assert set(dumped["rootCauses"][0]) == set(_INSIGHT_RESULTS[0]["rootCauses"][0]) + assert set(dumped["rootCauses"][0]["sampleSessions"][0]["findings"][0]) == set( + _INSIGHT_RESULTS[0]["rootCauses"][0]["sampleSessions"][0]["findings"][0] + ) + + +def test_run_keeps_empty_insight_results_distinct_from_absent(): + # An empty list means the analysis ran and found no failures, while an absent + # field means no analysis evaluator ran or its results were unreadable. + assert _run_with_insight_results([]).insight_results == [] + assert _run_with_insight_results(None).insight_results is None + + def test_run_start_batch_evaluation_failure_raises_runtime_error(): runner = _make_runner() runner.data_plane_client.start_batch_evaluation.side_effect = Exception("service unavailable") diff --git a/uv.lock b/uv.lock index 260efb60..174c0fef 100644 --- a/uv.lock +++ b/uv.lock @@ -592,8 +592,8 @@ requires-dist = [ { name = "a2a-sdk", extras = ["http-server"], marker = "extra == 'a2a-v1'", specifier = ">=1.0.1,<2.0" }, { name = "ag-ui-protocol", marker = "extra == 'ag-ui'", specifier = ">=0.1.10" }, { name = "autoevals", marker = "extra == 'autoevals'", specifier = ">=0.3.0,<1.0.0" }, - { name = "boto3", specifier = ">=1.43.103" }, - { name = "botocore", specifier = ">=1.43.103" }, + { name = "boto3", specifier = ">=1.43.115" }, + { name = "botocore", specifier = ">=1.43.115" }, { name = "deepeval", marker = "extra == 'deepeval'", specifier = ">=3.5.0,<5.0.0" }, { name = "httpx", marker = "extra == 'langgraph'", specifier = ">=0.27.0" }, { name = "jinja2", marker = "extra == 'simulation'", specifier = ">=3.1.0" }, @@ -649,30 +649,30 @@ dev = [ [[package]] name = "boto3" -version = "1.43.103" +version = "1.43.115" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/46/59/012898d78087105e9c20fe31605d3f1999921745e1890ee0f0d993846e2e/boto3-1.43.103.tar.gz", hash = "sha256:524821052527f6446d249bf710847b032d9b12135e346c9751777a0a2811cf04", size = 112687, upload-time = "2026-09-25T19:26:30.427Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/b0ce191492a1724be73d8aacaf70d508bc8cbb9674c046d2ff7181dc51fb/boto3-1.43.115.tar.gz", hash = "sha256:c0f35e5f9cdf0edb365195e306d5586d3970b224660a11a87a4ec361ec3d7168", size = 112677, upload-time = "2026-10-19T19:19:53.093Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/66/6eb2e92c57b30e54c87fe14c8cd71de3aed9184f3cec95c39132a3fcc61f/boto3-1.43.103-py3-none-any.whl", hash = "sha256:574796187eb4e4fcaaf5c1b88c1724a1ba5a95bc27aeae5573676178666acc71", size = 140044, upload-time = "2026-09-25T19:26:29.085Z" }, + { url = "https://files.pythonhosted.org/packages/be/bc/40dffae3238ef8d3415393132751fe2c88e8663ef84a31f0950e439ed2d5/boto3-1.43.115-py3-none-any.whl", hash = "sha256:6667a0c21d48dc125dbba88444bc8a601ce881a99341a0e6169f9333898e620d", size = 140044, upload-time = "2026-10-19T19:19:51.444Z" }, ] [[package]] name = "botocore" -version = "1.43.103" +version = "1.43.115" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/83/acbee6f2e3b1f02de935fef0c48a387511bf979149dc6b44466aaafba6a5/botocore-1.43.103.tar.gz", hash = "sha256:8c7f220e09f3b7ec99c59c716aa66cf6f5b2b57fd8731544054f3778aa744c6a", size = 16243609, upload-time = "2026-09-25T19:26:25.884Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/10/ca525e88db50f5153ddad2a3d55908b5ace915d10f443d5e5135ba808d8d/botocore-1.43.115.tar.gz", hash = "sha256:aa4d9c4e7ef99988b7347e0cd276ba84e1eb1625510557b0e36cd57ecb19bbf2", size = 16383796, upload-time = "2026-10-19T19:19:45.382Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/47/790ba88ec849d1e07b5866458b67e79b291164f3de7f23429b4dcb7e2ced/botocore-1.43.103-py3-none-any.whl", hash = "sha256:6a6a561a6de43845ae42bc7ff7e2c798742bfa277f5215398f9c15ed9b1bd67c", size = 15938608, upload-time = "2026-09-25T19:26:23.162Z" }, + { url = "https://files.pythonhosted.org/packages/77/2d/4b6b6f5704d1ec9b412c1dd9700bda0c315619dbac4a21418180a81a2780/botocore-1.43.115-py3-none-any.whl", hash = "sha256:9ceda640c868bc15f5399129e8cfdef8cbb4b350e8a99547c9b5f477cfb9619a", size = 16085101, upload-time = "2026-10-19T19:19:42.203Z" }, ] [package.optional-dependencies]