feat(evaluation): surface batch evaluation insight results

docs/examples/evaluation_insight_results.md

@@ -0,0 +1,102 @@
+# Evaluation insight results
+
+Some evaluators go beyond scoring and group related failures across the evaluated
+sessions. `Builtin.FailureAnalysis` is one. Add it to `evaluator_ids` and the
+completed result carries its insights in `insight_results`, alongside the scores
+from any other evaluators in the same run:
+
+```python
+from bedrock_agentcore.evaluation import (
+    BatchEvaluationRunConfig,
+    BatchEvaluationRunner,
+    BatchEvaluatorConfig,
+    CloudWatchDataSourceConfig,
+    FileDatasetProvider,
+)
+
+config = BatchEvaluationRunConfig(
+    batch_evaluation_name="my-failure-analysis",
+    evaluator_config=BatchEvaluatorConfig(
+        evaluator_ids=["Builtin.FailureAnalysis", "Builtin.GoalSuccessRate"],
+    ),
+    data_source=CloudWatchDataSourceConfig(
+        service_names=["MyAgent.DEFAULT"],
+        log_group_names=["/aws/bedrock-agentcore/runtimes/MyAgent-abc123-DEFAULT"],
+    ),
+)
+
+runner = BatchEvaluationRunner(region="us-west-2")
+result = runner.run_dataset_evaluation(
+    agent_invoker=agent_invoker,
+    dataset=FileDatasetProvider("dataset.json").get_dataset(),
+    config=config,
+)
+
+for insight in result.insight_results or []:
+    print(f"{insight.name} ({insight.affected_session_count} sessions)")
+    for cause in insight.root_causes or []:
+        print(f"  {cause.name}: {cause.suggested_fix}")
+```
+
+`run_dataset_evaluation` polls until the job reaches a terminal state, so the
+insights are already on the result it returns.
+
+An empty list and `None` mean different things. An empty list means the analysis
+ran and found no failures, while `None` means no analysis evaluator was requested
+or its results could not be read, in which case `error_details` names the
+evaluator. Branch on the two separately rather than treating both as "no
+insights":
+
+```python
+if result.insight_results is None:
+    print("no analysis ran, or its results were unavailable:", result.error_details)
+elif not result.insight_results:
+    print("analysis ran and found no failures")
+```
+
+The models take the response as it arrives on the wire and expose it as
+snake_case attributes. Fields the response omits come back as `None`:
+
+```python
+from bedrock_agentcore.evaluation import InsightResult
+
+insight = InsightResult.model_validate(
+    {
+        "type": "FAILURE",
+        "name": "Tool calls fail on missing permissions",
+        "description": "Tool calls that change service state fail for lack of permissions.",
+        "createdAt": "2026-10-15T21:21:11Z",
+        "affectedSessionCount": 2,
+        "rootCauses": [
+            {
+                "name": "Execution role lacks the actions the tools call",
+                "explanation": "The role is missing the actions the restart tool calls.",
+                "suggestedFix": "Grant the role those actions.",
+                "sampleSessions": [
+                    {
+                        "sessionId": "session-0001",
+                        "findings": [
+                            {
+                                "description": "restart_service failed with PermissionError.",
+                                "targetId": "1a027c6a3f598862",
+                                "targetType": "SPAN_ID",
+                            }
+                        ],
+                    }
+                ],
+            }
+        ],
+    }
+)
+
+assert insight.affected_session_count == 2
+assert insight.created_at.year == 2026
+
+finding = insight.root_causes[0].sample_sessions[0].findings[0]
+assert finding.target_type == "SPAN_ID"
+assert finding.confidence is None
+assert insight.root_causes[0].sample_sessions[0].last_seen_at is None
+```
+
+Sample sessions normally sit on each root cause. An insight carries its own
+`sample_sessions` when it has no root causes.

pyproject.toml

@@ -26,8 +26,8 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
-    "boto3>=1.43.103",
-    "botocore>=1.43.103",
+    "boto3>=1.43.115",
+    "botocore>=1.43.115",
     "pydantic>=2.0.0,<2.41.3",
     "urllib3>=1.26.0",
     "starlette>=0.46.2",

src/bedrock_agentcore/evaluation/__init__.py

@@ -19,7 +19,11 @@
     EvaluatorStatistics,
     EvaluatorSummary,
     FailedScenario,
+    InsightResult,
+    InsightRootCause,
     OnlineEvaluationDataSourceConfig,
+    SampleSession,
+    SessionFinding,
 )
 from bedrock_agentcore.evaluation.runner.batch.batch_evaluation_runner import (
     BatchEvaluationRunner,
@@ -81,6 +85,10 @@
     "EvaluatorStatistics",
     "EvaluatorSummary",
     "FailedScenario",
+    "InsightResult",
+    "InsightRootCause",
+    "SampleSession",
+    "SessionFinding",
     "AgentInvokerInput",
     "AgentInvokerOutput",
     "CloudWatchAgentSpanCollector",

src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_models.py

@@ -245,6 +245,103 @@ class CloudWatchOutputDataConfig(BaseModel):
     log_stream_name: str
 
 
+class SessionFinding(BaseModel):
+    """A failure found in a sampled session that placed it under an insight or root cause.
+
+    .. warning::
+        This feature is in preview and may change in future releases.
+
+    Attributes:
+        description: Evidence text describing the failure.
+        target_id: Identifier of the span, trace, or span name the finding points at,
+            interpreted according to ``target_type``.
+        target_type: What ``target_id`` refers to, such as ``"SPAN_ID"``.
+        category: Failure category assigned by the analysis.
+        confidence: Confidence in the finding, between 0 and 1.
+    """
+
+    model_config = ConfigDict(alias_generator=alias_generators.to_camel, populate_by_name=True)
+
+    description: Optional[str] = None
+    target_id: Optional[str] = None
+    target_type: Optional[str] = None
+    category: Optional[str] = None
+    confidence: Optional[float] = None
+
+
+class SampleSession(BaseModel):
+    """A sampled session illustrating an insight or root cause.
+
+    .. warning::
+        This feature is in preview and may change in future releases.
+
+    Attributes:
+        session_id: Identifier of the sampled session.
+        last_seen_at: Latest timestamp observed for the session in the batch input.
+        findings: Failures that placed this session under the insight or root cause.
+    """
+
+    model_config = ConfigDict(alias_generator=alias_generators.to_camel, populate_by_name=True)
+
+    session_id: Optional[str] = None
+    last_seen_at: Optional[datetime] = None
+    findings: Optional[List[SessionFinding]] = None
+
+
+class InsightRootCause(BaseModel):
+    """A root cause grouped under an insight.
+
+    .. warning::
+        This feature is in preview and may change in future releases.
+
+    Attributes:
+        name: Short label for the root cause.
+        explanation: Why the grouped failures happen.
+        suggested_fix: Recommended change to address the root cause.
+        affected_session_count: Number of distinct sessions affected. A session can
+            appear under more than one root cause, so these counts can add up to more
+            than the insight's own count.
+        sample_sessions: Sessions illustrating this root cause.
+    """
+
+    model_config = ConfigDict(alias_generator=alias_generators.to_camel, populate_by_name=True)
+
+    name: Optional[str] = None
+    explanation: Optional[str] = None
+    suggested_fix: Optional[str] = None
+    affected_session_count: Optional[int] = None
+    sample_sessions: Optional[List[SampleSession]] = None
+
+
+class InsightResult(BaseModel):
+    """A job-level insight grouping related failures across the batch.
+
+    .. warning::
+        This feature is in preview and may change in future releases.
+
+    Attributes:
+        type: Kind of insight, such as ``"FAILURE"``.
+        name: Short label for the insight.
+        description: What the grouped failures have in common.
+        created_at: When the insight was generated.
+        affected_session_count: Number of distinct sessions affected.
+        sample_sessions: Sessions illustrating the insight. Populated only when the
+            insight has no root causes, since otherwise the samples sit under each
+            root cause.
+        root_causes: Root causes grouped under this insight.
+    """
+
+    model_config = ConfigDict(alias_generator=alias_generators.to_camel, populate_by_name=True)
+
+    type: Optional[str] = None
+    name: Optional[str] = None
+    description: Optional[str] = None
+    created_at: Optional[datetime] = None
+    affected_session_count: Optional[int] = None
+    sample_sessions: Optional[List[SampleSession]] = None
+    root_causes: Optional[List[InsightRootCause]] = None
+
+
 class BatchEvaluationResult(BaseModel):
     """Result returned by :py:meth:`BatchEvaluationRunner.run`.
 
@@ -263,6 +360,11 @@ class BatchEvaluationResult(BaseModel):
             the service. ``None`` if the API did not return it.
         evaluation_results: Aggregated per-evaluator statistics. Present when
             the job completed successfully; ``None`` otherwise.
+        insight_results: Job-level insights produced by an analysis evaluator such
+            as ``Builtin.FailureAnalysis``. An empty list means the analysis ran and
+            found no failures. ``None`` means no analysis evaluator was requested, or
+            its results could not be read, in which case ``error_details`` names the
+            evaluator.
         error_details: Service-reported error messages when the job failed.
         agent_invocation_failures: Scenarios that failed during the agent
             invocation phase (before the evaluation job was started). A
@@ -285,6 +387,7 @@ class BatchEvaluationResult(BaseModel):
     created_at: datetime
     updated_at: Optional[datetime] = None
     evaluation_results: Optional[BatchEvaluationSummary] = None
+    insight_results: Optional[List[InsightResult]] = None
     error_details: Optional[List[str]] = None
     agent_invocation_failures: List[FailedScenario] = Field(default_factory=list)
     output_data_config: Optional[CloudWatchOutputDataConfig] = None

src/bedrock_agentcore/evaluation/runner/batch/batch_evaluation_runner.py

@@ -20,6 +20,7 @@
     BatchEvaluationSummary,
     CloudWatchOutputDataConfig,
     FailedScenario,
+    InsightResult,
 )
 from bedrock_agentcore.evaluation.runner.dataset_types import Dataset, PredefinedScenario, Scenario, SimulatedScenario
 from bedrock_agentcore.evaluation.runner.invoker_types import AgentInvokerFn
@@ -429,6 +430,13 @@ def run_dataset_evaluation(
         if "evaluationResults" in response:
             evaluation_results = BatchEvaluationSummary.model_validate(response["evaluationResults"])
 
+        # Absent and empty mean different things: the field is omitted when no
+        # analysis evaluator ran or its results were unreadable, and empty when the
+        # analysis ran and found no failures. Keep that distinction in the result.
+        insight_results = None
+        if "insightResults" in response:
+            insight_results = [InsightResult.model_validate(insight) for insight in response["insightResults"]]
+
         output_data_config = None
         if "outputConfig" in response:
             odc = response["outputConfig"].get("cloudWatchConfig")
@@ -448,6 +456,7 @@ def run_dataset_evaluation(
             description=response.get("description"),
             agent_invocation_failures=failed_scenarios,
             evaluation_results=evaluation_results,
+            insight_results=insight_results,
             error_details=response.get("errorDetails"),
             output_data_config=output_data_config,
             kms_key_arn=response.get("kmsKeyArn"),