diff --git a/agentplatform/_genai/_evals_visualization.py b/agentplatform/_genai/_evals_visualization.py index b6d335e7c5..470c71d952 100644 --- a/agentplatform/_genai/_evals_visualization.py +++ b/agentplatform/_genai/_evals_visualization.py @@ -23,7 +23,7 @@ from typing import Any, Optional import pandas as pd -from pydantic import errors +import pydantic_core from . import _evals_common from . import types @@ -1466,7 +1466,7 @@ def display_evaluation_result( result_dump = eval_result_obj.model_dump( mode="json", exclude_none=True, exclude={"evaluation_dataset"} ) - except errors.PydanticSerializationError as e: + except pydantic_core.PydanticSerializationError as e: logger.error( "Serialization Error: %s\nCould not display the evaluation " "result due to a data serialization issue. Please check the " diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index d59acc7cfa..eb4eb3194f 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -35,6 +35,7 @@ ) from google.genai import _common from google.genai import types as genai_types +import pydantic from pydantic import ( ConfigDict, Field, @@ -3130,6 +3131,32 @@ def _check_pandas_installed(cls, data: Any) -> Any: ) return data + @field_validator("eval_dataset_df", mode="before") + @classmethod + def _eval_dataset_df_from_records(cls, value: Any) -> Any: + if pd is not None and isinstance(value, list): + return pd.DataFrame(value) + return value + + @pydantic.field_serializer("eval_dataset_df", when_used="json") + def _eval_dataset_df_to_records(self, value: Any) -> Any: + if pd is None or not isinstance(value, pd.DataFrame): + return value + import numpy as np + + def to_serializable(item: Any) -> Any: + if isinstance(item, dict): + return {key: to_serializable(val) for key, val in item.items()} + if isinstance(item, (list, tuple, np.ndarray, pd.Series)): + return [to_serializable(val) for val in item] + if isinstance(item, np.datetime64): + item = pd.Timestamp(item) + elif isinstance(item, np.generic): + return item.item() + return None if item is pd.NaT or item is pd.NA else item + + return to_serializable(value.to_dict(orient="records")) + @classmethod def load_from_observability_eval_cases( cls, cases: list["ObservabilityEvalCase"] diff --git a/agentplatform/_genai/types/evals.py b/agentplatform/_genai/types/evals.py index b50253ce6c..23c2a22d78 100644 --- a/agentplatform/_genai/types/evals.py +++ b/agentplatform/_genai/types/evals.py @@ -948,7 +948,7 @@ class RubricVerdict(_common.BaseModel): generated.""", ) verdict: Optional[bool] = Field( - default=None, + default=False, description="""Required. Outcome of the evaluation against the rubric, represented as a boolean. `true` indicates a "Pass", `false` indicates a "Fail".""", ) diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 28807dfc67..cb86eee33e 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -44,11 +44,18 @@ types as agentplatform_genai_types, ) from agentplatform._genai.types import common as common_types +from vertexai._genai import ( + _evals_visualization as vertexai_evals_visualization, +) +from vertexai._genai import evals as vertexai_evals +from vertexai._genai import types as vertexai_genai_types from google.genai import client from google.genai import errors as genai_errors from google.genai import types as genai_types +import numpy as np import pandas as pd import pydantic +import pydantic_core import pytest _TEST_PROJECT = "test-project" @@ -2270,6 +2277,48 @@ def test_eval_evaluate_with_agent_info(self, mock_execute_evaluation): assert kwargs["agent_info"] == agent_info +class TestRubricVerdict: + + @pytest.mark.parametrize( + "evals_module", [evals, vertexai_evals], ids=["agent_platform", "vertexai"] + ) + def test_evaluate_instances_missing_verdict_is_false(self, evals_module): + api_client = mock.MagicMock() + api_client.vertexai = True + api_client.request.return_value.body = json.dumps( + { + "metricResults": [ + { + "score": 0.5, + "rubricVerdicts": [ + { + "evaluatedRubric": { + "content": { + "property": {"description": "In English."} + } + }, + "verdict": True, + }, + { + "evaluatedRubric": { + "content": { + "property": {"description": "One sentence."} + } + }, + "reasoning": "The response has two sentences.", + }, + ], + } + ] + } + ) + + response = evals_module.Evals(api_client_=api_client)._evaluate_instances() + + verdicts = response.metric_results[0].rubric_verdicts + assert [verdict.verdict for verdict in verdicts] == [True, False] + + class TestEvalsVisualization: # fmt: off @mock.patch( @@ -2407,6 +2456,31 @@ def test_display_evaluation_result_with_non_ascii_character(self, mock_is_ipytho del sys.modules["IPython"] del sys.modules["IPython.display"] + @pytest.mark.parametrize( + "visualization_module", + [_evals_visualization, vertexai_evals_visualization], + ids=["agent_platform", "vertexai"], + ) + @mock.patch.dict(sys.modules, {"IPython": mock.MagicMock()}) + def test_display_evaluation_result_logs_serialization_error( + self, visualization_module + ): + eval_result = mock.Mock() + eval_result.model_dump.side_effect = pydantic_core.PydanticSerializationError( + "bad value" + ) + + with ( + mock.patch.object( + visualization_module, "_is_ipython_env", return_value=True + ), + mock.patch.object(visualization_module, "logger") as mock_logger, + ): + visualization_module.display_evaluation_result(eval_result) + + mock_logger.error.assert_called_once() + assert "Serialization Error" in mock_logger.error.call_args[0][0] + class TestEvalsRunInference: """Unit tests for the Evals run_inference method.""" @@ -9442,6 +9516,78 @@ def test_predefined_metric_retry_fail_on_resource_exhausted( class TestEvaluationDataset: """Contains set of tests for the EvaluationDataset class methods.""" + @pytest.mark.parametrize( + "types_module", + [agentplatform_genai_types, vertexai_genai_types], + ids=["agent_platform", "vertexai"], + ) + def test_evaluation_dataset_json_with_arrays_missing_values_and_models( + self, types_module + ): + df = pd.DataFrame( + { + "tool_names": [ + np.array(["search", "book"]), + np.array([], dtype=object), + ], + "created_at": pd.to_datetime(["2026-10-02T10:00:00Z", None]), + "metadata": [{"turn": np.int64(3)}, None], + "score": [0.1 + 0.2, float("nan")], + "response": [ + genai_types.Content(parts=[genai_types.Part(text="r1")]), + None, + ], + } + ) + + records = json.loads( + types_module.EvaluationDataset(eval_dataset_df=df).model_dump_json( + exclude_none=True + ) + )["eval_dataset_df"] + + assert records == [ + { + "tool_names": ["search", "book"], + "created_at": "2026-10-02T10:00:00Z", + "metadata": {"turn": 3}, + "score": 0.30000000000000004, + "response": {"parts": [{"text": "r1"}]}, + }, + { + "tool_names": [], + "created_at": None, + "metadata": None, + "score": None, + "response": None, + }, + ] + + @pytest.mark.parametrize( + "types_module", + [agentplatform_genai_types, vertexai_genai_types], + ids=["agent_platform", "vertexai"], + ) + def test_evaluation_result_with_dataframe_json_round_trip(self, types_module): + df = pd.DataFrame([{"prompt": "p1", "response": "r1"}]) + result = types_module.EvaluationResult( + evaluation_dataset=[types_module.EvaluationDataset(eval_dataset_df=df)] + ) + + result_json = result.model_dump_json() + restored = types_module.EvaluationResult.model_validate_json(result_json) + + assert json.loads(result_json)["evaluation_dataset"][0]["eval_dataset_df"] == [ + {"prompt": "p1", "response": "r1"} + ] + pd.testing.assert_frame_equal( + restored.evaluation_dataset[0].eval_dataset_df, df + ) + assert isinstance( + result.model_dump()["evaluation_dataset"][0]["eval_dataset_df"], + pd.DataFrame, + ) + def test_load_from_adk_eval_set_file(self, tmp_path): path = tmp_path / "home_automation.evalset.json" path.write_text(json.dumps(_ADK_EVAL_SET)) diff --git a/vertexai/_genai/_evals_visualization.py b/vertexai/_genai/_evals_visualization.py index 45b50b6ebc..8e7ae11dd6 100644 --- a/vertexai/_genai/_evals_visualization.py +++ b/vertexai/_genai/_evals_visualization.py @@ -23,7 +23,7 @@ from typing import Any, Optional import pandas as pd -from pydantic import errors +import pydantic_core from . import types @@ -1425,7 +1425,7 @@ def display_evaluation_result( result_dump = eval_result_obj.model_dump( mode="json", exclude_none=True, exclude={"evaluation_dataset"} ) - except errors.PydanticSerializationError as e: + except pydantic_core.PydanticSerializationError as e: logger.error( "Serialization Error: %s\nCould not display the evaluation " "result due to a data serialization issue. Please check the " diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 385087ec8e..28d868c32c 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -35,6 +35,7 @@ ) from google.genai import _common from google.genai import types as genai_types +import pydantic from pydantic import ( ConfigDict, Field, @@ -3463,6 +3464,32 @@ def _check_pandas_installed(cls, data: Any) -> Any: ) return data + @field_validator("eval_dataset_df", mode="before") + @classmethod + def _eval_dataset_df_from_records(cls, value: Any) -> Any: + if pd is not None and isinstance(value, list): + return pd.DataFrame(value) + return value + + @pydantic.field_serializer("eval_dataset_df", when_used="json") + def _eval_dataset_df_to_records(self, value: Any) -> Any: + if pd is None or not isinstance(value, pd.DataFrame): + return value + import numpy as np + + def to_serializable(item: Any) -> Any: + if isinstance(item, dict): + return {key: to_serializable(val) for key, val in item.items()} + if isinstance(item, (list, tuple, np.ndarray, pd.Series)): + return [to_serializable(val) for val in item] + if isinstance(item, np.datetime64): + item = pd.Timestamp(item) + elif isinstance(item, np.generic): + return item.item() + return None if item is pd.NaT or item is pd.NA else item + + return to_serializable(value.to_dict(orient="records")) + @classmethod def load_from_observability_eval_cases( cls, cases: list["ObservabilityEvalCase"] diff --git a/vertexai/_genai/types/evals.py b/vertexai/_genai/types/evals.py index 080e46f5ec..4e90bb05ad 100644 --- a/vertexai/_genai/types/evals.py +++ b/vertexai/_genai/types/evals.py @@ -948,7 +948,7 @@ class RubricVerdict(_common.BaseModel): generated.""", ) verdict: Optional[bool] = Field( - default=None, + default=False, description="""Required. Outcome of the evaluation against the rubric, represented as a boolean. `true` indicates a "Pass", `false` indicates a "Fail".""", )