From 0ae1bb1920d713014e839fa7abbd3353c6d8dae0 Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Fri, 2 Oct 2026 12:10:36 -0700 Subject: [PATCH] fix: GenAI Client(evals) - Parse JSON-string, OpenAI-style and Message conversation history in evaluate() PiperOrigin-RevId: 992464091 --- .../_genai/_evals_data_converters.py | 85 +++++++-- tests/unit/agentplatform/genai/test_evals.py | 179 ++++++++++++++++++ vertexai/_genai/_evals_data_converters.py | 85 +++++++-- 3 files changed, 311 insertions(+), 38 deletions(-) diff --git a/agentplatform/_genai/_evals_data_converters.py b/agentplatform/_genai/_evals_data_converters.py index 2934ba92ea..c3d2c0f00c 100644 --- a/agentplatform/_genai/_evals_data_converters.py +++ b/agentplatform/_genai/_evals_data_converters.py @@ -18,6 +18,7 @@ import datetime import json import logging +import math from typing import Any, Optional, TypeVar, Union from google.genai import _common @@ -57,6 +58,20 @@ def _create_placeholder_response_candidate( ) +def _openai_message_to_eval_message( + turn_id: int, message: dict[str, Any] +) -> types.evals.Message: + """Converts an OpenAI chat message into a conversation history message.""" + role = message.get("role", "user") + return types.evals.Message( + turn_id=str(turn_id), + content=genai_types.Content( + parts=[genai_types.Part(text=message.get("content", ""))], role=role + ), + author=role, + ) + + class _GeminiEvalDataConverter(_evals_utils.EvalDataConverter): """Converter for dataset in the Gemini format.""" @@ -199,9 +214,11 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: if not prompt_data: prompt_data = item.pop("source", None) - conversation_history_data = item.pop("conversation_history", None) + history_column = "conversation_history" + conversation_history_data = item.pop(history_column, None) if conversation_history_data is None: - conversation_history_data = item.pop("history", None) + history_column = "history" + conversation_history_data = item.pop(history_column, None) response_data = item.pop("response", None) reference_data = item.pop("reference", None) system_instruction_data = item.pop("instruction", None) @@ -230,17 +247,43 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: "Invalid prompt type for case %s: %s" % (i, type(prompt_data)) ) + if isinstance(conversation_history_data, str): + try: + conversation_history_data = ( + json.loads(conversation_history_data) + if conversation_history_data.strip() + else None + ) + except json.JSONDecodeError as e: + logger.warning( + "Could not decode JSON string in '%s' column for case %s:" + " %s. Skipping conversation history.", + history_column, + eval_case_id, + e, + ) + conversation_history_data = None conversation_history: Optional[list[types.evals.Message]] = None if isinstance(conversation_history_data, list): conversation_history = [] for turn_id, content in enumerate(conversation_history_data): - if isinstance(content, genai_types.Content): + if isinstance(content, types.evals.Message): + conversation_history.append(content) + elif isinstance(content, genai_types.Content): conversation_history.append( types.evals.Message( turn_id=str(turn_id), content=content, ) ) + elif ( + isinstance(content, dict) + and isinstance(content.get("content"), str) + and "parts" not in content + ): + conversation_history.append( + _openai_message_to_eval_message(turn_id, content) + ) elif isinstance(content, dict): try: validated_content = genai_types.Content.model_validate( @@ -254,22 +297,36 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: ) except ValidationError as e: logger.warning( - "Item at index %s in 'history' column for case " - " %s is a dict but could not be validated as" + "Item at index %s in '%s' column for case %s is a" + " dict but could not be validated as" " genai_types.Content: %s", turn_id, + history_column, eval_case_id, e, ) else: logger.warning( - "Invalid type in 'history' column for case %s at index %s. " - "Expected genai_types.Content or dict, but got %s. " - "Skipping this history item.", + "Invalid type in '%s' column for case %s at index %s." + " Expected genai_types.Content, types.evals.Message or" + " dict, but got %s. Skipping this history item.", + history_column, eval_case_id, turn_id, type(content), ) + elif conversation_history_data is not None and not ( + isinstance(conversation_history_data, float) + and math.isnan(conversation_history_data) + ): + logger.warning( + "Invalid type in '%s' column for case %s. Expected a list or a" + " JSON string of a list, but got %s. Skipping conversation" + " history.", + history_column, + eval_case_id, + type(conversation_history_data), + ) responses: Optional[list[types.ResponseCandidate]] = None if isinstance(response_data, dict): @@ -492,17 +549,7 @@ def _parse_messages(self, messages: list[dict[str, Any]]) -> tuple[ messages = messages[1:] for turn_id, msg in enumerate(messages): - role = msg.get("role", "user") - content = msg.get("content", "") - conversation_history.append( - types.evals.Message( - turn_id=str(turn_id), - content=genai_types.Content( - parts=[genai_types.Part(text=content)], role=role - ), - author=role, - ) - ) + conversation_history.append(_openai_message_to_eval_message(turn_id, msg)) if conversation_history: last_message = conversation_history.pop() diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 28807dfc67..f31c69ea3c 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -18,6 +18,7 @@ import asyncio import enum import importlib +import io import json import os import re @@ -44,6 +45,9 @@ types as agentplatform_genai_types, ) from agentplatform._genai.types import common as common_types +from vertexai._genai import ( + _evals_data_converters as vertexai_evals_data_converters, +) from google.genai import client from google.genai import errors as genai_errors from google.genai import types as genai_types @@ -5594,6 +5598,25 @@ def test_convert_with_raw_string_response(self): ) +_CONVERTER_MODULES = pytest.mark.parametrize( + "converters", + [_evals_data_converters, vertexai_evals_data_converters], + ids=["agent_platform", "vertexai"], +) +_GEMINI_HISTORY = [ + {"role": "user", "parts": [{"text": "My code word is BLUE."}]}, + {"role": "model", "parts": [{"text": "Noted."}]}, +] +_OPENAI_HISTORY = [ + {"role": "user", "content": "My code word is BLUE."}, + {"role": "assistant", "content": "Noted."}, +] + + +def _csv_round_trip(row: dict[str, object]) -> pd.DataFrame: + return pd.read_csv(io.StringIO(pd.DataFrame([row]).to_csv(index=False))) + + class TestFlattenEvalDataConverter: """Unit tests for the _FlattenEvalDataConverter class.""" @@ -5815,6 +5838,162 @@ def test_convert_with_intermediate_events_as_event_objects(self): == "intermediate event" ) + @_CONVERTER_MODULES + @pytest.mark.parametrize("column", ["conversation_history", "history"]) + def test_convert_json_string_history_from_csv(self, converters, column): + raw_data_df = _csv_round_trip( + { + "prompt": "Code word?", + "response": "BLUE", + column: json.dumps(_GEMINI_HISTORY), + } + ) + + result_dataset = converters._FlattenEvalDataConverter().convert( + raw_data_df.to_dict(orient="records") + ) + + assert [ + message.content + for message in result_dataset.eval_cases[0].conversation_history + ] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY] + + @_CONVERTER_MODULES + @pytest.mark.parametrize( + "column,value,expected_warning", + [ + ( + "history", + "[not json", + "Could not decode JSON string in 'history' column for case eval_case_0", + ), + ( + "history", + json.dumps(_GEMINI_HISTORY[0]), + "Invalid type in 'history' column for case eval_case_0. Expected", + ), + ( + "history", + [{"role": "user", "text": "Hi"}], + "Item at index 0 in 'history' column for case eval_case_0 is a dict", + ), + ( + "conversation_history", + [42], + "'conversation_history' column for case eval_case_0 at index 0.", + ), + ], + ids=["invalid_json", "json_object", "invalid_dict_item", "invalid_item_type"], + ) + def test_convert_invalid_history_logs_warning( + self, converters, column, value, expected_warning, caplog + ): + raw_data = [{"prompt": "Hello", "response": "Hi", column: value}] + + with caplog.at_level("WARNING", logger=converters.logger.name): + result_dataset = converters._FlattenEvalDataConverter().convert(raw_data) + + assert not result_dataset.eval_cases[0].conversation_history + assert expected_warning in caplog.text + + @_CONVERTER_MODULES + @pytest.mark.parametrize( + "history", + [_OPENAI_HISTORY, json.dumps(_OPENAI_HISTORY)], + ids=["dicts", "json_string"], + ) + def test_convert_openai_style_history(self, converters, history, caplog): + raw_data = [ + { + "prompt": "Code word?", + "response": "BLUE", + "conversation_history": history, + } + ] + + with caplog.at_level("WARNING", logger=converters.logger.name): + result_dataset = converters._FlattenEvalDataConverter().convert(raw_data) + + _, openai_history, _, _ = converters._OpenAIDataConverter()._parse_messages( + _OPENAI_HISTORY + [{"role": "user", "content": "Code word?"}] + ) + assert result_dataset.eval_cases[0].conversation_history == openai_history + assert [message.content for message in openai_history] == [ + genai_types.Content( + parts=[genai_types.Part(text="My code word is BLUE.")], role="user" + ), + genai_types.Content( + parts=[genai_types.Part(text="Noted.")], role="assistant" + ), + ] + assert not caplog.records + + @_CONVERTER_MODULES + def test_convert_message_history_items(self, converters): + history = [ + converters.types.evals.Message( + turn_id=f"turn-{turn_id}", + content=genai_types.Content.model_validate(turn), + author=turn["role"], + ) + for turn_id, turn in enumerate(_GEMINI_HISTORY) + ] + + result_dataset = converters._FlattenEvalDataConverter().convert( + [{"prompt": "Code word?", "response": "BLUE", "history": history}] + ) + + assert result_dataset.eval_cases[0].conversation_history == history + + @_CONVERTER_MODULES + @pytest.mark.parametrize("value", [None, " "], ids=["nan", "blank"]) + def test_convert_empty_csv_history_cell(self, converters, value, caplog): + raw_data_df = _csv_round_trip( + {"prompt": "Hello", "response": "Hi", "conversation_history": value} + ) + + with caplog.at_level("WARNING", logger=converters.logger.name): + result_dataset = converters._FlattenEvalDataConverter().convert( + raw_data_df.to_dict(orient="records") + ) + + assert result_dataset.eval_cases[0].conversation_history is None + assert not caplog.records + + @pytest.mark.usefixtures("mock_eval_dependencies") + def test_evaluate_sends_csv_history_to_multi_turn_metric( + self, mock_api_client_fixture + ): + dataset_df = _csv_round_trip( + { + "prompt": "Code word?", + "response": "BLUE", + "conversation_history": json.dumps(_GEMINI_HISTORY), + } + ) + + with mock.patch.object( + evals.Evals, "_evaluate_instances" + ) as mock_evaluate_instances: + mock_evaluate_instances.return_value = ( + agentplatform_genai_types.EvaluateInstancesResponse( + metric_results=[agentplatform_genai_types.MetricResult(score=1.0)] + ) + ) + evals.Evals(api_client_=mock_api_client_fixture).evaluate( + dataset=dataset_df, + metrics=[ + agentplatform_genai_types.Metric( + name="multi_turn_general_quality_v1" + ) + ], + ) + + instance = mock_evaluate_instances.call_args.kwargs["instance"] + assert [ + content.parts[0].text for content in instance.prompt.contents.contents + ] == ["My code word is BLUE.", "Noted.", "Code word?"] + class TestOpenAIDataConverter: """Unit tests for the _OpenAIDataConverter class.""" diff --git a/vertexai/_genai/_evals_data_converters.py b/vertexai/_genai/_evals_data_converters.py index f293152cbe..6b66a300ea 100644 --- a/vertexai/_genai/_evals_data_converters.py +++ b/vertexai/_genai/_evals_data_converters.py @@ -18,6 +18,7 @@ import datetime import json import logging +import math from typing import Any, Optional, TypeVar, Union from google.genai import _common @@ -57,6 +58,20 @@ def _create_placeholder_response_candidate( ) +def _openai_message_to_eval_message( + turn_id: int, message: dict[str, Any] +) -> types.evals.Message: + """Converts an OpenAI chat message into a conversation history message.""" + role = message.get("role", "user") + return types.evals.Message( + turn_id=str(turn_id), + content=genai_types.Content( + parts=[genai_types.Part(text=message.get("content", ""))], role=role + ), + author=role, + ) + + class _GeminiEvalDataConverter(_evals_utils.EvalDataConverter): """Converter for dataset in the Gemini format.""" @@ -199,9 +214,11 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: if not prompt_data: prompt_data = item.pop("source", None) - conversation_history_data = item.pop("conversation_history", None) + history_column = "conversation_history" + conversation_history_data = item.pop(history_column, None) if conversation_history_data is None: - conversation_history_data = item.pop("history", None) + history_column = "history" + conversation_history_data = item.pop(history_column, None) response_data = item.pop("response", None) reference_data = item.pop("reference", None) system_instruction_data = item.pop("instruction", None) @@ -230,17 +247,43 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: "Invalid prompt type for case %s: %s" % (i, type(prompt_data)) ) + if isinstance(conversation_history_data, str): + try: + conversation_history_data = ( + json.loads(conversation_history_data) + if conversation_history_data.strip() + else None + ) + except json.JSONDecodeError as e: + logger.warning( + "Could not decode JSON string in '%s' column for case %s:" + " %s. Skipping conversation history.", + history_column, + eval_case_id, + e, + ) + conversation_history_data = None conversation_history: Optional[list[types.evals.Message]] = None if isinstance(conversation_history_data, list): conversation_history = [] for turn_id, content in enumerate(conversation_history_data): - if isinstance(content, genai_types.Content): + if isinstance(content, types.evals.Message): + conversation_history.append(content) + elif isinstance(content, genai_types.Content): conversation_history.append( types.evals.Message( turn_id=str(turn_id), content=content, ) ) + elif ( + isinstance(content, dict) + and isinstance(content.get("content"), str) + and "parts" not in content + ): + conversation_history.append( + _openai_message_to_eval_message(turn_id, content) + ) elif isinstance(content, dict): try: validated_content = genai_types.Content.model_validate( @@ -254,22 +297,36 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: ) except ValidationError as e: logger.warning( - "Item at index %s in 'history' column for case " - " %s is a dict but could not be validated as" + "Item at index %s in '%s' column for case %s is a" + " dict but could not be validated as" " genai_types.Content: %s", turn_id, + history_column, eval_case_id, e, ) else: logger.warning( - "Invalid type in 'history' column for case %s at index %s. " - "Expected genai_types.Content or dict, but got %s. " - "Skipping this history item.", + "Invalid type in '%s' column for case %s at index %s." + " Expected genai_types.Content, types.evals.Message or" + " dict, but got %s. Skipping this history item.", + history_column, eval_case_id, turn_id, type(content), ) + elif conversation_history_data is not None and not ( + isinstance(conversation_history_data, float) + and math.isnan(conversation_history_data) + ): + logger.warning( + "Invalid type in '%s' column for case %s. Expected a list or a" + " JSON string of a list, but got %s. Skipping conversation" + " history.", + history_column, + eval_case_id, + type(conversation_history_data), + ) responses: Optional[list[types.ResponseCandidate]] = None if isinstance(response_data, dict): @@ -492,17 +549,7 @@ def _parse_messages(self, messages: list[dict[str, Any]]) -> tuple[ messages = messages[1:] for turn_id, msg in enumerate(messages): - role = msg.get("role", "user") - content = msg.get("content", "") - conversation_history.append( - types.evals.Message( - turn_id=str(turn_id), - content=genai_types.Content( - parts=[genai_types.Part(text=content)], role=role - ), - author=role, - ) - ) + conversation_history.append(_openai_message_to_eval_message(turn_id, msg)) if conversation_history: last_message = conversation_history.pop()