Skip to content

fix: GenAI Client(evals) - Parse JSON-string, OpenAI-style and Message conversation history in evaluate() #7181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: GenAI Client(evals) - Parse JSON-string, OpenAI-style and Message conversation history in evaluate() #7181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 66 additions & 19 deletions agentplatform/_genai/_evals_data_converters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import datetime
import json
import logging
import math
from typing import Any, Optional, TypeVar, Union

from google.genai import _common
Expand Down Expand Up @@ -57,6 +58,20 @@ def _create_placeholder_response_candidate(
)


def _openai_message_to_eval_message(
turn_id: int, message: dict[str, Any]
) -> types.evals.Message:
"""Converts an OpenAI chat message into a conversation history message."""
role = message.get("role", "user")
return types.evals.Message(
turn_id=str(turn_id),
content=genai_types.Content(
parts=[genai_types.Part(text=message.get("content", ""))], role=role
),
author=role,
)


class _GeminiEvalDataConverter(_evals_utils.EvalDataConverter):
"""Converter for dataset in the Gemini format."""

Expand Down Expand Up @@ -199,9 +214,11 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
if not prompt_data:
prompt_data = item.pop("source", None)

conversation_history_data = item.pop("conversation_history", None)
history_column = "conversation_history"
conversation_history_data = item.pop(history_column, None)
if conversation_history_data is None:
conversation_history_data = item.pop("history", None)
history_column = "history"
conversation_history_data = item.pop(history_column, None)
response_data = item.pop("response", None)
reference_data = item.pop("reference", None)
system_instruction_data = item.pop("instruction", None)
Expand Down Expand Up @@ -230,17 +247,43 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
"Invalid prompt type for case %s: %s" % (i, type(prompt_data))
)

if isinstance(conversation_history_data, str):
try:
conversation_history_data = (
json.loads(conversation_history_data)
if conversation_history_data.strip()
else None
)
except json.JSONDecodeError as e:
logger.warning(
"Could not decode JSON string in '%s' column for case %s:"
" %s. Skipping conversation history.",
history_column,
eval_case_id,
e,
)
conversation_history_data = None
conversation_history: Optional[list[types.evals.Message]] = None
if isinstance(conversation_history_data, list):
conversation_history = []
for turn_id, content in enumerate(conversation_history_data):
if isinstance(content, genai_types.Content):
if isinstance(content, types.evals.Message):
conversation_history.append(content)
elif isinstance(content, genai_types.Content):
conversation_history.append(
types.evals.Message(
turn_id=str(turn_id),
content=content,
)
)
elif (
isinstance(content, dict)
and isinstance(content.get("content"), str)
and "parts" not in content
):
conversation_history.append(
_openai_message_to_eval_message(turn_id, content)
)
elif isinstance(content, dict):
try:
validated_content = genai_types.Content.model_validate(
Expand All @@ -254,22 +297,36 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
)
except ValidationError as e:
logger.warning(
"Item at index %s in 'history' column for case "
" %s is a dict but could not be validated as"
"Item at index %s in '%s' column for case %s is a"
" dict but could not be validated as"
" genai_types.Content: %s",
turn_id,
history_column,
eval_case_id,
e,
)
else:
logger.warning(
"Invalid type in 'history' column for case %s at index %s. "
"Expected genai_types.Content or dict, but got %s. "
"Skipping this history item.",
"Invalid type in '%s' column for case %s at index %s."
" Expected genai_types.Content, types.evals.Message or"
" dict, but got %s. Skipping this history item.",
history_column,
eval_case_id,
turn_id,
type(content),
)
elif conversation_history_data is not None and not (
isinstance(conversation_history_data, float)
and math.isnan(conversation_history_data)
):
logger.warning(
"Invalid type in '%s' column for case %s. Expected a list or a"
" JSON string of a list, but got %s. Skipping conversation"
" history.",
history_column,
eval_case_id,
type(conversation_history_data),
)

responses: Optional[list[types.ResponseCandidate]] = None
if isinstance(response_data, dict):
Expand Down Expand Up @@ -492,17 +549,7 @@ def _parse_messages(self, messages: list[dict[str, Any]]) -> tuple[
messages = messages[1:]

for turn_id, msg in enumerate(messages):
role = msg.get("role", "user")
content = msg.get("content", "")
conversation_history.append(
types.evals.Message(
turn_id=str(turn_id),
content=genai_types.Content(
parts=[genai_types.Part(text=content)], role=role
),
author=role,
)
)
conversation_history.append(_openai_message_to_eval_message(turn_id, msg))

if conversation_history:
last_message = conversation_history.pop()
Expand Down
179 changes: 179 additions & 0 deletions tests/unit/agentplatform/genai/test_evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import asyncio
import enum
import importlib
import io
import json
import os
import re
Expand All @@ -44,6 +45,9 @@
types as agentplatform_genai_types,
)
from agentplatform._genai.types import common as common_types
from vertexai._genai import (
_evals_data_converters as vertexai_evals_data_converters,
)
from google.genai import client
from google.genai import errors as genai_errors
from google.genai import types as genai_types
Expand Down Expand Up @@ -5594,6 +5598,25 @@ def test_convert_with_raw_string_response(self):
)


_CONVERTER_MODULES = pytest.mark.parametrize(
"converters",
[_evals_data_converters, vertexai_evals_data_converters],
ids=["agent_platform", "vertexai"],
)
_GEMINI_HISTORY = [
{"role": "user", "parts": [{"text": "My code word is BLUE."}]},
{"role": "model", "parts": [{"text": "Noted."}]},
]
_OPENAI_HISTORY = [
{"role": "user", "content": "My code word is BLUE."},
{"role": "assistant", "content": "Noted."},
]


def _csv_round_trip(row: dict[str, object]) -> pd.DataFrame:
return pd.read_csv(io.StringIO(pd.DataFrame([row]).to_csv(index=False)))


class TestFlattenEvalDataConverter:
"""Unit tests for the _FlattenEvalDataConverter class."""

Expand Down Expand Up @@ -5815,6 +5838,162 @@ def test_convert_with_intermediate_events_as_event_objects(self):
== "intermediate event"
)

@_CONVERTER_MODULES
@pytest.mark.parametrize("column", ["conversation_history", "history"])
def test_convert_json_string_history_from_csv(self, converters, column):
raw_data_df = _csv_round_trip(
{
"prompt": "Code word?",
"response": "BLUE",
column: json.dumps(_GEMINI_HISTORY),
}
)

result_dataset = converters._FlattenEvalDataConverter().convert(
raw_data_df.to_dict(orient="records")
)

assert [
message.content
for message in result_dataset.eval_cases[0].conversation_history
] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY]

@_CONVERTER_MODULES
@pytest.mark.parametrize(
"column,value,expected_warning",
[
(
"history",
"[not json",
"Could not decode JSON string in 'history' column for case eval_case_0",
),
(
"history",
json.dumps(_GEMINI_HISTORY[0]),
"Invalid type in 'history' column for case eval_case_0. Expected",
),
(
"history",
[{"role": "user", "text": "Hi"}],
"Item at index 0 in 'history' column for case eval_case_0 is a dict",
),
(
"conversation_history",
[42],
"'conversation_history' column for case eval_case_0 at index 0.",
),
],
ids=["invalid_json", "json_object", "invalid_dict_item", "invalid_item_type"],
)
def test_convert_invalid_history_logs_warning(
self, converters, column, value, expected_warning, caplog
):
raw_data = [{"prompt": "Hello", "response": "Hi", column: value}]

with caplog.at_level("WARNING", logger=converters.logger.name):
result_dataset = converters._FlattenEvalDataConverter().convert(raw_data)

assert not result_dataset.eval_cases[0].conversation_history
assert expected_warning in caplog.text

@_CONVERTER_MODULES
@pytest.mark.parametrize(
"history",
[_OPENAI_HISTORY, json.dumps(_OPENAI_HISTORY)],
ids=["dicts", "json_string"],
)
def test_convert_openai_style_history(self, converters, history, caplog):
raw_data = [
{
"prompt": "Code word?",
"response": "BLUE",
"conversation_history": history,
}
]

with caplog.at_level("WARNING", logger=converters.logger.name):
result_dataset = converters._FlattenEvalDataConverter().convert(raw_data)

_, openai_history, _, _ = converters._OpenAIDataConverter()._parse_messages(
_OPENAI_HISTORY + [{"role": "user", "content": "Code word?"}]
)
assert result_dataset.eval_cases[0].conversation_history == openai_history
assert [message.content for message in openai_history] == [
genai_types.Content(
parts=[genai_types.Part(text="My code word is BLUE.")], role="user"
),
genai_types.Content(
parts=[genai_types.Part(text="Noted.")], role="assistant"
),
]
assert not caplog.records

@_CONVERTER_MODULES
def test_convert_message_history_items(self, converters):
history = [
converters.types.evals.Message(
turn_id=f"turn-{turn_id}",
content=genai_types.Content.model_validate(turn),
author=turn["role"],
)
for turn_id, turn in enumerate(_GEMINI_HISTORY)
]

result_dataset = converters._FlattenEvalDataConverter().convert(
[{"prompt": "Code word?", "response": "BLUE", "history": history}]
)

assert result_dataset.eval_cases[0].conversation_history == history

@_CONVERTER_MODULES
@pytest.mark.parametrize("value", [None, " "], ids=["nan", "blank"])
def test_convert_empty_csv_history_cell(self, converters, value, caplog):
raw_data_df = _csv_round_trip(
{"prompt": "Hello", "response": "Hi", "conversation_history": value}
)

with caplog.at_level("WARNING", logger=converters.logger.name):
result_dataset = converters._FlattenEvalDataConverter().convert(
raw_data_df.to_dict(orient="records")
)

assert result_dataset.eval_cases[0].conversation_history is None
assert not caplog.records

@pytest.mark.usefixtures("mock_eval_dependencies")
def test_evaluate_sends_csv_history_to_multi_turn_metric(
self, mock_api_client_fixture
):
dataset_df = _csv_round_trip(
{
"prompt": "Code word?",
"response": "BLUE",
"conversation_history": json.dumps(_GEMINI_HISTORY),
}
)

with mock.patch.object(
evals.Evals, "_evaluate_instances"
) as mock_evaluate_instances:
mock_evaluate_instances.return_value = (
agentplatform_genai_types.EvaluateInstancesResponse(
metric_results=[agentplatform_genai_types.MetricResult(score=1.0)]
)
)
evals.Evals(api_client_=mock_api_client_fixture).evaluate(
dataset=dataset_df,
metrics=[
agentplatform_genai_types.Metric(
name="multi_turn_general_quality_v1"
)
],
)

instance = mock_evaluate_instances.call_args.kwargs["instance"]
assert [
content.parts[0].text for content in instance.prompt.contents.contents
] == ["My code word is BLUE.", "Noted.", "Code word?"]


class TestOpenAIDataConverter:
"""Unit tests for the _OpenAIDataConverter class."""
Expand Down
Loading
Loading