diff --git a/agentplatform/_genai/_evals_data_converters.py b/agentplatform/_genai/_evals_data_converters.py index cae94afed3..2934ba92ea 100644 --- a/agentplatform/_genai/_evals_data_converters.py +++ b/agentplatform/_genai/_evals_data_converters.py @@ -15,15 +15,18 @@ """Dataset converters for evals.""" import copy +import datetime import json import logging -from typing import Any, Optional, Union +from typing import Any, Optional, TypeVar, Union from google.genai import _common from google.genai import types as genai_types +from pydantic import alias_generators from pydantic import ValidationError from typing_extensions import override +from . import _evals_constant from . import _evals_utils from . import _observability_data_converter from . import types @@ -605,6 +608,308 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: return types.EvaluationDataset(eval_cases=eval_cases) +_ADK_EVAL_CASE_FIELDS = frozenset( + { + "eval_id", + "conversation", + "conversation_scenario", + "session_input", + "creation_timestamp", + "rubrics", + "final_session_state", + } +) +_ADK_EVAL_CASE_KEYS = _ADK_EVAL_CASE_FIELDS | { + alias_generators.to_camel(field) for field in _ADK_EVAL_CASE_FIELDS +} +_ADK_SESSION_INPUT_FIELDS = ("app_name", "user_id", "state") +_ADK_DEFAULT_RUBRIC_GROUP = "adk_rubrics" +_REFERENCE_TRAJECTORY = "reference_trajectory" + +_GenaiModel = TypeVar("_GenaiModel", bound=_common.BaseModel) + + +def _get_adk_field(data: dict[str, Any], name: str) -> Any: + """Returns an ADK eval set field, which ADK accepts in snake or camel case.""" + if not isinstance(data, dict): + raise TypeError(f"Expected a JSON object for '{name}', got {type(data)}.") + if name in data: + return data[name] + return data.get(alias_generators.to_camel(name)) + + +def _validate_adk_genai(model: type[_GenaiModel], data: Any) -> _GenaiModel: + """Validates ADK eval set JSON as a genai type. + + Validating in JSON mode decodes base64 bytes fields, such as inline data and + thought signatures, the same way ADK does when it reads an eval set file. + + Args: + model: The genai type to validate as. + data: The parsed JSON value from the eval set. + + Returns: + The validated genai object. + """ + return model.model_validate_json(json.dumps(data)) + + +def _adk_content(data: dict[str, Any], default_role: str) -> genai_types.Content: + """Validates an ADK Content, filling in the role when the file omits it.""" + content = _validate_adk_genai(genai_types.Content, data) + if not content.role: + content.role = default_role + return content + + +def _adk_user_content(invocation: dict[str, Any]) -> genai_types.Content: + """Returns the user content of an ADK invocation.""" + user_content = _get_adk_field(invocation, "user_content") + if user_content is None: + raise ValueError("Every invocation must have user_content.") + return _adk_content(user_content, "user") + + +def _adk_intermediate_events( + intermediate_data: Optional[dict[str, Any]], +) -> list[tuple[Optional[str], genai_types.Content]]: + """Returns (author, content) pairs from ADK IntermediateData or InvocationEvents.""" + if not intermediate_data: + return [] + invocation_events = _get_adk_field(intermediate_data, "invocation_events") + if invocation_events is not None: + return [ + ( + _get_adk_field(event, "author"), + _validate_adk_genai( + genai_types.Content, _get_adk_field(event, "content") + ), + ) + for event in invocation_events + if _get_adk_field(event, "content") + ] + + events: list[tuple[Optional[str], genai_types.Content]] = [] + tool_uses = _get_adk_field(intermediate_data, "tool_uses") + if tool_uses: + events.append( + ( + None, + genai_types.Content( + role="model", + parts=[ + genai_types.Part( + function_call=_validate_adk_genai( + genai_types.FunctionCall, tool_use + ) + ) + for tool_use in tool_uses + ], + ), + ) + ) + tool_responses = _get_adk_field(intermediate_data, "tool_responses") + if tool_responses: + events.append( + ( + None, + genai_types.Content( + role="user", + parts=[ + genai_types.Part( + function_response=_validate_adk_genai( + genai_types.FunctionResponse, tool_response + ) + ) + for tool_response in tool_responses + ], + ), + ) + ) + for author, parts in ( + _get_adk_field(intermediate_data, "intermediate_responses") or [] + ): + events.append( + ( + author, + genai_types.Content( + role="model", + parts=[ + _validate_adk_genai(genai_types.Part, part) for part in parts + ], + ), + ) + ) + return events + + +class AdkEvalSetConverter(_evals_utils.EvalDataConverter): + """Converter for ADK eval sets (the `.evalset.json` format). + + Reads the eval set JSON directly, so google-adk does not need to be + installed. + """ + + def _invocation_to_messages( + self, invocation: dict[str, Any], turn_index: int + ) -> list[types.evals.Message]: + """Converts an earlier ADK invocation into conversation history messages.""" + turn_id = _get_adk_field(invocation, "invocation_id") or str(turn_index) + timestamp = _get_adk_field(invocation, "creation_timestamp") + messages = [ + types.evals.Message( + turn_id=turn_id, + author="user", + content=_adk_user_content(invocation), + creation_timestamp=( + datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + if timestamp + else None + ), + ) + ] + messages.extend( + types.evals.Message(turn_id=turn_id, author=author, content=content) + for author, content in _adk_intermediate_events( + _get_adk_field(invocation, "intermediate_data") + ) + ) + final_response = _get_adk_field(invocation, "final_response") + if final_response: + messages.append( + types.evals.Message( + turn_id=turn_id, content=_adk_content(final_response, "model") + ) + ) + return messages + + def _convert_rubrics( + self, rubrics: list[dict[str, Any]] + ) -> dict[str, types.RubricGroup]: + """Groups ADK rubrics by their type, which ADK uses to select rubrics.""" + groups: dict[str, list[types.evals.Rubric]] = {} + for rubric in rubrics: + rubric_type = _get_adk_field(rubric, "type") + rubric_content = _get_adk_field(rubric, "rubric_content") or {} + groups.setdefault(rubric_type or _ADK_DEFAULT_RUBRIC_GROUP, []).append( + types.evals.Rubric( + rubric_id=_get_adk_field(rubric, "rubric_id"), + type=rubric_type, + content=types.evals.RubricContent( + property=types.evals.RubricContentProperty( + description=_get_adk_field(rubric_content, "text_property") + ) + ), + ) + ) + return { + name: types.RubricGroup(rubrics=group) for name, group in groups.items() + } + + def _convert_eval_case(self, case: dict[str, Any]) -> types.EvalCase: + """Converts one ADK EvalCase into an EvalCase.""" + if not isinstance(case, dict): + raise TypeError(f"Expected an ADK EvalCase object, got {type(case)}.") + conversation = _get_adk_field(case, "conversation") + scenario = _get_adk_field(case, "conversation_scenario") + if (conversation is None) == (scenario is None): + raise ValueError( + "Exactly one of conversation and conversation_scenario must be set." + ) + + fields: dict[str, Any] = {"eval_case_id": _get_adk_field(case, "eval_id")} + rubrics = list(_get_adk_field(case, "rubrics") or []) + if scenario is not None: + fields["user_scenario"] = types.evals.UserScenario( + starting_prompt=_get_adk_field(scenario, "starting_prompt"), + conversation_plan=_get_adk_field(scenario, "conversation_plan"), + ) + else: + if not conversation: + raise ValueError("conversation must have at least one invocation.") + *earlier_invocations, last_invocation = conversation + fields["prompt"] = _adk_user_content(last_invocation) + history = [] + for turn_index, invocation in enumerate(earlier_invocations): + history.extend(self._invocation_to_messages(invocation, turn_index)) + if history: + fields["conversation_history"] = history + final_response = _get_adk_field(last_invocation, "final_response") + if final_response: + fields["reference"] = types.ResponseCandidate( + response=_adk_content(final_response, "model") + ) + reference_trajectory: list[genai_types.Content] = [] + for _, content in _adk_intermediate_events( + _get_adk_field(last_invocation, "intermediate_data") + ): + reference_trajectory.extend( + genai_types.Content( + role="model", + parts=[genai_types.Part(function_call=part.function_call)], + ) + for part in content.parts or [] + if part.function_call + ) + if reference_trajectory: + fields[_REFERENCE_TRAJECTORY] = reference_trajectory + rubrics.extend(_get_adk_field(last_invocation, "rubrics") or []) + + if rubrics: + fields["rubric_groups"] = self._convert_rubrics(rubrics) + session_input = _get_adk_field(case, "session_input") + if session_input: + fields[_evals_constant.SESSION_INPUT] = { + name: value + for name in _ADK_SESSION_INPUT_FIELDS + if (value := _get_adk_field(session_input, name)) is not None + } + + reserved_keys = set(fields) + for field_name in types.EvalCase.model_fields: + reserved_keys.update((field_name, alias_generators.to_camel(field_name))) + for key, value in case.items(): + if key in _ADK_EVAL_CASE_KEYS: + continue + if key in reserved_keys: + logger.warning( + "Skipping ADK eval case field '%s' because it conflicts with" + " an EvalCase field.", + key, + ) + continue + fields[key] = value + return types.EvalCase(**fields) + + @override + def convert(self, raw_data: dict[str, Any]) -> types.EvaluationDataset: + """Converts a parsed ADK EvalSet into an EvaluationDataset.""" + if not isinstance(raw_data, dict): + raise ValueError( + "Expected an ADK EvalSet JSON object, got" + f" {type(raw_data)}. For the legacy list format, load the" + " file with google.adk.evaluation.local_eval_sets_manager" + ".load_eval_set_from_file() and pass" + " eval_set.model_dump(mode='json') instead." + ) + adk_eval_cases = _get_adk_field(raw_data, "eval_cases") + if not isinstance(adk_eval_cases, list): + raise ValueError("Expected an ADK EvalSet with an 'eval_cases' list.") + + eval_cases = [] + for i, case in enumerate(adk_eval_cases): + try: + eval_cases.append(self._convert_eval_case(case)) + except (TypeError, ValueError) as e: + case_id = ( + _get_adk_field(case, "eval_id") if isinstance(case, dict) else None + ) + raise ValueError( + f"Failed to convert ADK eval case '{case_id or i}': {e}" + ) from e + return types.EvaluationDataset(eval_cases=eval_cases) + + def auto_detect_dataset_schema( raw_dataset: list[dict[str, Any]], ) -> Union[EvalDatasetSchema, str]: diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 15c2fce06d..d59acc7cfa 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -3182,6 +3182,60 @@ def load_from_observability_eval_cases( return EvaluationDataset(eval_dataset_df=eval_dataset_df) + @classmethod + def load_from_adk_eval_set( + cls, eval_set: Union[str, "os.PathLike[str]", dict[str, Any]] + ) -> "EvaluationDataset": + """Loads an ADK eval set (`.evalset.json`) into an EvaluationDataset. + + Each ADK eval case becomes an EvalCase with `eval_case_id` set to its + `eval_id`. For a static conversation, the last invocation's user content + becomes `prompt` and its final response becomes `reference`. Earlier + invocations, including their tool calls, become `conversation_history`. + A conversation scenario becomes `user_scenario`. Rubrics on the case and + on the last invocation go into `rubric_groups`, keyed by rubric type + ("adk_rubrics" when unset). + + Some ADK data is kept as EvalCase extra fields: `reference_trajectory` + holds the tool calls of the last invocation, `session_inputs` holds the + session input, and custom ADK eval case fields keep their names. Other ADK + fields, such as `final_session_state` and the scenario's `user_persona`, + are not carried over. Steps that convert eval cases to a DataFrame, such + as `run_inference`, drop the extra fields, `rubric_groups` and + `eval_case_id`. google-adk does not need to be installed. + + Example: + import agentplatform + from agentplatform import types + + client = agentplatform.Client( + project="my-project", location="us-central1" + ) + dataset = types.EvaluationDataset.load_from_adk_eval_set( + "my_agent/my_eval_set.evalset.json" + ) + dataset_with_responses = client.evals.run_inference( + src=dataset, agent=my_agent + ) + result = client.evals.evaluate( + dataset=dataset_with_responses, + metrics=[types.RubricMetric.FINAL_RESPONSE_MATCH], + ) + + Args: + eval_set: The path to an ADK `.evalset.json` file, or the eval set as + a parsed JSON dict. + + Returns: + An EvaluationDataset with one EvalCase per ADK eval case. + """ + from .. import _evals_data_converters # pylint: disable=g-import-not-at-top + + if not isinstance(eval_set, dict): + with open(eval_set, "r", encoding="utf-8") as f: + eval_set = json.load(f) + return _evals_data_converters.AdkEvalSetConverter().convert(eval_set) + def show(self) -> None: """Shows the evaluation dataset.""" from .. import _evals_visualization diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 232d5ca1c7..28807dfc67 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -9296,9 +9296,407 @@ def test_predefined_metric_retry_fail_on_resource_exhausted( assert summary_metric.num_cases_error == 1 +_ADK_EVAL_SET = { + "eval_set_id": "home_automation", + "name": "Home automation", + "eval_cases": [ + { + "eval_id": "turn_off_device", + "conversation": [ + { + "invocation_id": "inv-1", + "user_content": { + "parts": [{"text": "Turn off device_2."}], + "role": "user", + }, + "final_response": {"parts": [{"text": "device_2 is now off."}]}, + "intermediate_data": { + "tool_uses": [ + { + "name": "set_device_info", + "args": {"device_id": "device_2", "status": "OFF"}, + } + ], + "intermediate_responses": [], + }, + } + ], + "session_input": { + "app_name": "home_automation_agent", + "user_id": "user", + "state": {"home": {"rooms": 3}}, + }, + "rubrics": [ + { + "rubric_id": "confirms_action", + "rubric_content": { + "text_property": "The response confirms the device is off." + }, + "type": "FINAL_RESPONSE_QUALITY", + } + ], + "tags": ["smoke"], + }, + { + "eval_id": "list_then_turn_off", + "conversation": [ + { + "invocation_id": "inv-2a", + "user_content": { + "parts": [{"text": "Which devices are on?"}], + "role": "user", + }, + "final_response": { + "parts": [{"text": "device_1 is on."}], + "role": "model", + }, + "intermediate_data": { + "invocation_events": [ + { + "author": "home_agent", + "content": { + "parts": [ + { + "function_call": { + "name": "list_devices", + "args": {"status": "ON"}, + } + } + ], + "role": "model", + }, + }, + { + "author": "home_agent", + "content": { + "parts": [ + { + "function_response": { + "name": "list_devices", + "response": {"devices": ["device_1"]}, + } + } + ], + "role": "user", + }, + }, + ] + }, + "creation_timestamp": 1700000000.0, + }, + { + "invocation_id": "inv-2b", + "user_content": { + "parts": [{"text": "Turn it off."}], + "role": "user", + }, + "final_response": { + "parts": [{"text": "device_1 is now off."}], + "role": "model", + }, + "intermediate_data": { + "invocation_events": [ + { + "author": "home_agent", + "content": { + "parts": [ + { + "function_call": { + "name": "set_device_info", + "args": { + "device_id": "device_1", + "status": "OFF", + }, + } + } + ], + "role": "model", + }, + } + ] + }, + "rubrics": [ + { + "rubric_id": "polite", + "rubric_content": { + "text_property": "The response is polite." + }, + } + ], + }, + ], + }, + { + "evalId": "comfortable_bedroom", + "conversationScenario": { + "startingPrompt": "I want my bedroom to be comfortable.", + "conversationPlan": "Ask for 21 degrees once the agent asks.", + "userPersona": "NOVICE", + }, + "sessionInput": {"appName": "home_automation_agent", "userId": "user"}, + }, + ], +} + + class TestEvaluationDataset: """Contains set of tests for the EvaluationDataset class methods.""" + def test_load_from_adk_eval_set_file(self, tmp_path): + path = tmp_path / "home_automation.evalset.json" + path.write_text(json.dumps(_ADK_EVAL_SET)) + + dataset = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + path + ) + + assert [case.eval_case_id for case in dataset.eval_cases] == [ + "turn_off_device", + "list_then_turn_off", + "comfortable_bedroom", + ] + + def test_load_from_adk_eval_set_single_turn_with_tool_uses(self): + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ).eval_cases[0] + + assert case.prompt == genai_types.Content( + parts=[genai_types.Part(text="Turn off device_2.")], role="user" + ) + assert case.reference.response == genai_types.Content( + parts=[genai_types.Part(text="device_2 is now off.")], role="model" + ) + assert case.conversation_history is None + assert case.reference_trajectory == [ + genai_types.Content( + role="model", + parts=[ + genai_types.Part( + function_call=genai_types.FunctionCall( + name="set_device_info", + args={"device_id": "device_2", "status": "OFF"}, + ) + ) + ], + ) + ] + assert case.session_inputs == { + "app_name": "home_automation_agent", + "user_id": "user", + "state": {"home": {"rooms": 3}}, + } + assert case.rubric_groups == { + "FINAL_RESPONSE_QUALITY": agentplatform_genai_types.RubricGroup( + rubrics=[ + agentplatform_genai_types.evals.Rubric( + rubric_id="confirms_action", + type="FINAL_RESPONSE_QUALITY", + content=agentplatform_genai_types.evals.RubricContent( + property=agentplatform_genai_types.evals.RubricContentProperty( + description="The response confirms the device is off." + ) + ), + ) + ] + ) + } + assert case.tags == ["smoke"] + + def test_load_from_adk_eval_set_multi_turn(self): + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ).eval_cases[1] + + assert case.prompt.parts[0].text == "Turn it off." + assert [ + (message.turn_id, message.author, message.content.role) + for message in case.conversation_history + ] == [ + ("inv-2a", "user", "user"), + ("inv-2a", "home_agent", "model"), + ("inv-2a", "home_agent", "user"), + ("inv-2a", None, "model"), + ] + history = case.conversation_history + assert history[0].content.parts[0].text == "Which devices are on?" + assert history[0].creation_timestamp.timestamp() == 1700000000.0 + assert history[1].content.parts[0].function_call.name == "list_devices" + assert history[2].content.parts[0].function_response.response == { + "devices": ["device_1"] + } + assert history[3].content.parts[0].text == "device_1 is on." + assert case.reference.response.parts[0].text == "device_1 is now off." + assert [ + content.parts[0].function_call.name for content in case.reference_trajectory + ] == ["set_device_info"] + assert case.rubric_groups["adk_rubrics"].rubrics[0].rubric_id == "polite" + + def test_load_from_adk_eval_set_conversation_scenario_camel_case(self): + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ).eval_cases[2] + + assert case.eval_case_id == "comfortable_bedroom" + assert case.prompt is None + assert case.user_scenario == agentplatform_genai_types.evals.UserScenario( + starting_prompt="I want my bedroom to be comfortable.", + conversation_plan="Ask for 21 degrees once the agent asks.", + ) + assert case.session_inputs == { + "app_name": "home_automation_agent", + "user_id": "user", + } + + def test_load_from_adk_eval_set_decodes_base64_inline_data(self): + eval_set = { + "eval_set_id": "images", + "eval_cases": [ + { + "eval_id": "image", + "conversation": [ + { + "user_content": { + "parts": [ + { + "inline_data": { + "mime_type": "image/png", + "data": base64.b64encode( + b"\x89PNG" + ).decode(), + } + } + ] + } + } + ], + } + ], + } + + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + eval_set + ).eval_cases[0] + + assert case.prompt.role == "user" + assert case.prompt.parts[0].inline_data.data == b"\x89PNG" + assert case.reference is None + + def test_load_from_adk_eval_set_camel_case_intermediate_data(self, caplog): + eval_set = { + "eval_set_id": "camel", + "eval_cases": [ + { + "evalId": "camel_case", + "conversation": [ + { + "invocationId": "first", + "userContent": {"parts": [{"text": "Look it up."}]}, + "finalResponse": {"parts": [{"text": "It is 42."}]}, + "intermediateData": { + "toolUses": [{"name": "lookup", "args": {"q": "x"}}], + "toolResponses": [ + {"name": "lookup", "response": {"answer": 42}} + ], + "intermediateResponses": [ + ["helper_agent", [{"text": "Looking."}]] + ], + }, + }, + {"userContent": {"parts": [{"text": "Thanks."}]}}, + ], + "evalCaseId": "override", + "userScenario": {"startingPrompt": "Hi"}, + "category": "lookup", + } + ], + } + + with caplog.at_level("WARNING", logger=_evals_data_converters.logger.name): + case = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + eval_set + ).eval_cases[0] + + assert case.eval_case_id == "camel_case" + assert case.user_scenario is None + assert case.category == "lookup" + assert "'evalCaseId'" in caplog.text + assert "'userScenario'" in caplog.text + assert case.prompt.parts[0].text == "Thanks." + assert [ + (message.turn_id, message.author, message.content.role) + for message in case.conversation_history + ] == [ + ("first", "user", "user"), + ("first", None, "model"), + ("first", None, "user"), + ("first", "helper_agent", "model"), + ("first", None, "model"), + ] + history = case.conversation_history + assert history[1].content.parts[0].function_call.args == {"q": "x"} + assert history[2].content.parts[0].function_response.response == {"answer": 42} + assert history[3].content.parts[0].text == "Looking." + assert history[4].content.parts[0].text == "It is 42." + assert case.reference is None + assert "reference_trajectory" not in case.model_extra + + @pytest.mark.parametrize( + "eval_set, error", + [ + ({"eval_set_id": "no_cases"}, "'eval_cases' list"), + ( + {"eval_cases": [{"eval_id": "neither"}]}, + "'neither'.*Exactly one of conversation and conversation_scenario", + ), + ( + {"eval_cases": [{"eval_id": "bad_turn", "conversation": ["Hi"]}]}, + "'bad_turn'.*Expected a JSON object", + ), + ( + { + "eval_cases": [ + { + "eval_id": "both", + "conversation": [], + "conversation_scenario": {"starting_prompt": "Hi"}, + } + ] + }, + "'both'.*Exactly one of conversation and conversation_scenario", + ), + ( + {"eval_cases": [{"eval_id": "empty", "conversation": []}]}, + "at least one invocation", + ), + ( + { + "eval_cases": [ + { + "eval_id": "no_user_content", + "conversation": [ + {"final_response": {"parts": [{"text": "Hi"}]}} + ], + } + ] + }, + "user_content", + ), + ], + ) + def test_load_from_adk_eval_set_invalid(self, eval_set, error): + with pytest.raises(ValueError, match=error): + agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set(eval_set) + + def test_load_from_adk_eval_set_legacy_list_format(self, tmp_path): + path = tmp_path / "legacy.evalset.json" + path.write_text(json.dumps([{"name": "legacy", "data": []}])) + + with pytest.raises(ValueError, match="legacy list format"): + agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + str(path) + ) + @mock.patch.object(_gcs_utils, "GcsUtils") def test_load_from_observability_eval_cases(self, mock_gcs_utils): """Tests that load_from_observability_eval_cases reads data from GCS.""" @@ -10207,6 +10605,96 @@ def test_resolve_dataset_preserves_conversation_history( ptd_values = uploaded_data["prompt"]["promptTemplateData"]["values"] assert "conversation_history" in ptd_values + @pytest.mark.parametrize( + "eval_id, expected_request", + [ + ( + "turn_off_device", + { + "prompt": {"text": "Turn off device_2."}, + "goldenResponse": {"text": "device_2 is now off."}, + }, + ), + ( + "comfortable_bedroom", + { + "prompt": { + "userScenario": { + "startingPrompt": "I want my bedroom to be comfortable.", + "conversationPlan": "Ask for 21 degrees once the agent asks.", + } + } + }, + ), + ], + ) + @mock.patch.object(_evals_common, "evals") + @mock.patch.object(_evals_common, "_gcs_utils") + def test_resolve_dataset_from_adk_eval_set( + self, mock_gcs_utils, mock_evals_module, eval_id, expected_request + ): + mock_gcs_instance = mock_gcs_utils.GcsUtils.return_value + mock_gcs_instance.upload_json_to_prefix.return_value = ( + "gs://bucket/path/request.json" + ) + mock_evals_instance = mock_evals_module.Evals.return_value + mock_evals_instance.create_evaluation_item.return_value.name = "eval_item_1" + mock_evals_instance.create_evaluation_set.return_value.name = "eval_set_1" + loaded = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[ + case for case in loaded.eval_cases if case.eval_case_id == eval_id + ] + ) + + result = _evals_common._resolve_dataset( + api_client=self.mock_api_client, + dataset=dataset, + dest="gs://bucket/prefix", + ) + + assert result.evaluation_set == "eval_set_1" + mock_evals_instance.create_evaluation_set.assert_called_once_with( + evaluation_items=["eval_item_1"] + ) + uploaded_data = mock_gcs_instance.upload_json_to_prefix.call_args.kwargs["data"] + assert uploaded_data == expected_request + + @mock.patch.object(_evals_common, "evals") + @mock.patch.object(_evals_common, "_gcs_utils") + def test_resolve_dataset_from_adk_eval_set_multi_turn( + self, mock_gcs_utils, mock_evals_module + ): + mock_gcs_instance = mock_gcs_utils.GcsUtils.return_value + mock_gcs_instance.upload_json_to_prefix.return_value = ( + "gs://bucket/path/request.json" + ) + mock_evals_instance = mock_evals_module.Evals.return_value + mock_evals_instance.create_evaluation_item.return_value.name = "eval_item_1" + mock_evals_instance.create_evaluation_set.return_value.name = "eval_set_1" + loaded = agentplatform_genai_types.EvaluationDataset.load_from_adk_eval_set( + _ADK_EVAL_SET + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[loaded.eval_cases[1]] + ) + + _evals_common._resolve_dataset( + api_client=self.mock_api_client, + dataset=dataset, + dest="gs://bucket/prefix", + ) + + uploaded_data = mock_gcs_instance.upload_json_to_prefix.call_args.kwargs["data"] + values = uploaded_data["prompt"]["promptTemplateData"]["values"] + assert values["prompt"]["parts"] == [{"text": "Turn it off."}] + history_text = values["conversation_history"]["parts"][0]["text"] + assert history_text.startswith("user: Which devices are on?") + assert history_text.endswith("model: device_1 is on.") + assert uploaded_data["goldenResponse"] == {"text": "device_1 is now off."} + class TestResolveDatasetWithInteractions: """Tests for resolving interactions_data_source in _resolve_dataset.""" diff --git a/vertexai/_genai/_evals_data_converters.py b/vertexai/_genai/_evals_data_converters.py index 16e7779ad9..f293152cbe 100644 --- a/vertexai/_genai/_evals_data_converters.py +++ b/vertexai/_genai/_evals_data_converters.py @@ -15,15 +15,18 @@ """Dataset converters for evals.""" import copy +import datetime import json import logging -from typing import Any, Optional, Union +from typing import Any, Optional, TypeVar, Union from google.genai import _common from google.genai import types as genai_types +from pydantic import alias_generators from pydantic import ValidationError from typing_extensions import override +from . import _evals_constant from . import _evals_utils from . import _observability_data_converter from . import types @@ -605,6 +608,308 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: return types.EvaluationDataset(eval_cases=eval_cases) +_ADK_EVAL_CASE_FIELDS = frozenset( + { + "eval_id", + "conversation", + "conversation_scenario", + "session_input", + "creation_timestamp", + "rubrics", + "final_session_state", + } +) +_ADK_EVAL_CASE_KEYS = _ADK_EVAL_CASE_FIELDS | { + alias_generators.to_camel(field) for field in _ADK_EVAL_CASE_FIELDS +} +_ADK_SESSION_INPUT_FIELDS = ("app_name", "user_id", "state") +_ADK_DEFAULT_RUBRIC_GROUP = "adk_rubrics" +_REFERENCE_TRAJECTORY = "reference_trajectory" + +_GenaiModel = TypeVar("_GenaiModel", bound=_common.BaseModel) + + +def _get_adk_field(data: dict[str, Any], name: str) -> Any: + """Returns an ADK eval set field, which ADK accepts in snake or camel case.""" + if not isinstance(data, dict): + raise TypeError(f"Expected a JSON object for '{name}', got {type(data)}.") + if name in data: + return data[name] + return data.get(alias_generators.to_camel(name)) + + +def _validate_adk_genai(model: type[_GenaiModel], data: Any) -> _GenaiModel: + """Validates ADK eval set JSON as a genai type. + + Validating in JSON mode decodes base64 bytes fields, such as inline data and + thought signatures, the same way ADK does when it reads an eval set file. + + Args: + model: The genai type to validate as. + data: The parsed JSON value from the eval set. + + Returns: + The validated genai object. + """ + return model.model_validate_json(json.dumps(data)) + + +def _adk_content(data: dict[str, Any], default_role: str) -> genai_types.Content: + """Validates an ADK Content, filling in the role when the file omits it.""" + content = _validate_adk_genai(genai_types.Content, data) + if not content.role: + content.role = default_role + return content + + +def _adk_user_content(invocation: dict[str, Any]) -> genai_types.Content: + """Returns the user content of an ADK invocation.""" + user_content = _get_adk_field(invocation, "user_content") + if user_content is None: + raise ValueError("Every invocation must have user_content.") + return _adk_content(user_content, "user") + + +def _adk_intermediate_events( + intermediate_data: Optional[dict[str, Any]], +) -> list[tuple[Optional[str], genai_types.Content]]: + """Returns (author, content) pairs from ADK IntermediateData or InvocationEvents.""" + if not intermediate_data: + return [] + invocation_events = _get_adk_field(intermediate_data, "invocation_events") + if invocation_events is not None: + return [ + ( + _get_adk_field(event, "author"), + _validate_adk_genai( + genai_types.Content, _get_adk_field(event, "content") + ), + ) + for event in invocation_events + if _get_adk_field(event, "content") + ] + + events: list[tuple[Optional[str], genai_types.Content]] = [] + tool_uses = _get_adk_field(intermediate_data, "tool_uses") + if tool_uses: + events.append( + ( + None, + genai_types.Content( + role="model", + parts=[ + genai_types.Part( + function_call=_validate_adk_genai( + genai_types.FunctionCall, tool_use + ) + ) + for tool_use in tool_uses + ], + ), + ) + ) + tool_responses = _get_adk_field(intermediate_data, "tool_responses") + if tool_responses: + events.append( + ( + None, + genai_types.Content( + role="user", + parts=[ + genai_types.Part( + function_response=_validate_adk_genai( + genai_types.FunctionResponse, tool_response + ) + ) + for tool_response in tool_responses + ], + ), + ) + ) + for author, parts in ( + _get_adk_field(intermediate_data, "intermediate_responses") or [] + ): + events.append( + ( + author, + genai_types.Content( + role="model", + parts=[ + _validate_adk_genai(genai_types.Part, part) for part in parts + ], + ), + ) + ) + return events + + +class AdkEvalSetConverter(_evals_utils.EvalDataConverter): + """Converter for ADK eval sets (the `.evalset.json` format). + + Reads the eval set JSON directly, so google-adk does not need to be + installed. + """ + + def _invocation_to_messages( + self, invocation: dict[str, Any], turn_index: int + ) -> list[types.evals.Message]: + """Converts an earlier ADK invocation into conversation history messages.""" + turn_id = _get_adk_field(invocation, "invocation_id") or str(turn_index) + timestamp = _get_adk_field(invocation, "creation_timestamp") + messages = [ + types.evals.Message( + turn_id=turn_id, + author="user", + content=_adk_user_content(invocation), + creation_timestamp=( + datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + if timestamp + else None + ), + ) + ] + messages.extend( + types.evals.Message(turn_id=turn_id, author=author, content=content) + for author, content in _adk_intermediate_events( + _get_adk_field(invocation, "intermediate_data") + ) + ) + final_response = _get_adk_field(invocation, "final_response") + if final_response: + messages.append( + types.evals.Message( + turn_id=turn_id, content=_adk_content(final_response, "model") + ) + ) + return messages + + def _convert_rubrics( + self, rubrics: list[dict[str, Any]] + ) -> dict[str, types.RubricGroup]: + """Groups ADK rubrics by their type, which ADK uses to select rubrics.""" + groups: dict[str, list[types.evals.Rubric]] = {} + for rubric in rubrics: + rubric_type = _get_adk_field(rubric, "type") + rubric_content = _get_adk_field(rubric, "rubric_content") or {} + groups.setdefault(rubric_type or _ADK_DEFAULT_RUBRIC_GROUP, []).append( + types.evals.Rubric( + rubric_id=_get_adk_field(rubric, "rubric_id"), + type=rubric_type, + content=types.evals.RubricContent( + property=types.evals.RubricContentProperty( + description=_get_adk_field(rubric_content, "text_property") + ) + ), + ) + ) + return { + name: types.RubricGroup(rubrics=group) for name, group in groups.items() + } + + def _convert_eval_case(self, case: dict[str, Any]) -> types.EvalCase: + """Converts one ADK EvalCase into an EvalCase.""" + if not isinstance(case, dict): + raise TypeError(f"Expected an ADK EvalCase object, got {type(case)}.") + conversation = _get_adk_field(case, "conversation") + scenario = _get_adk_field(case, "conversation_scenario") + if (conversation is None) == (scenario is None): + raise ValueError( + "Exactly one of conversation and conversation_scenario must be set." + ) + + fields: dict[str, Any] = {"eval_case_id": _get_adk_field(case, "eval_id")} + rubrics = list(_get_adk_field(case, "rubrics") or []) + if scenario is not None: + fields["user_scenario"] = types.evals.UserScenario( + starting_prompt=_get_adk_field(scenario, "starting_prompt"), + conversation_plan=_get_adk_field(scenario, "conversation_plan"), + ) + else: + if not conversation: + raise ValueError("conversation must have at least one invocation.") + *earlier_invocations, last_invocation = conversation + fields["prompt"] = _adk_user_content(last_invocation) + history = [] + for turn_index, invocation in enumerate(earlier_invocations): + history.extend(self._invocation_to_messages(invocation, turn_index)) + if history: + fields["conversation_history"] = history + final_response = _get_adk_field(last_invocation, "final_response") + if final_response: + fields["reference"] = types.ResponseCandidate( + response=_adk_content(final_response, "model") + ) + reference_trajectory: list[genai_types.Content] = [] + for _, content in _adk_intermediate_events( + _get_adk_field(last_invocation, "intermediate_data") + ): + reference_trajectory.extend( + genai_types.Content( + role="model", + parts=[genai_types.Part(function_call=part.function_call)], + ) + for part in content.parts or [] + if part.function_call + ) + if reference_trajectory: + fields[_REFERENCE_TRAJECTORY] = reference_trajectory + rubrics.extend(_get_adk_field(last_invocation, "rubrics") or []) + + if rubrics: + fields["rubric_groups"] = self._convert_rubrics(rubrics) + session_input = _get_adk_field(case, "session_input") + if session_input: + fields[_evals_constant.SESSION_INPUT] = { + name: value + for name in _ADK_SESSION_INPUT_FIELDS + if (value := _get_adk_field(session_input, name)) is not None + } + + reserved_keys = set(fields) + for field_name in types.EvalCase.model_fields: + reserved_keys.update((field_name, alias_generators.to_camel(field_name))) + for key, value in case.items(): + if key in _ADK_EVAL_CASE_KEYS: + continue + if key in reserved_keys: + logger.warning( + "Skipping ADK eval case field '%s' because it conflicts with" + " an EvalCase field.", + key, + ) + continue + fields[key] = value + return types.EvalCase(**fields) + + @override + def convert(self, raw_data: dict[str, Any]) -> types.EvaluationDataset: + """Converts a parsed ADK EvalSet into an EvaluationDataset.""" + if not isinstance(raw_data, dict): + raise ValueError( + "Expected an ADK EvalSet JSON object, got" + f" {type(raw_data)}. For the legacy list format, load the" + " file with google.adk.evaluation.local_eval_sets_manager" + ".load_eval_set_from_file() and pass" + " eval_set.model_dump(mode='json') instead." + ) + adk_eval_cases = _get_adk_field(raw_data, "eval_cases") + if not isinstance(adk_eval_cases, list): + raise ValueError("Expected an ADK EvalSet with an 'eval_cases' list.") + + eval_cases = [] + for i, case in enumerate(adk_eval_cases): + try: + eval_cases.append(self._convert_eval_case(case)) + except (TypeError, ValueError) as e: + case_id = ( + _get_adk_field(case, "eval_id") if isinstance(case, dict) else None + ) + raise ValueError( + f"Failed to convert ADK eval case '{case_id or i}': {e}" + ) from e + return types.EvaluationDataset(eval_cases=eval_cases) + + def auto_detect_dataset_schema( raw_dataset: list[dict[str, Any]], ) -> Union[EvalDatasetSchema, str]: diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 5bb3b8811b..385087ec8e 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -3515,6 +3515,60 @@ def load_from_observability_eval_cases( return EvaluationDataset(eval_dataset_df=eval_dataset_df) + @classmethod + def load_from_adk_eval_set( + cls, eval_set: Union[str, "os.PathLike[str]", dict[str, Any]] + ) -> "EvaluationDataset": + """Loads an ADK eval set (`.evalset.json`) into an EvaluationDataset. + + Each ADK eval case becomes an EvalCase with `eval_case_id` set to its + `eval_id`. For a static conversation, the last invocation's user content + becomes `prompt` and its final response becomes `reference`. Earlier + invocations, including their tool calls, become `conversation_history`. + A conversation scenario becomes `user_scenario`. Rubrics on the case and + on the last invocation go into `rubric_groups`, keyed by rubric type + ("adk_rubrics" when unset). + + Some ADK data is kept as EvalCase extra fields: `reference_trajectory` + holds the tool calls of the last invocation, `session_inputs` holds the + session input, and custom ADK eval case fields keep their names. Other ADK + fields, such as `final_session_state` and the scenario's `user_persona`, + are not carried over. Steps that convert eval cases to a DataFrame, such + as `run_inference`, drop the extra fields, `rubric_groups` and + `eval_case_id`. google-adk does not need to be installed. + + Example: + import vertexai + from vertexai._genai import types + + client = vertexai.Client( + project="my-project", location="us-central1" + ) + dataset = types.EvaluationDataset.load_from_adk_eval_set( + "my_agent/my_eval_set.evalset.json" + ) + dataset_with_responses = client.evals.run_inference( + src=dataset, agent=my_agent + ) + result = client.evals.evaluate( + dataset=dataset_with_responses, + metrics=[types.RubricMetric.FINAL_RESPONSE_MATCH], + ) + + Args: + eval_set: The path to an ADK `.evalset.json` file, or the eval set as + a parsed JSON dict. + + Returns: + An EvaluationDataset with one EvalCase per ADK eval case. + """ + from .. import _evals_data_converters # pylint: disable=g-import-not-at-top + + if not isinstance(eval_set, dict): + with open(eval_set, "r", encoding="utf-8") as f: + eval_set = json.load(f) + return _evals_data_converters.AdkEvalSetConverter().convert(eval_set) + def show(self) -> None: """Shows the evaluation dataset.""" from .. import _evals_visualization