diff --git a/src/google/adk/evaluation/agent_evaluator.py b/src/google/adk/evaluation/agent_evaluator.py index 40e1906f4db..796cd190ce3 100644 --- a/src/google/adk/evaluation/agent_evaluator.py +++ b/src/google/adk/evaluation/agent_evaluator.py @@ -59,6 +59,8 @@ from .evaluator import EvalStatus from .in_memory_eval_sets_manager import InMemoryEvalSetsManager from .local_eval_sets_manager import convert_eval_set_to_pydantic_schema +from .metric_evaluator_registry import MetricEvaluatorRegistry +from .metric_evaluator_registry import register_custom_metrics_from_config from .simulation.user_simulator_provider import UserSimulatorProvider logger = logging.getLogger("google_adk." + __name__) @@ -197,6 +199,9 @@ async def evaluate_eval_set( # under `app_name`. When no manager is given, nothing is saved, so a dummy # `app_name` is fine here. app_name = app_name or "test_app" + metric_evaluator_registry = register_custom_metrics_from_config( + eval_config, MetricEvaluatorRegistry() + ) # Step 1: Perform evals, basically inferencing and evaluation of metrics eval_results_by_eval_id = await AgentEvaluator._get_eval_results_by_eval_id( @@ -210,6 +215,7 @@ async def evaluate_eval_set( artifact_service=artifact_service, app=app, eval_set_results_manager=eval_set_results_manager, + metric_evaluator_registry=metric_evaluator_registry, ) # Step 2: Post-process the results! @@ -679,6 +685,7 @@ async def _get_eval_results_by_eval_id( *, app_name: str, eval_set_results_manager: Optional[EvalSetResultsManager] = None, + metric_evaluator_registry: Optional[MetricEvaluatorRegistry] = None, ) -> dict[str, list[EvalCaseResult]]: """Returns EvalCaseResults grouped by eval case id. @@ -704,6 +711,7 @@ async def _get_eval_results_by_eval_id( artifact_service=artifact_service, app=app, eval_set_results_manager=eval_set_results_manager, + metric_evaluator_registry=metric_evaluator_registry, ) if live_model_config: diff --git a/tests/integration/fixture/home_automation_agent/test_files/custom_metrics/metrics.py b/tests/integration/fixture/home_automation_agent/test_files/custom_metrics/metrics.py new file mode 100644 index 00000000000..2ecbf8bd5ec --- /dev/null +++ b/tests/integration/fixture/home_automation_agent/test_files/custom_metrics/metrics.py @@ -0,0 +1,69 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Optional + +from google.adk.evaluation.eval_case import ConversationScenario +from google.adk.evaluation.eval_case import get_all_tool_calls +from google.adk.evaluation.eval_case import Invocation +from google.adk.evaluation.eval_metrics import EvalMetric +from google.adk.evaluation.eval_metrics import EvalStatus +from google.adk.evaluation.evaluator import EvaluationResult +from google.adk.evaluation.evaluator import PerInvocationResult + + +def tool_trajectory_length_match( + eval_metric: EvalMetric, + actual_invocations: list[Invocation], + expected_invocations: Optional[list[Invocation]] = None, + conversation_scenario: Optional[ConversationScenario] = None, +) -> EvaluationResult: + del eval_metric + del conversation_scenario + expected_invocations = expected_invocations or [] + + per_invocation_results = [] + for idx, actual in enumerate(actual_invocations): + expected = ( + expected_invocations[idx] if idx < len(expected_invocations) else None + ) + actual_tools = get_all_tool_calls(actual.intermediate_data) + expected_tools = ( + get_all_tool_calls(expected.intermediate_data) if expected else [] + ) + match = len(actual_tools) == len(expected_tools) + per_invocation_results.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=expected, + score=1.0 if match else 0.0, + eval_status=EvalStatus.PASSED if match else EvalStatus.FAILED, + ) + ) + + overall_score = ( + sum(r.score for r in per_invocation_results) / len(per_invocation_results) + if per_invocation_results + else 0.0 + ) + overall_eval_status = ( + EvalStatus.PASSED if overall_score == 1.0 else EvalStatus.FAILED + ) + return EvaluationResult( + overall_score=overall_score, + overall_eval_status=overall_eval_status, + per_invocation_results=per_invocation_results, + ) diff --git a/tests/integration/fixture/home_automation_agent/test_files/custom_metrics/simple_custom_metric.test.json b/tests/integration/fixture/home_automation_agent/test_files/custom_metrics/simple_custom_metric.test.json new file mode 100644 index 00000000000..42a8d514703 --- /dev/null +++ b/tests/integration/fixture/home_automation_agent/test_files/custom_metrics/simple_custom_metric.test.json @@ -0,0 +1,65 @@ +{ + "eval_set_id": "custom_metrics_eval_set", + "name": "custom_metrics_eval_set", + "description": "Custom metric evaluation sample.", + "eval_cases": [ + { + "eval_id": "tests/integration/fixture/home_automation_agent/test_files/custom_metrics/simple_custom_metric.test.json", + "conversation": [ + { + "invocation_id": "a9e4f840-7f1e-4b69-b9c1-3b85c03a60a4", + "user_content": { + "parts": [ + { + "video_metadata": null, + "thought": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "Turn off device_2 in the Bedroom." + } + ], + "role": "user" + }, + "final_response": { + "parts": [ + { + "video_metadata": null, + "thought": null, + "code_execution_result": null, + "executable_code": null, + "file_data": null, + "function_call": null, + "function_response": null, + "inline_data": null, + "text": "I have set the device_2 status to off." + } + ], + "role": "model" + }, + "intermediate_data": { + "tool_uses": [ + { + "id": null, + "args": { + "location": "Bedroom", + "device_id": "device_2", + "status": "OFF" + }, + "name": "set_device_info" + } + ], + "intermediate_responses": [] + }, + "creation_timestamp": 1747337309.2360144 + } + ], + "session_input": null, + "creation_timestamp": 1747337309.2360282 + } + ], + "creation_timestamp": 1747337309.2360387 +} diff --git a/tests/integration/fixture/home_automation_agent/test_files/custom_metrics/test_config.json b/tests/integration/fixture/home_automation_agent/test_files/custom_metrics/test_config.json new file mode 100644 index 00000000000..405f7910f83 --- /dev/null +++ b/tests/integration/fixture/home_automation_agent/test_files/custom_metrics/test_config.json @@ -0,0 +1,13 @@ +{ + "criteria": { + "tool_trajectory_length_match": 1.0 + }, + "custom_metrics": { + "tool_trajectory_length_match": { + "code_config": { + "name": "tests.integration.fixture.home_automation_agent.test_files.custom_metrics.metrics.tool_trajectory_length_match" + }, + "description": "Checks that actual and expected tool trajectories have the same length." + } + } +} diff --git a/tests/integration/test_with_test_file.py b/tests/integration/test_with_test_file.py index 27426bf5b91..cf6e30d7769 100644 --- a/tests/integration/test_with_test_file.py +++ b/tests/integration/test_with_test_file.py @@ -26,6 +26,18 @@ async def test_with_single_test_file(): ) +@pytest.mark.asyncio +async def test_with_custom_metric(): + """Test eval with a custom metric.""" + await AgentEvaluator.evaluate( + agent_module="tests.integration.fixture.home_automation_agent", + eval_dataset_file_path_or_dir=( + "tests/integration/fixture/home_automation_agent/test_files/custom_metrics/simple_custom_metric.test.json" + ), + num_runs=1, + ) + + @pytest.mark.asyncio async def test_with_folder_of_test_files_long_running(): """Test the agent's basic ability via a folder of session files.""" diff --git a/tests/unittests/evaluation/test_agent_evaluator.py b/tests/unittests/evaluation/test_agent_evaluator.py index b5f4dfec0d7..56bdbbcb7de 100644 --- a/tests/unittests/evaluation/test_agent_evaluator.py +++ b/tests/unittests/evaluation/test_agent_evaluator.py @@ -724,6 +724,51 @@ def test_migrate_eval_data_to_new_schema_missing_reference_rejected(tmp_path): ) +@pytest.mark.asyncio +async def test_evaluate_eval_set_registers_custom_metrics(mocker): + """Custom metrics in the eval config are available during evaluation.""" + eval_set = SimpleNamespace( + eval_set_id="eval_set_1", + eval_cases=[SimpleNamespace(eval_id="case_a")], + ) + eval_config = EvalConfig( + custom_metrics={ + "my_custom_metric": { + "code_config": {"name": "math.sqrt"}, + } + } + ) + mocker.patch.object( + AgentEvaluator, + "_get_agent_for_eval", + new=AsyncMock(return_value=(mocker.Mock(), None)), + ) + mocker.patch( + "google.adk.evaluation.agent_evaluator.get_eval_metrics_from_config", + return_value=[], + ) + get_results_mock = mocker.patch.object( + AgentEvaluator, + "_get_eval_results_by_eval_id", + new=AsyncMock(return_value={}), + ) + + await AgentEvaluator.evaluate_eval_set( + agent_module="my.pkg.search_agent", + eval_set=eval_set, + eval_config=eval_config, + print_detailed_results=False, + ) + + metric_evaluator_registry = get_results_mock.await_args.kwargs[ + "metric_evaluator_registry" + ] + assert "my_custom_metric" in { + metric_info.metric_name + for metric_info in metric_evaluator_registry.get_registered_metrics() + } + + @pytest.mark.asyncio async def test_evaluate_eval_set_forwards_results_manager_and_app_name(mocker): """Results manager and resolved app_name are handed to the eval service