AgentEvaluator support custom metric

src/google/adk/evaluation/agent_evaluator.py

@@ -59,6 +59,8 @@
 from .evaluator import EvalStatus
 from .in_memory_eval_sets_manager import InMemoryEvalSetsManager
 from .local_eval_sets_manager import convert_eval_set_to_pydantic_schema
+from .metric_evaluator_registry import MetricEvaluatorRegistry
+from .metric_evaluator_registry import register_custom_metrics_from_config
 from .simulation.user_simulator_provider import UserSimulatorProvider
 
 logger = logging.getLogger("google_adk." + __name__)
@@ -197,6 +199,9 @@ async def evaluate_eval_set(
     # under `app_name`. When no manager is given, nothing is saved, so a dummy
     # `app_name` is fine here.
     app_name = app_name or "test_app"
+    metric_evaluator_registry = register_custom_metrics_from_config(
+        eval_config, MetricEvaluatorRegistry()
+    )
 
     # Step 1: Perform evals, basically inferencing and evaluation of metrics
     eval_results_by_eval_id = await AgentEvaluator._get_eval_results_by_eval_id(
@@ -210,6 +215,7 @@ async def evaluate_eval_set(
         artifact_service=artifact_service,
         app=app,
         eval_set_results_manager=eval_set_results_manager,
+        metric_evaluator_registry=metric_evaluator_registry,
     )
 
     # Step 2: Post-process the results!
@@ -679,6 +685,7 @@ async def _get_eval_results_by_eval_id(
       *,
       app_name: str,
       eval_set_results_manager: Optional[EvalSetResultsManager] = None,
+      metric_evaluator_registry: Optional[MetricEvaluatorRegistry] = None,
   ) -> dict[str, list[EvalCaseResult]]:
     """Returns EvalCaseResults grouped by eval case id.
 
@@ -704,6 +711,7 @@ async def _get_eval_results_by_eval_id(
         artifact_service=artifact_service,
         app=app,
         eval_set_results_manager=eval_set_results_manager,
+        metric_evaluator_registry=metric_evaluator_registry,
     )
 
     if live_model_config:

tests/integration/fixture/home_automation_agent/test_files/custom_metrics/metrics.py

@@ -0,0 +1,69 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Optional
+
+from google.adk.evaluation.eval_case import ConversationScenario
+from google.adk.evaluation.eval_case import get_all_tool_calls
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetric
+from google.adk.evaluation.eval_metrics import EvalStatus
+from google.adk.evaluation.evaluator import EvaluationResult
+from google.adk.evaluation.evaluator import PerInvocationResult
+
+
+def tool_trajectory_length_match(
+    eval_metric: EvalMetric,
+    actual_invocations: list[Invocation],
+    expected_invocations: Optional[list[Invocation]] = None,
+    conversation_scenario: Optional[ConversationScenario] = None,
+) -> EvaluationResult:
+  del eval_metric
+  del conversation_scenario
+  expected_invocations = expected_invocations or []
+
+  per_invocation_results = []
+  for idx, actual in enumerate(actual_invocations):
+    expected = (
+        expected_invocations[idx] if idx < len(expected_invocations) else None
+    )
+    actual_tools = get_all_tool_calls(actual.intermediate_data)
+    expected_tools = (
+        get_all_tool_calls(expected.intermediate_data) if expected else []
+    )
+    match = len(actual_tools) == len(expected_tools)
+    per_invocation_results.append(
+        PerInvocationResult(
+            actual_invocation=actual,
+            expected_invocation=expected,
+            score=1.0 if match else 0.0,
+            eval_status=EvalStatus.PASSED if match else EvalStatus.FAILED,
+        )
+    )
+
+  overall_score = (
+      sum(r.score for r in per_invocation_results) / len(per_invocation_results)
+      if per_invocation_results
+      else 0.0
+  )
+  overall_eval_status = (
+      EvalStatus.PASSED if overall_score == 1.0 else EvalStatus.FAILED
+  )
+  return EvaluationResult(
+      overall_score=overall_score,
+      overall_eval_status=overall_eval_status,
+      per_invocation_results=per_invocation_results,
+  )

tests/integration/fixture/home_automation_agent/test_files/custom_metrics/simple_custom_metric.test.json

@@ -0,0 +1,65 @@
+{
+  "eval_set_id": "custom_metrics_eval_set",
+  "name": "custom_metrics_eval_set",
+  "description": "Custom metric evaluation sample.",
+  "eval_cases": [
+    {
+      "eval_id": "tests/integration/fixture/home_automation_agent/test_files/custom_metrics/simple_custom_metric.test.json",
+      "conversation": [
+        {
+          "invocation_id": "a9e4f840-7f1e-4b69-b9c1-3b85c03a60a4",
+          "user_content": {
+            "parts": [
+              {
+                "video_metadata": null,
+                "thought": null,
+                "code_execution_result": null,
+                "executable_code": null,
+                "file_data": null,
+                "function_call": null,
+                "function_response": null,
+                "inline_data": null,
+                "text": "Turn off device_2 in the Bedroom."
+              }
+            ],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [
+              {
+                "video_metadata": null,
+                "thought": null,
+                "code_execution_result": null,
+                "executable_code": null,
+                "file_data": null,
+                "function_call": null,
+                "function_response": null,
+                "inline_data": null,
+                "text": "I have set the device_2 status to off."
+              }
+            ],
+            "role": "model"
+          },
+          "intermediate_data": {
+            "tool_uses": [
+              {
+                "id": null,
+                "args": {
+                  "location": "Bedroom",
+                  "device_id": "device_2",
+                  "status": "OFF"
+                },
+                "name": "set_device_info"
+              }
+            ],
+            "intermediate_responses": []
+          },
+          "creation_timestamp": 1747337309.2360144
+        }
+      ],
+      "session_input": null,
+      "creation_timestamp": 1747337309.2360282
+    }
+  ],
+  "creation_timestamp": 1747337309.2360387
+}

tests/integration/fixture/home_automation_agent/test_files/custom_metrics/test_config.json

@@ -0,0 +1,13 @@
+{
+  "criteria": {
+    "tool_trajectory_length_match": 1.0
+  },
+  "custom_metrics": {
+    "tool_trajectory_length_match": {
+      "code_config": {
+        "name": "tests.integration.fixture.home_automation_agent.test_files.custom_metrics.metrics.tool_trajectory_length_match"
+      },
+      "description": "Checks that actual and expected tool trajectories have the same length."
+    }
+  }
+}

tests/integration/test_with_test_file.py

@@ -26,6 +26,18 @@ async def test_with_single_test_file():
   )
 
 
+@pytest.mark.asyncio
+async def test_with_custom_metric():
+  """Test eval with a custom metric."""
+  await AgentEvaluator.evaluate(
+      agent_module="tests.integration.fixture.home_automation_agent",
+      eval_dataset_file_path_or_dir=(
+          "tests/integration/fixture/home_automation_agent/test_files/custom_metrics/simple_custom_metric.test.json"
+      ),
+      num_runs=1,
+  )
+
+
 @pytest.mark.asyncio
 async def test_with_folder_of_test_files_long_running():
   """Test the agent's basic ability via a folder of session files."""

tests/unittests/evaluation/test_agent_evaluator.py

@@ -724,6 +724,51 @@ def test_migrate_eval_data_to_new_schema_missing_reference_rejected(tmp_path):
     )
 
 
+@pytest.mark.asyncio
+async def test_evaluate_eval_set_registers_custom_metrics(mocker):
+  """Custom metrics in the eval config are available during evaluation."""
+  eval_set = SimpleNamespace(
+      eval_set_id="eval_set_1",
+      eval_cases=[SimpleNamespace(eval_id="case_a")],
+  )
+  eval_config = EvalConfig(
+      custom_metrics={
+          "my_custom_metric": {
+              "code_config": {"name": "math.sqrt"},
+          }
+      }
+  )
+  mocker.patch.object(
+      AgentEvaluator,
+      "_get_agent_for_eval",
+      new=AsyncMock(return_value=(mocker.Mock(), None)),
+  )
+  mocker.patch(
+      "google.adk.evaluation.agent_evaluator.get_eval_metrics_from_config",
+      return_value=[],
+  )
+  get_results_mock = mocker.patch.object(
+      AgentEvaluator,
+      "_get_eval_results_by_eval_id",
+      new=AsyncMock(return_value={}),
+  )
+
+  await AgentEvaluator.evaluate_eval_set(
+      agent_module="my.pkg.search_agent",
+      eval_set=eval_set,
+      eval_config=eval_config,
+      print_detailed_results=False,
+  )
+
+  metric_evaluator_registry = get_results_mock.await_args.kwargs[
+      "metric_evaluator_registry"
+  ]
+  assert "my_custom_metric" in {
+      metric_info.metric_name
+      for metric_info in metric_evaluator_registry.get_registered_metrics()
+  }
+
+
 @pytest.mark.asyncio
 async def test_evaluate_eval_set_forwards_results_manager_and_app_name(mocker):
   """Results manager and resolved app_name are handed to the eval service