How to Evaluate Live & Voice Agents in ADK

from google.adk.agents.llm_agent import Agent
from google.adk.tools.tool_context import ToolContext
from google.adk.workflow import START, Workflow
from pydantic import BaseModel, Field

LIVE_MODEL = "gemini-live-2.5-flash-native-audio"

def validate_date_of_birth(dob: str, tool_context: ToolContext) -> dict:
  """验证确认的出生日期与记录（模拟）."""
  match = dob == "1985-07-12"
  tool_context.state["dob_verified"] = match
  return {"match": match}

greeter_agent = Agent(
    model=LIVE_MODEL,
    name="greeter_agent",
    mode="task",
    instruction="你叫 Sam，是友好的护理团队助理。向来电者问候，确认你正在与 John Doe 通话，然后再分享其他信息。每回合只问一个问题，然后在确认姓名后完成任务。",
)

dob_verifier_agent = Agent(
    model=LIVE_MODEL,
    name="dob_verifier_agent",
    mode="task",
    tools=[validate_date_of_birth],
    instruction="询问来电者的出生日期，读回以确认，然后以 YYYY-MM-DD 格式调用 validate_date_of_birth。任务完成后返回 'verified' 或 'unverified'。",
)

goals_agent = Agent(
    model=LIVE_MODEL,
    name="goals_agent",
    mode="task",
    instruction="身份已验证。主动告知下周二 6 月 16 日下午 3 点与 Example 博士的预约，回答任何问题，然后温暖结束对话并以 'Goodbye.' 结束。",
)

root_agent = Workflow(
    name="live_workflow",
    edges=[
        (START, greeter_agent),
        (greeter_agent, dob_verifier_agent),
        (dob_verifier_agent, goals_agent),
    ],
)

{
  "eval_id": "example_scenario_case",
  "conversation_scenario": {
    "starting_prompt": "Hello?",
    "conversation_plan": "你叫约翰·多伊。当被问候时确认你的名字。当被询问出生日期时，给出 1985 年 7 月 12 日，并在读取回传时确认。聆听预约详情，询问需要带什么去就诊，然后表示没有其他问题并让通话结束。",
    "user_persona": "NOVICE"
  },
  "session_input": {
    "app_name": "live_workflow",
    "user_id": "test_user_id",
    "state": {}
  }
}

{
  "eval_id": "example_fixed_case",
  "conversation": [
    {
      "user_content": {
        "role": "user",
        "parts": [{ "text": "你好，是的，我是约翰·多伊。" }]
      }
    },
    {
      "user_content": {
        "role": "user",
        "parts": [{ "text": "我的出生日期是 1985 年 7 月 12 日。" }]
      }
    }
  ]
}

{
  "criteria": {
    "rubric_based_multi_turn_trajectory_quality_v1": {
      "threshold": 0.7,
      "judge_model_options": { "judge_model": "gemini-3.7-flash" },
      "rubrics": [
        {
          "rubric_id": "verifies_identity_first",
          "rubric_content": {
            "text_property": "在整个对话过程中，代理首先确认来电者姓名并验证其出生日期，之后才透露任何预约详情。"
          }
        }
        // ...其他端到端评分标准
      ]
    }
  },
  "live_model_config": {
    "timeout_seconds": 300
  },
  "user_simulator_config": {
    "type": "llm_audio",
    "model": "gemini-3.7-flash",
    "max_allowed_invocations": 10,
    "audio_model": "gemini-3.1-flash-tts-preview",
    "audio_model_configuration": {
      "response_modalities": ["AUDIO"],
      "speech_config": {
        "voice_config": {
          "prebuilt_voice_config": { "voice_name": "Kore" }
        },
        "language_code": "en-US"
      }
    }
  }
}

uv run adk eval \
  contributing/samples/live/live_workflow \
  contributing/samples/live/live_workflow/live_workflow.evalset.json \
  --config_file_path contributing/samples/live/live_workflow/test_config.json