From 7b490ffd705aa9019dc751fa88833ff39fda8151 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:09:15 -0700 Subject: [PATCH 1/2] Adding t^3-bench for testing --- README.md | 6 + cli/README.md | 1 + cli/src/builtins/mod.rs | 3 + cli/src/builtins/tau3/README.md | 40 +++ cli/src/builtins/tau3/environment.rs | 124 +++++++++ cli/src/builtins/tau3/mod.rs | 377 ++++++++++++++++++++++++++ cli/src/builtins/tau3/model.rs | 179 ++++++++++++ cli/src/builtins/tau3/orchestrator.rs | 165 +++++++++++ cli/src/builtins/tau3/scoring.rs | 103 +++++++ 9 files changed, 998 insertions(+) create mode 100644 cli/src/builtins/tau3/README.md create mode 100644 cli/src/builtins/tau3/environment.rs create mode 100644 cli/src/builtins/tau3/mod.rs create mode 100644 cli/src/builtins/tau3/model.rs create mode 100644 cli/src/builtins/tau3/orchestrator.rs create mode 100644 cli/src/builtins/tau3/scoring.rs diff --git a/README.md b/README.md index 36ead5c..062616b 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,12 @@ qt run simpleqa-verified > The command above runs [`simpleqa-verified`](https://quantiles.io/benchmark-hub/benchmark/simpleqa-verified) with a demo model that generates random text. It validates the evaluation workflow without requiring provider API keys or incurring inference costs. Do not use its results to draw conclusions about model quality. +The CLI also includes `tau3-mock`, a small native conformance benchmark for +structured tool calling, model-driven user simulation, mutable task +environments, trajectory recording, component rewards, and `pass_at_k`. It is +not an official τ³ leaderboard domain; see the +[`tau3-mock` implementation notes](cli/src/builtins/tau3/README.md). + Inspect the recorded run: ```bash diff --git a/cli/README.md b/cli/README.md index dcfbe36..e5bfbf2 100644 --- a/cli/README.md +++ b/cli/README.md @@ -37,6 +37,7 @@ The CLI supports three evaluation types: - [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) run predefined datasets and scoring methods. They work without configuration, but you can override settings such as the model, sample count, and concurrency. - [`custom_nocode` evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) define the dataset, prompt template, model, and scoring method entirely in configuration. Supported scoring styles include exact match, multiple choice, and text similarity. +- `tau3-mock` is a native structured-tool-calling conformance benchmark for the in-process τ³-style agent harness. It records tool trajectories, isolated environment state, component rewards, and `pass_at_k`; it is not an official τ³ leaderboard domain. See [`src/builtins/tau3/README.md`](src/builtins/tau3/README.md). - [`custom_code` evaluations](https://quantiles.io/documentation/custom-evaluations) run your own Python evaluation through the Quantiles Python SDK. Add a `quantiles.toml` or `.quantiles.toml` file to configure an evaluation. For example: diff --git a/cli/src/builtins/mod.rs b/cli/src/builtins/mod.rs index 978c581..d66f5b0 100644 --- a/cli/src/builtins/mod.rs +++ b/cli/src/builtins/mod.rs @@ -7,6 +7,7 @@ mod output; mod pubmedqa; mod similarity; mod simpleqa_verified; +mod tau3; pub use custom_nocode::CustomNoCodeBuiltin; pub use custom_nocode::metrics::{ @@ -49,6 +50,8 @@ pub fn resolve(name: &str) -> Option> { Some(Box::new(pubmedqa::PubmedqaBuiltin)) } else if name == simpleqa_verified::SimpleqaVerifiedBuiltin.name() { Some(Box::new(simpleqa_verified::SimpleqaVerifiedBuiltin)) + } else if name == tau3::Tau3MockBuiltin.name() { + Some(Box::new(tau3::Tau3MockBuiltin)) } else { None } diff --git a/cli/src/builtins/tau3/README.md b/cli/src/builtins/tau3/README.md new file mode 100644 index 0000000..30b569a --- /dev/null +++ b/cli/src/builtins/tau3/README.md @@ -0,0 +1,40 @@ +# Native τ³ harness + +`tau3-mock` is a built-in conformance benchmark for Quantiles' native τ³-style +agent harness. It exercises the parts that differ from prompt/response +benchmarks: + +- provider-neutral structured tool calling; +- a model-driven user simulator; +- a turn-based agent/user/tool orchestrator; +- an isolated mutable environment for every task trial; +- durable trajectories and component rewards; and +- aggregate `pass_at_k` metrics using the unbiased estimator. + +It intentionally uses a small bundled mock domain. It is not a substitute for +the official τ³-bench 1.0.1 airline, retail, telecom, banking-knowledge, or voice +tracks, and its scores must not be submitted to or compared with the official +leaderboard. + +Run it with separate agent and user models: + +```console +qt run tau3-mock --input '{ + "model": "openai:gpt-5.2", + "user_model": "openai:gpt-5.2", + "trials": 4, + "max_turns": 20 +}' +``` + +The built-in supports OpenAI, Anthropic, and Gemini model configurations because +those existing Quantiles backends expose structured tool calls through `genai`. +Remote model calls occur only when the user runs the benchmark with one of those +models. Quantiles continues to store runs, steps, trajectories, and metrics +locally. + +Official domain parity requires a separately reviewable port of the versioned +task data, domain policies, databases, exact tool behavior, banking retrieval +corpus and graders, and full-duplex voice orchestration. Until that work lands, +only the explicit `tau3-mock` name resolves; `tau3` and the official domain names +do not. diff --git a/cli/src/builtins/tau3/environment.rs b/cli/src/builtins/tau3/environment.rs new file mode 100644 index 0000000..5bf10e3 --- /dev/null +++ b/cli/src/builtins/tau3/environment.rs @@ -0,0 +1,124 @@ +use anyhow::{Result, bail}; +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use super::model::{ToolDefinition, ToolInvocation}; + +#[async_trait] +pub(crate) trait ToolEnvironment: Send { + fn tools(&self) -> Vec; + async fn invoke(&mut self, call: &ToolInvocation) -> Result; + fn state(&self) -> Value; + fn terminated(&self) -> bool; +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub(crate) struct MockState { + pub(crate) customer_id: String, + pub(crate) name: String, + pub(crate) email: String, +} + +pub(crate) struct MockEnvironment { + state: MockState, + terminated: bool, +} + +impl MockEnvironment { + pub(crate) fn new(state: MockState) -> Self { + Self { + state, + terminated: false, + } + } +} + +#[async_trait] +impl ToolEnvironment for MockEnvironment { + fn tools(&self) -> Vec { + vec![ + ToolDefinition { + name: "get_customer".to_owned(), + description: "Look up a customer record by customer ID.".to_owned(), + schema: json!({ + "type": "object", + "properties": {"customer_id": {"type": "string"}}, + "required": ["customer_id"], + "additionalProperties": false + }), + }, + ToolDefinition { + name: "update_customer_email".to_owned(), + description: "Update the email address on a customer record.".to_owned(), + schema: json!({ + "type": "object", + "properties": { + "customer_id": {"type": "string"}, + "email": {"type": "string"} + }, + "required": ["customer_id", "email"], + "additionalProperties": false + }), + }, + ToolDefinition { + name: "transfer_to_human".to_owned(), + description: "End the interaction and transfer the customer to a human agent." + .to_owned(), + schema: json!({ + "type": "object", + "properties": {"reason": {"type": "string"}}, + "required": ["reason"], + "additionalProperties": false + }), + }, + ] + } + + async fn invoke(&mut self, call: &ToolInvocation) -> Result { + match call.name.as_str() { + "get_customer" => { + require_customer_id(&call.arguments, &self.state.customer_id)?; + Ok(serde_json::to_value(&self.state)?) + } + "update_customer_email" => { + require_customer_id(&call.arguments, &self.state.customer_id)?; + let email = required_string(&call.arguments, "email")?; + if !email.contains('@') { + bail!("email must contain @"); + } + email.clone_into(&mut self.state.email); + Ok(json!({"status": "success", "customer": self.state})) + } + "transfer_to_human" => { + let _ = required_string(&call.arguments, "reason")?; + self.terminated = true; + Ok(json!({"status": "transferred"})) + } + other => bail!("unknown tool `{other}`"), + } + } + + fn state(&self) -> Value { + serde_json::to_value(&self.state).expect("MockState is serializable") + } + + fn terminated(&self) -> bool { + self.terminated + } +} + +fn require_customer_id(arguments: &Value, expected: &str) -> Result<()> { + let actual = required_string(arguments, "customer_id")?; + if actual != expected { + bail!("customer `{actual}` not found"); + } + Ok(()) +} + +fn required_string<'a>(arguments: &'a Value, key: &str) -> Result<&'a str> { + arguments + .get(key) + .and_then(Value::as_str) + .ok_or_else(|| anyhow::anyhow!("missing or invalid `{key}` argument")) +} diff --git a/cli/src/builtins/tau3/mod.rs b/cli/src/builtins/tau3/mod.rs new file mode 100644 index 0000000..803504a --- /dev/null +++ b/cli/src/builtins/tau3/mod.rs @@ -0,0 +1,377 @@ +use std::sync::Arc; + +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::builtins::common::{BuiltinConfig, hash_input, run_timed_step}; +use crate::builtins::{BuiltinContext, BuiltinWorkflow}; +use crate::llm::Sampler; + +use environment::{MockEnvironment, MockState, ToolEnvironment}; +use model::GenaiToolChatModel; +use orchestrator::{RunConversation, Trajectory, run_conversation, visible_transcript}; +use scoring::{Scores, pass_at_k, score}; + +mod environment; +mod model; +mod orchestrator; +mod scoring; + +const HARNESS_VERSION: &str = "tau3-native-conformance-v1"; +const DEFAULT_AGENT_POLICY: &str = "You are a customer-service agent. Follow policy, use tools to inspect or change records, never claim an action succeeded before a tool confirms it, and clearly communicate the outcome to the customer."; + +/// Native structured-tool-calling conformance benchmark for the Quantiles τ³ harness. +pub struct Tau3MockBuiltin; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Tau3Config { + #[serde(flatten)] + base: BuiltinConfig, + user_model: Option, + #[serde(default = "default_trials")] + trials: usize, + #[serde(default = "default_max_turns")] + max_turns: usize, +} + +impl Default for Tau3Config { + fn default() -> Self { + Self { + base: BuiltinConfig::default(), + user_model: None, + trials: default_trials(), + max_turns: default_max_turns(), + } + } +} + +const fn default_trials() -> usize { + 1 +} +const fn default_max_turns() -> usize { + 20 +} + +#[derive(Clone)] +struct Task { + id: &'static str, + user_goal: &'static str, + initial_state: MockState, + expected_state: Value, + required_communication: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct TrialOutput { + task_id: String, + trial: usize, + trajectory: Trajectory, + final_state: Value, + reward: Scores, +} + +#[derive(Serialize)] +struct RunInput<'a> { + harness_version: &'a str, + domain: &'a str, + agent_model: String, + user_model: String, + tasks: usize, + trials: usize, + max_turns: usize, +} + +#[derive(Serialize)] +struct RunOutput { + harness_version: &'static str, + domain: &'static str, + tasks_completed: usize, + trials_completed: usize, +} + +#[async_trait::async_trait] +impl BuiltinWorkflow for Tau3MockBuiltin { + fn name(&self) -> String { + "tau3-mock".to_owned() + } + + #[expect(clippy::cast_precision_loss)] + async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()> { + let config: Tau3Config = ctx + .input + .map(serde_json::from_str) + .transpose() + .context("invalid tau3-mock input JSON")? + .unwrap_or_default(); + if config.trials == 0 { + bail!("trials must be > 0"); + } + if config.max_turns == 0 { + bail!("max_turns must be > 0"); + } + if config.base.limit == Some(0) { + bail!("limit must be > 0"); + } + + let agent_sampler = config + .base + .model + .as_ref() + .context("tau3-mock requires `model` (openai, anthropic, or gemini)")?; + let user_sampler = config.user_model.as_ref().unwrap_or(agent_sampler); + let agent = Arc::new(GenaiToolChatModel::from_sampler(agent_sampler)?); + let user = Arc::new(GenaiToolChatModel::from_sampler(user_sampler)?); + let mut tasks = tasks(); + if let Some(limit) = config.base.limit { + tasks.truncate(limit.min(tasks.len())); + } + + let run_input = RunInput { + harness_version: HARNESS_VERSION, + domain: "mock", + agent_model: agent_sampler.to_string(), + user_model: user_sampler.to_string(), + tasks: tasks.len(), + trials: config.trials, + max_turns: config.max_turns, + }; + crate::db::set_run_input(ctx.db, ctx.run_id, &serde_json::to_string(&run_input)?).await?; + + let mut rewards_by_task = Vec::with_capacity(tasks.len()); + let mut trials_completed = 0usize; + for task in tasks { + let mut task_rewards = Vec::with_capacity(config.trials); + for trial in 0..config.trials { + let step_key = format!("task-{}-trial-{trial}", task.id); + let input_hash = hash_input(&format!( + "{HARNESS_VERSION}\ntask={}\nagent={agent_sampler}\nuser={user_sampler}\nmax_turns={}", + task.id, config.max_turns + )); + let agent = Arc::clone(&agent); + let user = Arc::clone(&user); + let task = task.clone(); + let (output, step_id) = run_timed_step( + ctx.db, + ctx.metrics_store, + ctx.run_id, + &step_key, + &input_hash, + async move { + run_trial( + &task, + trial, + agent.as_ref(), + user.as_ref(), + config.max_turns, + ) + .await + }, + ) + .await?; + if let Some(step_id) = step_id { + for (name, value) in [ + ("reward", output.reward.overall), + ("database_reward", output.reward.database), + ("communicate_reward", output.reward.communication), + ("tool_calls", output.trajectory.tool_calls as f64), + ("turns", output.trajectory.turns as f64), + ] { + ctx.metrics_store + .emit(ctx.run_id, Some(step_id), name, value, None) + .await; + } + } + task_rewards.push(output.reward.overall); + trials_completed += 1; + } + rewards_by_task.push(task_rewards); + } + + emit_aggregate_metrics(&ctx, &rewards_by_task).await; + crate::db::set_run_output( + ctx.db, + ctx.run_id, + &serde_json::to_string(&RunOutput { + harness_version: HARNESS_VERSION, + domain: "mock", + tasks_completed: rewards_by_task.len(), + trials_completed, + })?, + ) + .await?; + Ok(()) + } +} + +async fn run_trial( + task: &Task, + trial: usize, + agent: &dyn model::ToolChatModel, + user: &dyn model::ToolChatModel, + max_turns: usize, +) -> Result { + let mut environment = MockEnvironment::new(task.initial_state.clone()); + let trajectory = run_conversation(RunConversation { + agent, + user, + environment: &mut environment, + agent_policy: DEFAULT_AGENT_POLICY, + user_goal: task.user_goal, + max_turns, + }) + .await?; + let final_state = environment.state(); + let reward = score( + &final_state, + &task.expected_state, + &visible_transcript(&trajectory), + &task.required_communication, + ); + Ok(TrialOutput { + task_id: task.id.to_owned(), + trial, + trajectory, + final_state, + reward, + }) +} + +#[expect(clippy::cast_precision_loss)] +async fn emit_aggregate_metrics(ctx: &BuiltinContext<'_>, rewards_by_task: &[Vec]) { + let rewards = rewards_by_task + .iter() + .flatten() + .copied() + .collect::>(); + if !rewards.is_empty() { + let mean = rewards.iter().sum::() / rewards.len() as f64; + ctx.metrics_store + .emit(ctx.run_id, None, "reward", mean, None) + .await; + } + let trials = rewards_by_task.first().map_or(0, Vec::len); + for k in 1..=trials { + if let Some(value) = pass_at_k(rewards_by_task, k) { + ctx.metrics_store + .emit(ctx.run_id, None, &format!("pass_at_{k}"), value, None) + .await; + } + } + ctx.metrics_store + .emit( + ctx.run_id, + None, + "task_count", + rewards_by_task.len() as f64, + None, + ) + .await; + ctx.metrics_store + .emit(ctx.run_id, None, "trial_count", rewards.len() as f64, None) + .await; +} + +fn tasks() -> Vec { + vec![Task { + id: "mock-update-email", + user_goal: "You are Morgan Lee, customer ID C-100. Ask the agent to change your account email from old@example.com to morgan.lee@example.com. Do not provide information the agent has not requested.", + initial_state: MockState { + customer_id: "C-100".to_owned(), + name: "Morgan Lee".to_owned(), + email: "old@example.com".to_owned(), + }, + expected_state: json!({"customer_id": "C-100", "email": "morgan.lee@example.com"}), + required_communication: vec!["morgan.lee@example.com".to_owned()], + }] +} + +#[cfg(test)] +mod tests { + use super::model::{ModelMessage, ModelTurn, ToolChatModel, ToolDefinition, ToolInvocation}; + use super::*; + use anyhow::Result; + use async_trait::async_trait; + use serde_json::json; + use std::collections::VecDeque; + use std::sync::Mutex; + + struct ScriptedModel(Mutex>); + + impl ScriptedModel { + fn new(turns: Vec) -> Self { + Self(Mutex::new(turns.into())) + } + } + + #[async_trait] + impl ToolChatModel for ScriptedModel { + async fn generate( + &self, + _: &str, + _: &[ModelMessage], + _: &[ToolDefinition], + ) -> Result { + self.0 + .lock() + .unwrap() + .pop_front() + .ok_or_else(|| anyhow::anyhow!("script exhausted")) + } + } + + #[tokio::test] + async fn native_tool_loop_updates_state_and_scores_trajectory() { + let user = ScriptedModel::new(vec![ + ModelTurn { + content: Some("Please change my email. My customer ID is C-100.".to_owned()), + tool_calls: vec![], + }, + ModelTurn { + content: Some("".to_owned()), + tool_calls: vec![], + }, + ]); + let agent = ScriptedModel::new(vec![ + ModelTurn { + content: None, + tool_calls: vec![ToolInvocation { + call_id: "1".to_owned(), + name: "get_customer".to_owned(), + arguments: json!({"customer_id": "C-100"}), + thought_signatures: None, + }], + }, + ModelTurn { + content: None, + tool_calls: vec![ToolInvocation { + call_id: "2".to_owned(), + name: "update_customer_email".to_owned(), + arguments: json!({"customer_id": "C-100", "email": "morgan.lee@example.com"}), + thought_signatures: None, + }], + }, + ModelTurn { + content: Some("Your email is now morgan.lee@example.com.".to_owned()), + tool_calls: vec![], + }, + ]); + let output = run_trial(&tasks()[0], 0, &agent, &user, 5).await.unwrap(); + assert!((output.reward.overall - 1.0).abs() < f64::EPSILON); + assert_eq!(output.trajectory.tool_calls, 2); + assert_eq!(output.final_state["email"], "morgan.lee@example.com"); + } + + #[test] + fn rejects_sampler_without_structured_tools() { + let result = GenaiToolChatModel::from_sampler(&Sampler::Random); + assert!(result.is_err()); + } + + #[test] + fn resolver_exposes_only_explicit_conformance_name() { + assert!(crate::builtins::resolve("tau3-mock").is_some()); + assert!(crate::builtins::resolve("tau3").is_none()); + } +} diff --git a/cli/src/builtins/tau3/model.rs b/cli/src/builtins/tau3/model.rs new file mode 100644 index 0000000..f35eb77 --- /dev/null +++ b/cli/src/builtins/tau3/model.rs @@ -0,0 +1,179 @@ +use anyhow::{Context, Result, bail}; +use async_trait::async_trait; +use genai::chat::{ + ChatMessage, ChatRequest, ContentPart, MessageContent, Tool, ToolCall, ToolResponse, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::llm::Sampler; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub(crate) struct ToolDefinition { + pub(crate) name: String, + pub(crate) description: String, + pub(crate) schema: Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub(crate) struct ToolInvocation { + pub(crate) call_id: String, + pub(crate) name: String, + pub(crate) arguments: Value, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) thought_signatures: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub(crate) struct ToolResult { + pub(crate) call_id: String, + pub(crate) name: String, + pub(crate) content: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "role", rename_all = "snake_case")] +pub(crate) enum ModelMessage { + User { + content: String, + }, + Assistant { + #[serde(default, skip_serializing_if = "Option::is_none")] + content: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + tool_calls: Vec, + }, + Tool { + results: Vec, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub(crate) struct ModelTurn { + pub(crate) content: Option, + pub(crate) tool_calls: Vec, +} + +#[async_trait] +pub(crate) trait ToolChatModel: Send + Sync { + async fn generate( + &self, + system: &str, + messages: &[ModelMessage], + tools: &[ToolDefinition], + ) -> Result; +} + +pub(crate) struct GenaiToolChatModel { + client: genai::Client, + model: String, +} + +impl GenaiToolChatModel { + pub(crate) fn from_sampler(sampler: &Sampler) -> Result { + let model = match sampler { + Sampler::OpenAI { model_id } => format!("openai::{model_id}"), + Sampler::Anthropic { model_id } => format!("anthropic::{model_id}"), + Sampler::Gemini { model_id } => format!("gemini::{model_id}"), + Sampler::Random | Sampler::RandomLabel | Sampler::CloudflareAIGateway { .. } => { + bail!( + "tau3-mock requires a model backend with structured tool calling; supported providers are openai, anthropic, and gemini" + ) + } + }; + Ok(Self { + client: genai::Client::default(), + model, + }) + } +} + +#[async_trait] +impl ToolChatModel for GenaiToolChatModel { + async fn generate( + &self, + system: &str, + messages: &[ModelMessage], + tools: &[ToolDefinition], + ) -> Result { + let messages = messages + .iter() + .map(to_genai_message) + .collect::>>()?; + let tools = tools + .iter() + .map(|tool| { + Tool::new(tool.name.clone()) + .with_description(tool.description.clone()) + .with_schema(tool.schema.clone()) + }) + .collect::>(); + let mut request = ChatRequest::from_messages(messages).with_system(system); + if !tools.is_empty() { + request = request.with_tools(tools); + } + let response = self + .client + .exec_chat(&self.model, request, None) + .await + .with_context(|| format!("{} chat request failed", self.model))?; + + let content = { + let texts = response.content.texts(); + (!texts.is_empty()).then(|| texts.join("\n")) + }; + let tool_calls = response + .content + .tool_calls() + .into_iter() + .map(|call| ToolInvocation { + call_id: call.call_id.clone(), + name: call.fn_name.clone(), + arguments: call.fn_arguments.clone(), + thought_signatures: call.thought_signatures.clone(), + }) + .collect(); + Ok(ModelTurn { + content, + tool_calls, + }) + } +} + +fn to_genai_message(message: &ModelMessage) -> Result { + Ok(match message { + ModelMessage::User { content } => ChatMessage::user(content.clone()), + ModelMessage::Assistant { + content, + tool_calls, + } => { + let mut parts = Vec::new(); + if let Some(content) = content { + parts.push(ContentPart::Text(content.clone())); + } + for call in tool_calls { + parts.push(ContentPart::ToolCall(ToolCall { + call_id: call.call_id.clone(), + fn_name: call.name.clone(), + fn_arguments: call.arguments.clone(), + thought_signatures: call.thought_signatures.clone(), + })); + } + ChatMessage::assistant(MessageContent::from_parts(parts)) + } + ModelMessage::Tool { results } => { + if results.is_empty() { + bail!("tool message must contain at least one result"); + } + ChatMessage::from( + results + .iter() + .map(|result| { + ToolResponse::new(&result.call_id, &result.content) + .with_fn_name(&result.name) + }) + .collect::>(), + ) + } + }) +} diff --git a/cli/src/builtins/tau3/orchestrator.rs b/cli/src/builtins/tau3/orchestrator.rs new file mode 100644 index 0000000..66ea0d6 --- /dev/null +++ b/cli/src/builtins/tau3/orchestrator.rs @@ -0,0 +1,165 @@ +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; + +use super::environment::ToolEnvironment; +use super::model::{ModelMessage, ToolChatModel, ToolResult}; + +const END_TOKEN: &str = ""; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub(crate) struct Trajectory { + pub(crate) messages: Vec, + pub(crate) tool_calls: usize, + pub(crate) turns: usize, + pub(crate) terminated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "role", rename_all = "snake_case")] +pub(crate) enum TrajectoryMessage { + User { + content: String, + }, + Assistant { + content: String, + }, + Tool { + name: String, + arguments: serde_json::Value, + result: serde_json::Value, + }, +} + +pub(crate) struct RunConversation<'a> { + pub(crate) agent: &'a dyn ToolChatModel, + pub(crate) user: &'a dyn ToolChatModel, + pub(crate) environment: &'a mut dyn ToolEnvironment, + pub(crate) agent_policy: &'a str, + pub(crate) user_goal: &'a str, + pub(crate) max_turns: usize, +} + +pub(crate) async fn run_conversation(args: RunConversation<'_>) -> Result { + let mut trajectory = Trajectory { + messages: Vec::new(), + tool_calls: 0, + turns: 0, + terminated: false, + }; + let mut agent_messages = Vec::new(); + + for turn in 0..args.max_turns { + let user_text = generate_user_message(args.user, args.user_goal, &trajectory, turn).await?; + if user_text.trim() == END_TOKEN { + trajectory.terminated = true; + break; + } + trajectory.messages.push(TrajectoryMessage::User { + content: user_text.clone(), + }); + agent_messages.push(ModelMessage::User { content: user_text }); + + loop { + let response = args + .agent + .generate( + args.agent_policy, + &agent_messages, + &args.environment.tools(), + ) + .await + .context("agent generation failed")?; + if response.content.is_none() && response.tool_calls.is_empty() { + bail!("agent returned neither text nor a tool call"); + } + agent_messages.push(ModelMessage::Assistant { + content: response.content.clone(), + tool_calls: response.tool_calls.clone(), + }); + + if !response.tool_calls.is_empty() { + let mut results = Vec::with_capacity(response.tool_calls.len()); + for call in &response.tool_calls { + let result = + args.environment.invoke(call).await.unwrap_or_else( + |error| serde_json::json!({"error": format!("{error:#}")}), + ); + trajectory.messages.push(TrajectoryMessage::Tool { + name: call.name.clone(), + arguments: call.arguments.clone(), + result: result.clone(), + }); + results.push(ToolResult { + call_id: call.call_id.clone(), + name: call.name.clone(), + content: serde_json::to_string(&result) + .expect("JSON value is serializable"), + }); + trajectory.tool_calls += 1; + } + agent_messages.push(ModelMessage::Tool { results }); + if args.environment.terminated() { + trajectory.terminated = true; + break; + } + continue; + } + + if let Some(content) = response.content { + trajectory + .messages + .push(TrajectoryMessage::Assistant { content }); + } + break; + } + trajectory.turns = turn + 1; + if trajectory.terminated { + break; + } + } + + Ok(trajectory) +} + +async fn generate_user_message( + user: &dyn ToolChatModel, + goal: &str, + trajectory: &Trajectory, + turn: usize, +) -> Result { + let transcript = visible_transcript(trajectory); + let prompt = if turn == 0 { + "Begin the conversation as the customer.".to_owned() + } else { + format!( + "Conversation so far:\n{transcript}\n\nRespond as the customer, or output {END_TOKEN} if the request is complete." + ) + }; + let response = user + .generate( + &format!( + "You simulate a customer for an evaluation. Follow this private goal exactly and do not invent facts:\n{goal}\nOutput only the customer's next message. Output {END_TOKEN} once the goal is satisfied." + ), + &[ModelMessage::User { content: prompt }], + &[], + ) + .await + .context("user simulator generation failed")?; + if !response.tool_calls.is_empty() { + bail!("user simulator attempted a tool call"); + } + response.content.context("user simulator returned no text") +} + +pub(crate) fn visible_transcript(trajectory: &Trajectory) -> String { + trajectory + .messages + .iter() + .filter_map(|message| match message { + TrajectoryMessage::User { content } => Some(format!("Customer: {content}")), + TrajectoryMessage::Assistant { content } => Some(format!("Agent: {content}")), + TrajectoryMessage::Tool { .. } => None, + }) + .collect::>() + .join("\n") +} diff --git a/cli/src/builtins/tau3/scoring.rs b/cli/src/builtins/tau3/scoring.rs new file mode 100644 index 0000000..e6a1269 --- /dev/null +++ b/cli/src/builtins/tau3/scoring.rs @@ -0,0 +1,103 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub(crate) struct Scores { + #[serde(rename = "reward")] + pub(crate) overall: f64, + #[serde(rename = "database_reward")] + pub(crate) database: f64, + #[serde(rename = "communicate_reward")] + pub(crate) communication: f64, +} + +pub(crate) fn score( + final_state: &Value, + expected_state: &Value, + transcript: &str, + required: &[String], +) -> Scores { + let database_reward = f64::from(value_contains(final_state, expected_state)); + let transcript = transcript.to_lowercase(); + let communicate_reward = f64::from( + required + .iter() + .all(|needle| transcript.contains(&needle.to_lowercase())), + ); + Scores { + overall: database_reward * communicate_reward, + database: database_reward, + communication: communicate_reward, + } +} + +fn value_contains(actual: &Value, expected: &Value) -> bool { + match (actual, expected) { + (Value::Object(actual), Value::Object(expected)) => expected.iter().all(|(key, value)| { + actual + .get(key) + .is_some_and(|actual| value_contains(actual, value)) + }), + (Value::Array(actual), Value::Array(expected)) => expected + .iter() + .all(|value| actual.iter().any(|actual| value_contains(actual, value))), + _ => actual == expected, + } +} + +#[expect(clippy::cast_precision_loss)] +pub(crate) fn pass_at_k(rewards_by_task: &[Vec], k: usize) -> Option { + if k == 0 + || rewards_by_task.is_empty() + || rewards_by_task.iter().any(|rewards| rewards.len() < k) + { + return None; + } + let total = rewards_by_task + .iter() + .map(|rewards| { + let n = rewards.len(); + let failures = rewards.iter().filter(|reward| **reward < 1.0).count(); + if failures < k { + 1.0 + } else { + 1.0 - combination_ratio(failures, n, k) + } + }) + .sum::(); + Some(total / rewards_by_task.len() as f64) +} + +#[expect(clippy::cast_precision_loss)] +fn combination_ratio(failures: usize, trials: usize, k: usize) -> f64 { + (0..k).fold(1.0, |ratio, i| { + ratio * (failures - i) as f64 / (trials - i) as f64 + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn scores_database_and_communication_requirements() { + let reward = score( + &json!({"email": "new@example.com", "untouched": true}), + &json!({"email": "new@example.com"}), + "Agent: Your email is now new@example.com", + &["new@example.com".to_owned()], + ); + assert!((reward.overall - 1.0).abs() < f64::EPSILON); + assert!((reward.database - 1.0).abs() < f64::EPSILON); + assert!((reward.communication - 1.0).abs() < f64::EPSILON); + } + + #[test] + fn computes_unbiased_pass_at_k() { + let rewards = vec![vec![1.0, 0.0, 0.0, 0.0], vec![1.0, 1.0, 0.0, 0.0]]; + assert_eq!(pass_at_k(&rewards, 1), Some(0.375)); + assert_eq!(pass_at_k(&rewards, 4), Some(1.0)); + assert_eq!(pass_at_k(&rewards, 5), None); + } +} From 3319bee6f31a2180070d5dbc65611c5187cca652 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:59:27 -0700 Subject: [PATCH 2/2] adding demo model for tau3-mock --- README.md | 2 + cli/README.md | 2 +- cli/src/builtins/tau3/README.md | 11 +++- cli/src/builtins/tau3/mod.rs | 95 ++++++++++++++++++++++++----- cli/src/builtins/tau3/model.rs | 105 ++++++++++++++++++++++++++++++++ cli/src/commands/run.rs | 6 +- 6 files changed, 202 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ba3e083..40cef0c 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ structured tool calling, model-driven user simulation, mutable task environments, trajectory recording, component rewards, and `pass_at_k`. It is not an official τ³ leaderboard domain; see the [`tau3-mock` implementation notes](cli/src/builtins/tau3/README.md). +Running `qt run tau3-mock` uses a local deterministic structured-tool demo model +to validate the harness without provider calls. The `tau3-airline` name is reserved for the forthcoming airline implementation and currently returns a not-implemented error. diff --git a/cli/README.md b/cli/README.md index 2111018..628c072 100644 --- a/cli/README.md +++ b/cli/README.md @@ -36,7 +36,7 @@ See the [CLI reference](https://quantiles.io/documentation/reference/cli) for a The CLI supports two locally configured evaluation types and remote registry benchmarks: - [`custom_nocode` evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) define the dataset, prompt template, model, and scoring method entirely in configuration. Supported scoring styles include exact match, multiple choice, and text similarity. -- `tau3-mock` is a native structured-tool-calling conformance benchmark for the in-process τ³-style agent harness. It records tool trajectories, isolated environment state, component rewards, and `pass_at_k`; it is not an official τ³ leaderboard domain. See [`src/builtins/tau3/README.md`](src/builtins/tau3/README.md). +- `tau3-mock` is a native structured-tool-calling conformance benchmark for the in-process τ³-style agent harness. With no model configured, it uses a local deterministic structured-tool demo model. It records tool trajectories, isolated environment state, component rewards, and `pass_at_k`; it is not an official τ³ leaderboard domain. See [`src/builtins/tau3/README.md`](src/builtins/tau3/README.md). - `tau3-airline` is reserved for the forthcoming official-compatible airline implementation and currently returns a not-implemented error. - [`custom_code` evaluations](https://quantiles.io/documentation/custom-evaluations) run your own Python evaluation through the Quantiles Python SDK. - Registry benchmarks are downloaded by name from the Quantiles remote benchmark service and executed locally using the native `custom_nocode` runtime. diff --git a/cli/src/builtins/tau3/README.md b/cli/src/builtins/tau3/README.md index f81e2db..b84eafc 100644 --- a/cli/src/builtins/tau3/README.md +++ b/cli/src/builtins/tau3/README.md @@ -16,7 +16,16 @@ the official τ³-bench 1.0.1 airline, retail, telecom, banking-knowledge, or vo tracks, and its scores must not be submitted to or compared with the official leaderboard. -Run it with separate agent and user models: +Run it without configuration to use the local deterministic structured-tool +demo model for both the agent and simulated user: + +```console +qt run tau3-mock +``` + +The demo run validates the harness without making network calls and is not +model-quality evidence. To evaluate a provider model, pass separate agent and +user models: ```console qt run tau3-mock --input '{ diff --git a/cli/src/builtins/tau3/mod.rs b/cli/src/builtins/tau3/mod.rs index a197da4..d80f230 100644 --- a/cli/src/builtins/tau3/mod.rs +++ b/cli/src/builtins/tau3/mod.rs @@ -9,7 +9,7 @@ use crate::builtins::{BuiltinContext, BuiltinWorkflow}; use crate::llm::Sampler; use environment::{MockEnvironment, MockState, ToolEnvironment}; -use model::GenaiToolChatModel; +use model::{GenaiToolChatModel, Tau3DemoToolChatModel, ToolChatModel}; use orchestrator::{RunConversation, Trajectory, run_conversation, visible_transcript}; use scoring::{Scores, pass_at_k, score}; @@ -107,6 +107,48 @@ struct RunOutput { trials_completed: usize, } +struct ResolvedModels { + agent: Arc, + user: Arc, + agent_name: String, + user_name: String, +} + +fn resolve_models(config: &Tau3Config) -> Result { + let (agent, agent_name): (Arc, String) = match &config.model { + Some(sampler) => ( + Arc::new(GenaiToolChatModel::from_sampler(sampler)?), + sampler.to_string(), + ), + None => ( + Arc::new(Tau3DemoToolChatModel::agent()), + Tau3DemoToolChatModel::NAME.to_owned(), + ), + }; + let (user, user_name): (Arc, String) = match &config.user_model { + Some(sampler) => ( + Arc::new(GenaiToolChatModel::from_sampler(sampler)?), + sampler.to_string(), + ), + None if config.model.is_some() => ( + Arc::new(GenaiToolChatModel::from_sampler( + config.model.as_ref().expect("model is present"), + )?), + agent_name.clone(), + ), + None => ( + Arc::new(Tau3DemoToolChatModel::user()), + Tau3DemoToolChatModel::NAME.to_owned(), + ), + }; + Ok(ResolvedModels { + agent, + user, + agent_name, + user_name, + }) +} + #[async_trait::async_trait] impl BuiltinWorkflow for Tau3MockBuiltin { fn name(&self) -> String { @@ -131,13 +173,7 @@ impl BuiltinWorkflow for Tau3MockBuiltin { bail!("limit must be > 0"); } - let agent_sampler = config - .model - .as_ref() - .context("tau3-mock requires `model` (openai, anthropic, or gemini)")?; - let user_sampler = config.user_model.as_ref().unwrap_or(agent_sampler); - let agent = Arc::new(GenaiToolChatModel::from_sampler(agent_sampler)?); - let user = Arc::new(GenaiToolChatModel::from_sampler(user_sampler)?); + let models = resolve_models(&config)?; let mut tasks = tasks(); if let Some(limit) = config.limit { tasks.truncate(limit.min(tasks.len())); @@ -146,8 +182,8 @@ impl BuiltinWorkflow for Tau3MockBuiltin { let run_input = RunInput { harness_version: HARNESS_VERSION, domain: "mock", - agent_model: agent_sampler.to_string(), - user_model: user_sampler.to_string(), + agent_model: models.agent_name.clone(), + user_model: models.user_name.clone(), tasks: tasks.len(), trials: config.trials, max_turns: config.max_turns, @@ -161,11 +197,11 @@ impl BuiltinWorkflow for Tau3MockBuiltin { for trial in 0..config.trials { let step_key = format!("task-{}-trial-{trial}", task.id); let input_hash = hash_input(&format!( - "{HARNESS_VERSION}\ntask={}\nagent={agent_sampler}\nuser={user_sampler}\nmax_turns={}", - task.id, config.max_turns + "{HARNESS_VERSION}\ntask={}\nagent={}\nuser={}\nmax_turns={}", + task.id, models.agent_name, models.user_name, config.max_turns )); - let agent = Arc::clone(&agent); - let user = Arc::clone(&user); + let agent = Arc::clone(&models.agent); + let user = Arc::clone(&models.user); let task = task.clone(); let (output, step_id) = run_timed_step( ctx.db, @@ -420,4 +456,35 @@ mod tests { "tau3-airline is recognized but not implemented yet" ); } + + #[tokio::test] + async fn mock_defaults_to_local_structured_tool_demo_models() { + let tmpdir = tempfile::tempdir().unwrap(); + crate::db::init_workspace(tmpdir.path()).await.unwrap(); + let db = crate::db::open_workspace(tmpdir.path()).await.unwrap(); + let metrics_store = + crate::metrics_store::MetricsStore::new(crate::db::metrics_dir(tmpdir.path())).unwrap(); + let run_id = crate::db::create_run(&db, "tau3-mock", None).await.unwrap(); + let builtin = crate::builtins::resolve("tau3-mock").unwrap(); + + builtin + .execute(BuiltinContext { + db: &db, + metrics_store: &metrics_store, + run_id, + workflow_name: "tau3-mock", + input: None, + quiet: true, + }) + .await + .unwrap(); + + let run = crate::db::get_run(&db, run_id).await.unwrap(); + let input: Value = serde_json::from_str(run.input.as_deref().unwrap()).unwrap(); + let output: Value = serde_json::from_str(run.output.as_deref().unwrap()).unwrap(); + assert_eq!(input["agent_model"], Tau3DemoToolChatModel::NAME); + assert_eq!(input["user_model"], Tau3DemoToolChatModel::NAME); + assert_eq!(output["tasks_completed"], 1); + assert_eq!(output["trials_completed"], 1); + } } diff --git a/cli/src/builtins/tau3/model.rs b/cli/src/builtins/tau3/model.rs index f35eb77..06ac131 100644 --- a/cli/src/builtins/tau3/model.rs +++ b/cli/src/builtins/tau3/model.rs @@ -64,6 +64,111 @@ pub(crate) trait ToolChatModel: Send + Sync { ) -> Result; } +/// Local deterministic model used to validate the bundled mock harness without +/// making provider calls. It intentionally supports only the bundled mock task. +pub(crate) struct Tau3DemoToolChatModel { + role: DemoRole, +} + +enum DemoRole { + Agent, + User, +} + +impl Tau3DemoToolChatModel { + pub(crate) const NAME: &'static str = "tau3-demo"; + + pub(crate) const fn agent() -> Self { + Self { + role: DemoRole::Agent, + } + } + + pub(crate) const fn user() -> Self { + Self { + role: DemoRole::User, + } + } + + fn generate_agent(messages: &[ModelMessage], tools: &[ToolDefinition]) -> Result { + let tool_call = match messages.last() { + Some(ModelMessage::User { .. }) => ToolInvocation { + call_id: "tau3-demo-get-customer".to_owned(), + name: "get_customer".to_owned(), + arguments: serde_json::json!({"customer_id": "C-100"}), + thought_signatures: None, + }, + Some(ModelMessage::Tool { results }) + if results + .last() + .is_some_and(|result| result.name == "get_customer") => + { + ToolInvocation { + call_id: "tau3-demo-update-email".to_owned(), + name: "update_customer_email".to_owned(), + arguments: serde_json::json!({ + "customer_id": "C-100", + "email": "morgan.lee@example.com" + }), + thought_signatures: None, + } + } + Some(ModelMessage::Tool { results }) + if results + .last() + .is_some_and(|result| result.name == "update_customer_email") => + { + return Ok(ModelTurn { + content: Some("Your email is now morgan.lee@example.com.".to_owned()), + tool_calls: Vec::new(), + }); + } + _ => bail!("tau3 demo agent received an unexpected conversation state"), + }; + + if !tools.iter().any(|tool| tool.name == tool_call.name) { + bail!("tau3 demo agent requires the `{}` tool", tool_call.name); + } + Ok(ModelTurn { + content: None, + tool_calls: vec![tool_call], + }) + } + + fn generate_user(messages: &[ModelMessage], tools: &[ToolDefinition]) -> Result { + if !tools.is_empty() { + bail!("tau3 demo user does not support tools"); + } + let Some(ModelMessage::User { content: prompt }) = messages.last() else { + bail!("tau3 demo user received an unexpected conversation state"); + }; + let content = if prompt == "Begin the conversation as the customer." { + "Please change my email to morgan.lee@example.com. My customer ID is C-100." + } else { + "" + }; + Ok(ModelTurn { + content: Some(content.to_owned()), + tool_calls: Vec::new(), + }) + } +} + +#[async_trait] +impl ToolChatModel for Tau3DemoToolChatModel { + async fn generate( + &self, + _system: &str, + messages: &[ModelMessage], + tools: &[ToolDefinition], + ) -> Result { + match self.role { + DemoRole::Agent => Self::generate_agent(messages, tools), + DemoRole::User => Self::generate_user(messages, tools), + } + } +} + pub(crate) struct GenaiToolChatModel { client: genai::Client, model: String, diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index f3601b6..eecc1ce 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -39,7 +39,9 @@ pub async fn run( run_configured_benchmark(workflow_name, cli_input, json, process_start, bench).await } None => { - if let Some(remote) = + if let Some(builtin) = builtins::resolve(workflow_name) { + run_native_benchmark(workflow_name, cli_input, json, process_start, builtin).await + } else if let Some(remote) = qt::benchmark_registry::resolve_and_download(workflow_name, None, &remote_url) .await? { @@ -52,8 +54,6 @@ pub async fn run( remote, ) .await - } else if let Some(builtin) = builtins::resolve(workflow_name) { - run_native_benchmark(workflow_name, cli_input, json, process_start, builtin).await } else { bail!("no config section found for benchmark `{workflow_name}`"); }