diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 9230922b449..e1b0a0e3b28 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -39,6 +39,16 @@ on: description: 'Run benchmarks without uploading results' required: false default: 'false' + skip_task_catalog_upload: + description: 'Skip uploading the benchmark task catalog' + required: false + type: boolean + default: false + post_discord: + description: 'Post the analysis summary to Discord' + required: false + type: boolean + default: false permissions: contents: read @@ -167,7 +177,8 @@ jobs: LLM_BENCH_CONCURRENCY: "8" LLM_BENCH_RUST_CONCURRENCY: "4" LLM_BENCH_CSHARP_CONCURRENCY: "2" - LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || '2' }} + LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || matrix.lang == 'csharp' && '1' || '2' }} + LLM_BENCHMARK_REPORT_DIR: ${{ runner.temp }}/llm-benchmark-reports INPUT_LANGUAGE: ${{ matrix.lang }} INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }} INPUT_MODELS: ${{ inputs.models || '' }} @@ -175,6 +186,7 @@ jobs: INPUT_CATEGORIES: ${{ inputs.categories || '' }} INPUT_TASKS: ${{ inputs.tasks || '' }} INPUT_DRY_RUN: ${{ inputs.dry_run || 'false' }} + INPUT_SKIP_TASK_CATALOG_UPLOAD: ${{ inputs.skip_task_catalog_upload || 'false' }} run: | LANG="$INPUT_LANGUAGE" MODEL_SET="$INPUT_MODEL_SET" @@ -183,6 +195,7 @@ jobs: CATEGORIES="$INPUT_CATEGORIES" TASKS="$INPUT_TASKS" DRY_RUN="$INPUT_DRY_RUN" + SKIP_TASK_CATALOG_UPLOAD="$INPUT_SKIP_TASK_CATALOG_UPLOAD" case "$MODEL_SET" in website_active) @@ -220,6 +233,9 @@ jobs: if [ "$DRY_RUN" = "true" ]; then EXTRA_ARGS+=(--dry-run) fi + if [ "$SKIP_TASK_CATALOG_UPLOAD" = "true" ]; then + EXTRA_ARGS+=(--skip-task-catalog-upload) + fi if [ "$MODEL_SET" = "website_active" ]; then llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}" @@ -228,3 +244,80 @@ jobs: else llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}" fi + + - name: Upload analysis reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: llm-benchmark-analysis-${{ matrix.lang }} + path: ${{ runner.temp }}/llm-benchmark-reports + if-no-files-found: ignore + retention-days: 14 + + summarize: + name: Summarize benchmark analysis + if: always() + needs: run-benchmarks + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: dsherret/rust-toolchain-file@v1 + + - name: Download analysis reports + continue-on-error: true + uses: actions/download-artifact@v4 + with: + pattern: llm-benchmark-analysis-* + path: reports + merge-multiple: true + + - name: Publish workflow summary + run: | + echo "# LLM benchmark analysis" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + + if [ ! -d reports ] || ! find reports -type f -name '*.md' -print -quit | grep -q .; then + echo "No analysis reports were produced." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "Full per-model reports are available in the workflow artifacts." >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + + find reports -type f -name '*.md' -print | sort | while IFS= read -r report; do + awk '/^## Failure patterns/{exit} NR <= 80 {print}' "$report" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + echo "---" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + done + + - name: Prepare Discord summary + if: github.event_name == 'schedule' || inputs.post_discord + env: + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + RUN_LABEL: ${{ github.event_name == 'schedule' && 'Weekly scheduled run' || 'Manual run' }} + run: > + cargo ci other-workflows llm-benchmark-summary + --reports-dir reports + --run-url "$RUN_URL" + --run-label "$RUN_LABEL" + --output discord-payload.json + + - name: Post Discord summary + if: github.event_name == 'schedule' || inputs.post_discord + continue-on-error: true + env: + DISCORD_WEBHOOK_URL: ${{ secrets.LLM_BENCHMARK_DISCORD_WEBHOOK_URL }} + run: | + if [ -z "$DISCORD_WEBHOOK_URL" ]; then + echo "::warning::LLM_BENCHMARK_DISCORD_WEBHOOK_URL is not configured" + exit 0 + fi + + curl --fail --silent --show-error \ + --header 'Content-Type: application/json' \ + --data-binary @discord-payload.json \ + "$DISCORD_WEBHOOK_URL" diff --git a/Cargo.lock b/Cargo.lock index 07e571bf069..4239f6af115 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1022,6 +1022,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ci-llm-benchmark-summary" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "serde_json", +] + [[package]] name = "ci-module-latest-deps" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a2b27d448c4..f8e544505da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,7 @@ members = [ "tools/ci/commands/workflow-coordinator", "tools/ci/commands/workflow-watch", "tools/ci/commands/check-release-deps", + "tools/ci/commands/llm-benchmark-summary", "tools/ci/common", "tools/keynote-bench-harness", "tools/license-check", diff --git a/tools/ci/commands/llm-benchmark-summary/Cargo.toml b/tools/ci/commands/llm-benchmark-summary/Cargo.toml new file mode 100644 index 00000000000..5f304c2b8b1 --- /dev/null +++ b/tools/ci/commands/llm-benchmark-summary/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci-llm-benchmark-summary" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/tools/ci/commands/llm-benchmark-summary/src/main.rs b/tools/ci/commands/llm-benchmark-summary/src/main.rs new file mode 100644 index 00000000000..d7ae1dc4f9f --- /dev/null +++ b/tools/ci/commands/llm-benchmark-summary/src/main.rs @@ -0,0 +1,309 @@ +use anyhow::{Context, Result}; +use clap::Parser; +use serde_json::{json, Value}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +const GREEN: u32 = 0x57F287; +const YELLOW: u32 = 0xFEE75C; +const RED: u32 = 0xED4245; +const MAX_ITEMS: usize = 5; +const MAX_FIELD_LENGTH: usize = 1000; + +#[derive(Parser)] +#[command(about = "Build a Discord embed from LLM benchmark analysis reports.")] +struct Cli { + #[arg(long)] + reports_dir: PathBuf, + #[arg(long)] + run_url: String, + #[arg(long)] + run_label: String, + #[arg(long)] + output: PathBuf, +} + +struct Report { + language: String, + passed_tasks: u64, + total_tasks: u64, + actions: Vec, + other_findings: Vec, +} + +fn field<'a>(text: &'a str, name: &str) -> &'a str { + let prefix = format!("- {name}: "); + text.lines() + .find_map(|line| line.strip_prefix(&prefix)) + .unwrap_or("unknown") +} + +fn language_name(language: &str) -> &str { + match language { + "csharp" => "C#", + "rust" => "Rust", + "typescript" => "TypeScript", + other => other, + } +} + +fn parse_report(text: &str) -> Report { + let language = field(text, "Language"); + let mode = field(text, "Mode"); + let model = field(text, "Model"); + let counts = field(text, "Tasks").split_whitespace().next().unwrap_or_default(); + let (passed_tasks, total_tasks) = counts + .split_once('/') + .and_then(|(passed, total)| Some((passed.parse().ok()?, total.parse().ok()?))) + .unwrap_or((0, 0)); + let mut report = Report { + language: language.to_owned(), + passed_tasks, + total_tasks, + actions: Vec::new(), + other_findings: Vec::new(), + }; + let language = language_name(language); + let mut section = ""; + let mut title = ""; + for line in text.lines() { + if let Some(heading) = line.strip_prefix("## ") { + section = heading; + title = ""; + } else if section == "Recommended actions" && line.starts_with("- **[") { + report.actions.push(format!("- {language} / {mode}: {}", &line[2..])); + } else if section == "Failure patterns" { + if let Some(heading) = line.strip_prefix("### ") { + title = heading + .rsplit_once(" (") + .filter(|(_, suffix)| { + suffix + .strip_suffix(" tasks)") + .or_else(|| suffix.strip_suffix(" task)")) + .is_some_and(|count| count.parse::().is_ok()) + }) + .map_or(heading, |(title, _)| title); + } else if let Some(classification) = line.strip_prefix("- **Classification:** ") + && !title.is_empty() + && matches!( + classification, + "Model limitation" | "Infrastructure/provider problem" | "No action" + ) + { + report + .other_findings + .push(format!("- {language} / {mode} / {model}: {title} - {classification}")); + } + } + } + report +} + +fn load_reports(directory: &Path) -> Result> { + let entries = match fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error).with_context(|| format!("reading reports in {}", directory.display())), + }; + let mut entries = entries.collect::>>()?; + entries.sort_by_key(|entry| entry.path()); + let mut reports = Vec::new(); + for entry in entries { + let path = entry.path(); + let kind = entry.file_type()?; + if kind.is_dir() { + reports.extend(load_reports(&path)?); + } else if kind.is_file() && path.extension().is_some_and(|extension| extension == "md") { + let text = fs::read_to_string(&path).with_context(|| format!("reading report {}", path.display()))?; + reports.push(parse_report(&text)); + } + } + Ok(reports) +} + +fn rate(passed: u64, total: u64) -> String { + let percent = if total == 0 { + 0.0 + } else { + passed as f64 * 100.0 / total as f64 + }; + format!("{passed}/{total} ({percent:.1}%)") +} + +fn field_value(lines: &[String], empty: &str, overflow_label: &str) -> String { + let mut seen = BTreeSet::new(); + let unique: Vec<_> = lines.iter().filter(|line| seen.insert(line.as_str())).collect(); + let mut value = if unique.is_empty() { + empty.to_owned() + } else { + unique + .iter() + .take(MAX_ITEMS) + .map(|line| line.as_str()) + .collect::>() + .join("\n") + }; + if unique.len() > MAX_ITEMS { + value.push_str(&format!( + "\n- ...and {} more {overflow_label}(s)", + unique.len() - MAX_ITEMS + )); + } + // Discord counts UTF-16 units; keep Unicode text intact when limiting a field. + if value.encode_utf16().count() > MAX_FIELD_LENGTH { + let suffix = "\n... View the full analysis."; + let mut length = 0; + value = value + .chars() + .take_while(|ch| { + length += ch.len_utf16(); + length <= MAX_FIELD_LENGTH - suffix.len() + }) + .collect(); + value.truncate(value.trim_end().len()); + value.push_str(suffix); + } + value +} + +fn build_payload(reports: &[Report], run_url: &str, run_label: &str) -> Value { + let (mut passed, mut total) = (0, 0); + let mut by_language = BTreeMap::<&str, (u64, u64)>::new(); + let mut actions = Vec::new(); + let mut other_findings = Vec::new(); + for report in reports { + passed += report.passed_tasks; + total += report.total_tasks; + let counts = by_language.entry(&report.language).or_default(); + counts.0 += report.passed_tasks; + counts.1 += report.total_tasks; + actions.extend(report.actions.iter().cloned()); + other_findings.extend(report.other_findings.iter().cloned()); + } + let percent = if total == 0 { + 0.0 + } else { + passed as f64 * 100.0 / total as f64 + }; + let color = if total == 0 + || percent < 90.0 + || other_findings + .iter() + .any(|item| item.contains("Infrastructure/provider problem")) + { + RED + } else if !actions.is_empty() || percent < 95.0 { + YELLOW + } else { + GREEN + }; + let language_rates: Vec<_> = by_language + .into_iter() + .map(|(language, (passed, total))| format!("- **{}:** {}", language_name(language), rate(passed, total))) + .collect(); + json!({ + "username": "SpacetimeDB LLM Benchmarks", + "allowed_mentions": {"parse": []}, + "embeds": [{ + "title": "LLM Benchmark Analysis", + "url": run_url, + "description": format!("**{}** task runs passed", rate(passed, total)), + "color": color, + "fields": [ + {"name": "By language", "value": field_value(&language_rates, "No analysis reports were produced.", "language"), "inline": false}, + {"name": "Action items", "value": field_value(&actions, "None", "action"), "inline": false}, + {"name": "Other failures", "value": field_value(&other_findings, "None", "finding"), "inline": false} + ], + "footer": {"text": run_label} + }] + }) +} + +fn main() -> Result<()> { + let args = Cli::parse(); + let reports = load_reports(&args.reports_dir)?; + let payload = build_payload(&reports, &args.run_url, &args.run_label); + fs::write(&args.output, serde_json::to_vec(&payload)?) + .with_context(|| format!("writing Discord payload {}", args.output.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn report(language: &str, tasks: &str, action: &str, classification: &str) -> Report { + parse_report(&format!( + "# LLM Benchmark Analysis\n\n- Language: {language}\n- Mode: guidelines\n- Model: test-model\n- Tasks: {tasks}\n- Scorers: 100/100 (100.0%)\n\n## Recommended actions\n\n{action}\n\n## Failure patterns\n\n### Incorrect sum-type syntax (1 task)\n\n- **Classification:** {classification}\n" + )) + } + + #[test] + fn aggregates_rates_and_formats_findings() { + let reports = [ + report("csharp", "36/37 (97.3%)", "", ""), + report("rust", "34/37 (91.9%)", "", "Model limitation"), + ]; + let payload = build_payload(&reports, "https://example.com/run", "Weekly run"); + let embed = &payload["embeds"][0]; + assert_eq!(embed["description"], "**70/74 (94.6%)** task runs passed"); + assert_eq!(embed["color"], YELLOW); + assert_eq!( + embed["fields"][0]["value"], + "- **C#:** 36/37 (97.3%)\n- **Rust:** 34/37 (91.9%)" + ); + assert_eq!( + embed["fields"][2]["value"], + "- Rust / guidelines / test-model: Incorrect sum-type syntax - Model limitation" + ); + assert_eq!(payload["allowed_mentions"], json!({"parse": []})); + assert_eq!(embed["url"], "https://example.com/run"); + assert_eq!(embed["footer"]["text"], "Weekly run"); + } + + #[test] + fn colors_reflect_pass_rate_actions_and_infrastructure() { + for (tasks, action, classification, color) in [ + ("37/37", "", "", GREEN), + ("35/37", "", "Model limitation", YELLOW), + ("30/37", "", "Model limitation", RED), + ( + "37/37", + "- **[Skill problem | High] Clarify transactions** — Update the skill. Evidence: t_075.", + "Skill problem", + YELLOW, + ), + ("37/37", "", "Infrastructure/provider problem", RED), + ] { + let payload = build_payload(&[report("csharp", tasks, action, classification)], "", ""); + assert_eq!(payload["embeds"][0]["color"], color); + if !action.is_empty() { + assert_eq!( + payload["embeds"][0]["fields"][1]["value"], + format!("- C# / guidelines: {}", &action[2..]) + ); + assert_eq!(payload["embeds"][0]["fields"][2]["value"], "None"); + } + } + } + + #[test] + fn empty_and_long_fields_stay_valid() { + let payload = build_payload(&[], "", ""); + assert_eq!(payload["embeds"][0]["color"], RED); + assert_eq!( + payload["embeds"][0]["fields"][0]["value"], + "No analysis reports were produced." + ); + let lines: Vec<_> = (0..6).map(|i| format!("- {i} {}", "🦀".repeat(400))).collect(); + let value = field_value(&lines, "None", "finding"); + assert!(value.encode_utf16().count() <= MAX_FIELD_LENGTH); + assert!(value.ends_with("... View the full analysis.")); + let mut lines: Vec<_> = (0..6).map(|i| format!("- {i}")).collect(); + lines.insert(1, "- 0".to_owned()); + assert_eq!( + field_value(&lines, "None", "finding"), + "- 0\n- 1\n- 2\n- 3\n- 4\n- ...and 1 more finding(s)" + ); + } +} diff --git a/tools/ci/commands/llm-benchmark-summary/tests/cli.rs b/tools/ci/commands/llm-benchmark-summary/tests/cli.rs new file mode 100644 index 00000000000..42d96ca3d72 --- /dev/null +++ b/tools/ci/commands/llm-benchmark-summary/tests/cli.rs @@ -0,0 +1,67 @@ +use serde_json::Value; +use std::fs; +use std::process::Command; + +#[test] +fn formats_downloaded_reports_and_reports_io_errors() { + let root = std::env::temp_dir().join(format!( + "llm-summary-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let reports = root.join("reports"); + fs::create_dir_all(&root).unwrap(); + let output = root.join("payload.json"); + let run = || { + Command::new(env!("CARGO_BIN_EXE_ci-llm-benchmark-summary")) + .arg("--reports-dir") + .arg(&reports) + .args(["--run-url", "https://example.com/run", "--run-label", "Manual run"]) + .arg("--output") + .arg(&output) + .output() + .unwrap() + }; + let payload = || serde_json::from_slice::(&fs::read(&output).unwrap()).unwrap(); + + // A failed benchmark job can leave no report artifact to download. + assert!(run().status.success()); + assert_eq!(payload()["embeds"][0]["description"], "**0/0 (0.0%)** task runs passed"); + + fs::create_dir_all(reports.join("nested")).unwrap(); + fs::write( + reports.join("nested").join("typescript.md"), + "# LLM Benchmark Analysis\r\n- Language: typescript\r\n- Mode: guidelines\r\n- Model: test-model\r\n- Tasks: 2/3 (66.7%)\r\n\r\n## Recommended actions\r\n\r\nNo repository changes recommended.\r\n\r\n## Failure patterns\r\n\r\n### API usage (1 task)\r\n\r\n- **Classification:** Model limitation\r\n", + ) + .unwrap(); + fs::write(reports.join("rust.md"), "- Language: rust\n- Tasks: 1/1 (100.0%)\n").unwrap(); + fs::write(reports.join("ignored.json"), "not a report").unwrap(); + assert!(run().status.success()); + let result = payload(); + let embed = &result["embeds"][0]; + assert_eq!(embed["description"], "**3/4 (75.0%)** task runs passed"); + assert_eq!( + embed["fields"][0]["value"], + "- **Rust:** 1/1 (100.0%)\n- **TypeScript:** 2/3 (66.7%)" + ); + assert_eq!( + embed["fields"][2]["value"], + "- TypeScript / guidelines / test-model: API usage - Model limitation" + ); + + let broken = reports.join("broken.md"); + fs::write(&broken, [0xff]).unwrap(); + let failure = run(); + assert!(!failure.status.success()); + assert!(String::from_utf8_lossy(&failure.stderr).contains("broken.md")); + fs::remove_file(&broken).unwrap(); + fs::remove_file(&output).unwrap(); + fs::create_dir(&output).unwrap(); + let failure = run(); + assert!(!failure.status.success()); + assert!(String::from_utf8_lossy(&failure.stderr).contains("writing Discord payload")); + fs::remove_dir_all(root).unwrap(); +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 4859363bc7c..2be03751cc1 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -88,6 +88,10 @@ const COMMANDS: &[Command] = &[ path: &["other-workflows", "check-release-deps"], package: "ci-check-release-deps", }, + Command { + path: &["other-workflows", "llm-benchmark-summary"], + package: "ci-llm-benchmark-summary", + }, ]; fn print_help() { diff --git a/tools/xtask-llm-benchmark/src/api/client.rs b/tools/xtask-llm-benchmark/src/api/client.rs index 0b43ccb5bac..2c1b0cb6cb3 100644 --- a/tools/xtask-llm-benchmark/src/api/client.rs +++ b/tools/xtask-llm-benchmark/src/api/client.rs @@ -256,9 +256,10 @@ impl ApiClient { } let task_name = task_entry.file_name().to_string_lossy().to_string(); - // Humanize task_name for title let title = task_name - .trim_start_matches(|c: char| c == 't' || c == '_' || c.is_ascii_digit()) + .strip_prefix("t_") + .and_then(|name| name.split_once('_')) + .map_or(task_name.as_str(), |(_, name)| name) .replace('_', " ") .trim() .to_string(); diff --git a/tools/xtask-llm-benchmark/src/bench/analysis.rs b/tools/xtask-llm-benchmark/src/bench/analysis.rs index cb23fbb6cf5..2b2f014b772 100644 --- a/tools/xtask-llm-benchmark/src/bench/analysis.rs +++ b/tools/xtask-llm-benchmark/src/bench/analysis.rs @@ -7,11 +7,14 @@ use anyhow::Result; use spacetimedb_data_structures::map::HashMap; use std::path::Path; +const MAX_CONTEXT_CHARS: usize = 20_000; + pub async fn run_analysis( outcomes: &[RunOutcome], lang: &str, mode: &str, model_name: &str, + context: &str, bench_root: &Path, llm: &dyn LlmProvider, ) -> Result> { @@ -24,7 +27,7 @@ pub async fn run_analysis( return Ok(None); } - let prompt = build_prompt(lang, mode, model_name, bench_root, &failures); + let prompt = build_prompt(lang, mode, model_name, context, bench_root, &failures); let route = ModelRoute::new( "gpt-5.4-mini", @@ -49,9 +52,10 @@ pub fn system_prompt() -> String { } pub const SYSTEM_PROMPT: &str = "\ -You summarize LLM benchmark failures for SpacetimeDB into structured markdown. \ +You turn LLM benchmark failures for SpacetimeDB into evidence-based, structured markdown. \ Each failure includes the model's generated code, the scorer error, and the golden (correct) answer when available. \ -Write in third person for a public benchmark page. Do not address the reader."; +Write in third person for a public benchmark page. Do not address the reader. \ +Recommend repository changes only when the supplied evidence supports them."; fn context_description(mode: &str) -> &'static str { match mode { @@ -103,7 +107,14 @@ fn read_golden(bench_root: &Path, task_id: &str, lang: &str) -> Option { None } -pub fn build_prompt(lang: &str, mode: &str, model_name: &str, bench_root: &Path, failures: &[&RunOutcome]) -> String { +pub fn build_prompt( + lang: &str, + mode: &str, + model_name: &str, + context: &str, + bench_root: &Path, + failures: &[&RunOutcome], +) -> String { let lang_display = match lang { "rust" => "Rust", "csharp" => "C#", @@ -118,6 +129,16 @@ pub fn build_prompt(lang: &str, mode: &str, model_name: &str, bench_root: &Path, count = failures.len(), ); + let context_is_complete = context.chars().count() <= MAX_CONTEXT_CHARS; + if has_context(mode) { + prompt.push_str("### Context supplied to the model\n\n"); + prompt.push_str(&format!("```\n{}\n```\n", truncate(context, MAX_CONTEXT_CHARS))); + if !context_is_complete { + prompt.push_str("The context excerpt was truncated. Treat context-gap conclusions as low confidence.\n"); + } + prompt.push('\n'); + } + for f in failures.iter().take(15) { prompt.push_str(&format!("### {} ({}/{})\n", f.task, f.passed_tests, f.total_tests)); @@ -141,40 +162,177 @@ pub fn build_prompt(lang: &str, mode: &str, model_name: &str, bench_root: &Path, prompt.push_str(&format!("({} more failures not shown)\n\n", failures.len() - 15)); } - prompt.push_str(&analysis_instructions(mode)); + prompt.push_str(&analysis_instructions_with_context(mode, context_is_complete)); prompt } pub fn analysis_instructions(mode: &str) -> String { - let fix_line = if has_context(mode) { + analysis_instructions_with_context(mode, false) +} + +fn analysis_instructions_with_context(mode: &str, context_is_complete: bool) -> String { + let context_gap_line = if has_context(mode) { let name = context_name(mode); - format!("5. **{name} gap:** What's missing or unclear in the {name} that led to this mistake\n") + if context_is_complete { + format!("- **{name} gap:** What is missing or unclear in the {name}, or `None`\n") + } else { + format!("- **{name} gap:** `Needs manual review`; the complete {name} was not supplied\n") + } } else { String::new() }; + let context_rule = if has_context(mode) && !context_is_complete { + "- Because the complete context was not supplied, do not classify a group as `Skill problem` or `Documentation problem`.\n" + } else { + "" + }; format!( "\ --- -Group failures by root cause pattern. Use this exact structure for each group: +Begin with this section: + +## Recommended actions + +List at most five distinct, repository-owned actions supported by the evidence. Use: + +- **[Classification | Confidence] Short title** — Plain-language action. Evidence: task IDs. + +If the failures are isolated model mistakes, provider failures, or otherwise do not justify a repository change, write: +`No repository changes recommended.` + +Then group failures by root cause using this exact structure: + +## Failure patterns ### [Pattern Name] (N tasks) -1. **What the model wrote:** Show the relevant incorrect lines from the generated code -2. **What was expected:** Show the relevant lines from the golden answer -3. **What the error says:** Quote the scorer error that identifies the problem -4. **Why this happened:** Why the model likely made this mistake (e.g. confused with another framework, hallucinated API, singular vs plural naming) -5. **Affected tasks:** list of task IDs -{fix_line} +- **Classification:** One of `Eval problem`, `Skill problem`, `Documentation problem`, `API/ergonomics problem`, `Model limitation`, `Infrastructure/provider problem`, or `No action` +- **Confidence:** `High`, `Medium`, or `Low` +- **What the model wrote:** Relevant incorrect lines from the generated code +- **What was expected:** Relevant lines from the golden answer +- **What the error says:** The scorer error that identifies the problem +- **Why this happened:** The likely root cause +- **Suggested action:** A plain-language repository change, or `None` +- **Suggested area:** The likely repository area, or `None` +- **Affected tasks:** Task IDs +{context_gap_line} Rules: - Group tasks that fail for the same reason. Do not repeat the same analysis per task. - Show only the relevant lines, not entire files. - Skip provider errors (timeouts, 429s) with a brief note. +- Do not recommend changing an eval, skill, documentation, or API merely because one model made an isolated mistake. +- Do not invent evidence, repository paths, or implementation details that were not supplied. +- Prefer `Model limitation`, `Infrastructure/provider problem`, or `No action` when the evidence does not support a repository change. +- Classify a context gap as `Skill problem` or `Documentation problem` only when the supplied context directly supports that conclusion. +- `Eval problem`, `Skill problem`, `Documentation problem`, and `API/ergonomics problem` require a non-`None` suggested action and a matching entry under `Recommended actions`. +- When `Suggested action` is `None`, use `Model limitation`, `Infrastructure/provider problem`, or `No action`. +- Write `No repository changes recommended.` only when no failure group recommends a repository change. +{context_rule}\ " ) } +pub fn build_report( + outcomes: &[RunOutcome], + lang: &str, + mode: &str, + model_name: &str, + analysis: Option<&str>, +) -> String { + let passed_tasks = outcomes + .iter() + .filter(|outcome| outcome.total_tests > 0 && outcome.passed_tests == outcome.total_tests) + .count(); + let total_tasks = outcomes.len(); + let passed_scorers: u32 = outcomes.iter().map(|outcome| outcome.passed_tests).sum(); + let total_scorers: u32 = outcomes.iter().map(|outcome| outcome.total_tests).sum(); + let failures_without_output: Vec<&str> = outcomes + .iter() + .filter(|outcome| outcome.passed_tests < outcome.total_tests && outcome.llm_output.is_none()) + .map(|outcome| outcome.task.as_str()) + .collect(); + let unavailable_output_section = if failures_without_output.is_empty() { + None + } else { + let task_count = failures_without_output.len(); + let task_label = if task_count == 1 { "task" } else { "tasks" }; + let tasks = failures_without_output + .iter() + .map(|task| format!("`{task}`")) + .collect::>() + .join(", "); + Some(format!( + "\ +### Model output unavailable ({task_count} {task_label}) + +- **Classification:** Infrastructure/provider problem +- **Tasks:** {tasks} +- **What happened:** The model request failed before producing output, so source-level failure analysis was not possible. +- **Suggested action:** Retry the affected tasks and inspect the benchmark logs if the failure persists." + )) + }; + + let mut report = format!( + "\ +# LLM Benchmark Analysis + +- Language: {lang} +- Mode: {mode} +- Model: {model_name} +- Tasks: {passed_tasks}/{total_tasks} ({task_percent:.1}%) +- Scorers: {passed_scorers}/{total_scorers} ({scorer_percent:.1}%) + +", + task_percent = percent(passed_tasks as u32, total_tasks as u32), + scorer_percent = percent(passed_scorers, total_scorers), + ); + + match analysis { + Some(analysis) => { + report.push_str(analysis.trim()); + if let Some(section) = unavailable_output_section { + report.push_str("\n\n"); + report.push_str(§ion); + } + } + None if unavailable_output_section.is_some() => { + report.push_str(&format!( + "\ +## Recommended actions + +No repository changes recommended. + +## Failure patterns + +{}", + unavailable_output_section.unwrap() + )); + } + None => report.push_str( + "\ +## Recommended actions + +No repository changes recommended. + +## Failure patterns + +No failures detected.", + ), + } + report.push('\n'); + report +} + +fn percent(passed: u32, total: u32) -> f64 { + if total == 0 { + 0.0 + } else { + f64::from(passed) * 100.0 / f64::from(total) + } +} + fn extract_reasons(details: &HashMap) -> Vec { details .iter() @@ -192,3 +350,137 @@ fn truncate(s: &str, max: usize) -> &str { None => s, } } + +#[cfg(test)] +mod tests { + use super::*; + + fn outcome(task: &str, passed_tests: u32, total_tests: u32) -> RunOutcome { + RunOutcome { + hash: String::new(), + task: task.to_string(), + lang: "typescript".to_string(), + golden_published: true, + model_name: "test-model".to_string(), + total_tests, + passed_tests, + llm_output: None, + category: None, + route_api_model: None, + golden_db: None, + llm_db: None, + work_dir_golden: None, + work_dir_llm: None, + scorer_details: None, + vendor: String::new(), + input_tokens: None, + output_tokens: None, + generation_duration_ms: None, + started_at: None, + finished_at: None, + } + } + + #[test] + fn instructions_request_actionable_evidence_based_output() { + let instructions = analysis_instructions("guidelines"); + + assert!(instructions.contains("## Recommended actions")); + assert!(instructions.contains("Eval problem")); + assert!(instructions.contains("Skill problem")); + assert!(instructions.contains("Model limitation")); + assert!(instructions.contains("Do not invent evidence")); + assert!(instructions.contains("AI guidelines gap")); + assert!(instructions.contains("Needs manual review")); + assert!(instructions.contains("do not classify a group as `Skill problem` or `Documentation problem`")); + } + + #[test] + fn no_context_does_not_request_a_context_gap() { + let instructions = analysis_instructions("no_context"); + + assert!(!instructions.contains("context gap:")); + assert!(!instructions.contains("guidelines gap:")); + } + + #[test] + fn report_includes_task_and_scorer_pass_rates() { + let outcomes = vec![outcome("t_001", 3, 3), outcome("t_002", 1, 2), outcome("t_003", 0, 0)]; + let report = build_report( + &outcomes, + "typescript", + "guidelines", + "test-model", + Some("## Recommended actions\n\n- Fix the skill."), + ); + + assert!(report.contains("- Tasks: 1/3 (33.3%)")); + assert!(report.contains("- Scorers: 4/5 (80.0%)")); + assert!(report.contains("- Fix the skill.")); + } + + #[test] + fn live_prompt_supplies_context_for_gap_analysis() { + let failed = outcome("t_001", 0, 1); + let prompt = build_prompt( + "typescript", + "guidelines", + "test-model", + "Use transactions for procedure writes.", + Path::new("missing-benchmark-root"), + &[&failed], + ); + + assert!(prompt.contains("### Context supplied to the model")); + assert!(prompt.contains("Use transactions for procedure writes.")); + assert!(prompt.contains("What is missing or unclear in the AI guidelines")); + assert!(!prompt.contains("complete AI guidelines was not supplied")); + } + + #[test] + fn passing_report_recommends_no_changes() { + let report = build_report( + &[outcome("t_001", 3, 3)], + "typescript", + "guidelines", + "test-model", + None, + ); + + assert!(report.contains("No repository changes recommended.")); + assert!(report.contains("No failures detected.")); + } + + #[test] + fn failed_request_without_output_is_reported_as_infrastructure() { + let report = build_report( + &[outcome("t_001", 0, 1)], + "typescript", + "guidelines", + "test-model", + None, + ); + + assert!(report.contains("Model output unavailable (1 task)")); + assert!(report.contains("Infrastructure/provider problem")); + assert!(report.contains("`t_001`")); + assert!(!report.contains("No failures detected.")); + } + + #[test] + fn failed_request_without_output_is_added_to_model_analysis() { + let mut generated_failure = outcome("t_001", 0, 1); + generated_failure.llm_output = Some("generated source".to_string()); + let report = build_report( + &[generated_failure, outcome("t_002", 0, 1)], + "typescript", + "guidelines", + "test-model", + Some("## Recommended actions\n\nNo repository changes recommended.\n\n## Failure patterns\n\n### Invalid API"), + ); + + assert!(report.contains("### Invalid API")); + assert!(report.contains("Model output unavailable (1 task)")); + assert!(report.contains("`t_002`")); + } +} diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index e6e05bbefec..ad6131a54bb 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -744,6 +744,9 @@ impl Publisher for SpacetimeRustPublisher { .current_dir(source); if let Some(target_dir) = env::var_os("LLM_BENCH_RUST_TARGET_DIR") { pubcmd.env("CARGO_TARGET_DIR", target_dir); + } else { + // Generated modules share a crate name, so they cannot safely share the workspace target directory. + pubcmd.env_remove("CARGO_TARGET_DIR"); } run(&mut pubcmd, "spacetime publish")?; diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index fc6da3884af..f993f80c883 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -522,12 +522,28 @@ fn dry_run_analysis_path(run_id: &str, lang_name: &str, mode: &str, route: &Mode .join(format!("run-{run_id}-{lang_name}-{mode}-{route_tag}-analysis.md")) } +fn configured_analysis_path(lang_name: &str, mode: &str, route: &ModelRoute) -> Option { + let report_dir = std::env::var_os("LLM_BENCHMARK_REPORT_DIR").filter(|value| !value.is_empty())?; + let route_tag = sanitize_db_name(&route.display_name); + Some(PathBuf::from(report_dir).join(format!("{lang_name}-{mode}-{route_tag}.md"))) +} + +fn write_analysis_report( + path: &Path, + cfg: &BenchRunContext<'_>, + outcomes: &[RunOutcome], + analysis: Option<&str>, +) -> Result<()> { + fs::create_dir_all(path.parent().unwrap_or_else(|| Path::new(".")))?; + let report = + crate::bench::analysis::build_report(outcomes, cfg.lang.as_str(), cfg.mode, &cfg.route.display_name, analysis); + fs::write(path, report)?; + Ok(()) +} + async fn maybe_generate_analysis(cfg: &BenchRunContext<'_>, outcomes: &[RunOutcome]) -> Result> { - let should_run = if cfg.dry_run { - cfg.local_analysis - } else { - cfg.api_client.is_some() - }; + let configured_path = configured_analysis_path(cfg.lang.as_str(), cfg.mode, cfg.route); + let should_run = cfg.local_analysis || configured_path.is_some() || (!cfg.dry_run && cfg.api_client.is_some()); if !should_run { return Ok(None); @@ -538,27 +554,30 @@ async fn maybe_generate_analysis(cfg: &BenchRunContext<'_>, outcomes: &[RunOutco cfg.lang.as_str(), cfg.mode, &cfg.route.display_name, + cfg.context, cfg.bench_root, cfg.llm, ) .await?; if cfg.dry_run - && let (Some(text), Some(run_id)) = (analysis.as_deref(), cfg.dry_run_id.as_deref()) + && let Some(run_id) = cfg.dry_run_id.as_deref() { let path = dry_run_analysis_path(run_id, cfg.lang.as_str(), cfg.mode, cfg.route); - let _ = fs::create_dir_all(path.parent().unwrap_or_else(|| Path::new("."))); - let contents = format!( - "# Local Benchmark Analysis\n\n- Lang: {}\n- Mode: {}\n- Model: {}\n\n{}", - cfg.lang.as_str(), - cfg.mode, - cfg.route.display_name, - text - ); - match fs::write(&path, contents) { + match write_analysis_report(&path, cfg, outcomes, analysis.as_deref()) { Ok(()) => println!("Local analysis: {}", path.display()), Err(e) => eprintln!("[warn] failed to write local analysis: {e}"), } + } + + if let Some(path) = configured_path { + match write_analysis_report(&path, cfg, outcomes, analysis.as_deref()) { + Ok(()) => println!("Analysis report: {}", path.display()), + Err(e) => eprintln!("[warn] failed to write analysis report: {e}"), + } + } + + if cfg.dry_run { return Ok(None); } diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index be76423286d..fae506b4813 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -21,7 +21,9 @@ use xtask_llm_benchmark::context::constants::ALL_MODES; use xtask_llm_benchmark::context::{build_context, compute_processed_context_hash}; use xtask_llm_benchmark::eval::Lang; use xtask_llm_benchmark::llm::types::Vendor; -use xtask_llm_benchmark::llm::{default_model_routes, make_provider_from_env, LlmProvider, ModelRoute}; +use xtask_llm_benchmark::llm::{ + default_model_routes, make_provider_from_env, LlmProvider, ModelRoute, ReasoningEffort, +}; #[derive(Clone, Debug)] struct ModelGroup { @@ -68,6 +70,10 @@ impl std::str::FromStr for ModelGroup { after_help = "Notes:\n • Anthropic ids: claude-sonnet-4-5, claude-sonnet-4, claude-3-7-sonnet-latest, claude-3-5-sonnet-latest\n • Base URLs must not include /v1; models must be valid for the chosen provider.\n" )] struct Cli { + /// Reasoning effort used for every model request + #[arg(long, value_enum, default_value_t = ReasoningEffort::Medium, global = true)] + reasoning: ReasoningEffort, + #[command(subcommand)] command: Commands, } @@ -139,6 +145,10 @@ struct RunArgs { #[arg(long)] dry_run: bool, + /// Skip uploading the benchmark task catalog + #[arg(long)] + skip_task_catalog_upload: bool, + /// When used with --dry-run, also generate local markdown analysis files #[arg(long, requires = "dry_run")] local_analysis: bool, @@ -215,20 +225,20 @@ fn main() -> Result<()> { let cli = Cli::parse(); match cli.command { - Commands::Run(args) => cmd_run(args), - Commands::Analyze(args) => cmd_analyze(args), + Commands::Run(args) => cmd_run(args, cli.reasoning), + Commands::Analyze(args) => cmd_analyze(args, cli.reasoning), } } /* ------------------------------ run ------------------------------ */ -fn cmd_run(args: RunArgs) -> Result<()> { - run_benchmarks(args)?; +fn cmd_run(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { + run_benchmarks(args, reasoning)?; Ok(()) } /// Core benchmark runner used by both `run` and `ci-quickfix` -fn run_benchmarks(args: RunArgs) -> Result<()> { +fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { let dry_run = args.dry_run; let local_analysis = args.local_analysis; let dry_run_id = dry_run.then(|| { @@ -281,7 +291,8 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { let bench_root = find_bench_root(); // Upload task catalog before running benchmarks - if let Some(ref api) = upload_client + if !args.skip_task_catalog_upload + && let Some(ref api) = upload_client && let Err(e) = api.upload_task_catalog(&bench_root) { eprintln!("[warn] failed to upload task catalog: {e}"); @@ -314,7 +325,7 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { } let llm_provider = if !config.goldens_only && !config.hash_only { - let provider = make_provider_from_env()?; + let provider = make_provider_from_env(reasoning)?; let rt = runtime.as_ref().expect("failed to initialize runtime for preflight"); let routes = filter_routes(&config); preflight_llm_routes(rt, provider.as_ref(), &routes, &modes)?; @@ -378,7 +389,7 @@ fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { /* ------------------------------ analyze ------------------------------ */ -fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { +fn cmd_analyze(args: AnalyzeArgs, reasoning: ReasoningEffort) -> Result<()> { let api = ApiClient::from_env() .context("failed to initialize API client")? .context("LLM_BENCHMARK_UPLOAD_URL required for analyze")?; @@ -434,7 +445,7 @@ fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { // Initialize LLM provider for analysis let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; - let provider = make_provider_from_env()?; + let provider = make_provider_from_env(reasoning)?; let analysis_route = ModelRoute::new( "gpt-5.4-mini", @@ -964,6 +975,7 @@ mod tests { models: None, model_source: ModelSource::Static, dry_run: false, + skip_task_catalog_upload: false, local_analysis: false, route_overrides: None, } @@ -989,6 +1001,24 @@ mod tests { } } + #[test] + fn reasoning_defaults_to_medium_and_accepts_an_override() { + let default = Cli::try_parse_from(["llm", "run", "--hash-only"]).unwrap(); + assert_eq!(default.reasoning, ReasoningEffort::Medium); + + let overridden = Cli::try_parse_from(["llm", "run", "--hash-only", "--reasoning", "high"]).unwrap(); + assert_eq!(overridden.reasoning, ReasoningEffort::High); + } + + #[test] + fn task_catalog_upload_can_be_skipped() { + let cli = Cli::try_parse_from(["llm", "run", "--hash-only", "--skip-task-catalog-upload"]).unwrap(); + let Commands::Run(args) = cli.command else { + panic!("expected run command"); + }; + assert!(args.skip_task_catalog_upload); + } + #[test] fn explicit_models_bypass_remote_model_source() { let mut args = base_run_args(); diff --git a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs index 8bb0d1ac734..b8bd6cd55e6 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs @@ -4,7 +4,7 @@ use crate::llm::segmentation::{ anthropic_ctx_limit_tokens, build_anthropic_messages, desired_output_tokens, deterministic_trim_prefix, estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; use anyhow::{anyhow, bail, Context, Result}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue, CONTENT_TYPE}; use reqwest::{Client, StatusCode}; @@ -30,7 +30,7 @@ impl AnthropicClient { format!("{}/v1/messages", self.base.trim_end_matches('/')) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let system = prompt.system.clone(); let segs = prompt.segments.clone(); let mut static_prefix = prompt.static_prefix.clone().unwrap_or_default(); @@ -73,13 +73,25 @@ impl AnthropicClient { struct Req { model: String, max_tokens: u32, + thinking: ThinkingConfig, + output_config: OutputConfig, #[serde(skip_serializing_if = "Option::is_none")] system: Option, messages: Vec, } + #[derive(Serialize)] + struct ThinkingConfig { + r#type: &'static str, + } + #[derive(Serialize)] + struct OutputConfig { + effort: ReasoningEffort, + } let req = Req { model: model_norm.to_string(), max_tokens, + thinking: ThinkingConfig { r#type: "adaptive" }, + output_config: OutputConfig { effort: reasoning }, system: system_json, messages, }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs index 5b7bc7b8636..ccf3164d856 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs @@ -7,7 +7,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deepseek_ctx_limit_tokens, deterministic_trim_prefix, estimate_tokens, non_context_reserve_tokens_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[derive(Clone)] pub struct DeepSeekClient { @@ -21,7 +21,7 @@ impl DeepSeekClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); let system = prompt.system.clone(); @@ -47,6 +47,12 @@ impl DeepSeekClient { model: &'a str, messages: Vec>, temperature: f32, + thinking: ThinkingConfig, + reasoning_effort: ReasoningEffort, + } + #[derive(Serialize)] + struct ThinkingConfig { + r#type: &'static str, } #[derive(Serialize)] struct Msg<'a> { @@ -78,6 +84,8 @@ impl DeepSeekClient { model, messages, temperature: 0.0, + thinking: ThinkingConfig { r#type: "enabled" }, + reasoning_effort: reasoning, }; let auth = HttpClient::bearer(&self.api_key); diff --git a/tools/xtask-llm-benchmark/src/llm/clients/google.rs b/tools/xtask-llm-benchmark/src/llm/clients/google.rs index ad309a88477..f822be51e5f 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/google.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/google.rs @@ -8,7 +8,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, gemini_ctx_limit_tokens, non_context_reserve_tokens_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; /// Google uses API key in the query string rather than Authorization header. #[derive(Clone)] @@ -23,7 +23,7 @@ impl GoogleGeminiClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { // ---- Never trim system or dynamic segments ---- let system = prompt.system.clone(); let segs: Vec> = prompt.segments.clone(); @@ -49,10 +49,24 @@ impl GoogleGeminiClient { #[serde(skip_serializing_if = "Option::is_none")] system_instruction: Option>, contents: Vec>, + #[serde(rename = "generationConfig")] + generation_config: GenerationConfig, #[serde(skip_serializing_if = "Option::is_none")] safety_settings: Option>, } + #[derive(Serialize)] + struct GenerationConfig { + #[serde(rename = "thinkingConfig")] + thinking_config: ThinkingConfig, + } + + #[derive(Serialize)] + struct ThinkingConfig { + #[serde(rename = "thinkingLevel")] + thinking_level: ReasoningEffort, + } + #[derive(Serialize)] struct SystemInstruction<'a> { parts: [Part<'a>; 1], @@ -100,6 +114,11 @@ impl GoogleGeminiClient { let req = Req { system_instruction, contents, + generation_config: GenerationConfig { + thinking_config: ThinkingConfig { + thinking_level: reasoning, + }, + }, safety_settings: None, }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs index 05ea663019e..9bf4d87df9d 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs @@ -7,7 +7,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, meta_ctx_limit_tokens, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[derive(Clone)] pub struct MetaLlamaClient { @@ -22,7 +22,7 @@ impl MetaLlamaClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, _reasoning: ReasoningEffort) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); // Build input like other clients diff --git a/tools/xtask-llm-benchmark/src/llm/clients/mod.rs b/tools/xtask-llm-benchmark/src/llm/clients/mod.rs index 254fe5b8f63..604afed9c60 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/mod.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/mod.rs @@ -20,7 +20,7 @@ pub use openrouter::OpenRouterClient; pub use xai::XaiGrokClient; use crate::llm::prompt::BuiltPrompt; -use crate::llm::types::LlmOutput; +use crate::llm::types::{LlmOutput, ReasoningEffort}; #[derive(Debug, Clone)] pub struct ClientPreflight { @@ -51,7 +51,7 @@ pub trait LlmClient: Send + Sync { ))) } - async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result; + async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result; } macro_rules! impl_direct_llm_client { @@ -62,8 +62,13 @@ macro_rules! impl_direct_llm_client { $provider_name } - async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { - <$ty>::generate(self, model, prompt).await + async fn generate( + &self, + model: &str, + prompt: &BuiltPrompt, + reasoning: ReasoningEffort, + ) -> Result { + <$ty>::generate(self, model, prompt, reasoning).await } } }; @@ -87,7 +92,7 @@ impl LlmClient for OpenRouterClient { Ok(ClientPreflight::new(status.summary())) } - async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { - OpenRouterClient::generate(self, model, prompt).await + async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { + OpenRouterClient::generate(self, model, prompt, reasoning).await } } diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openai.rs b/tools/xtask-llm-benchmark/src/llm/clients/openai.rs index 9fed933d3fb..537b4a6b84a 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openai.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openai.rs @@ -4,7 +4,7 @@ use crate::llm::segmentation::{ build_openai_responses_input, deterministic_trim_prefix, estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, openai_ctx_limit_tokens, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; use anyhow::{bail, Context, Result}; use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; @@ -29,7 +29,7 @@ impl OpenAiClient { format!("{}/v1/responses", self.base.trim_end_matches('/')) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let system = prompt.system.clone(); let segs = prompt.segments.clone(); @@ -85,14 +85,21 @@ impl OpenAiClient { struct Req<'a> { model: &'a str, input: Vec, + reasoning: ReasoningConfig, #[serde(skip_serializing_if = "Option::is_none")] max_output_tokens: Option, } + #[derive(Serialize)] + struct ReasoningConfig { + effort: ReasoningEffort, + } + let url = self.responses_url(); let payload = Req { model, input, + reasoning: ReasoningConfig { effort: reasoning }, max_output_tokens: None, }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs index df4c062c1bb..c4e806666ee 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs @@ -8,7 +8,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; const OPENROUTER_BASE: &str = "https://openrouter.ai/api/v1"; @@ -162,7 +162,7 @@ impl OpenRouterClient { Ok(()) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); let system = prompt.system.clone(); @@ -183,6 +183,7 @@ impl OpenRouterClient { model: &'a str, messages: Vec>, temperature: f32, + reasoning: ReasoningConfig, #[serde(skip_serializing_if = "Option::is_none")] top_p: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -195,6 +196,11 @@ impl OpenRouterClient { content: &'a str, } + #[derive(Serialize)] + struct ReasoningConfig { + effort: ReasoningEffort, + } + let mut messages: Vec = Vec::new(); if let Some(sys) = system.as_deref() @@ -222,6 +228,7 @@ impl OpenRouterClient { model, messages, temperature: 0.0, + reasoning: ReasoningConfig { effort: reasoning }, top_p: None, max_tokens: output_token_limit_env().map(|limit| limit.max(1) as u32), }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/xai.rs b/tools/xtask-llm-benchmark/src/llm/clients/xai.rs index be345c82b32..46062c6d9bc 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/xai.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/xai.rs @@ -6,7 +6,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, non_context_reserve_tokens_env, xai_ctx_limit_tokens, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[derive(Clone)] pub struct XaiGrokClient { @@ -21,7 +21,7 @@ impl XaiGrokClient { } /// Uses BuiltPrompt (system, static_prefix, segments) and maps to xAI /chat/completions. - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let url = format!("{}/v1/chat/completions", self.base.trim_end_matches('/')); // Never trim system or dynamic segments @@ -41,6 +41,7 @@ impl XaiGrokClient { model: &'a str, messages: Vec>, temperature: f32, + reasoning_effort: ReasoningEffort, } #[derive(Serialize)] @@ -75,6 +76,7 @@ impl XaiGrokClient { model, messages, temperature: 0.0, + reasoning_effort: reasoning, }; let auth = HttpClient::bearer(&self.api_key); diff --git a/tools/xtask-llm-benchmark/src/llm/config.rs b/tools/xtask-llm-benchmark/src/llm/config.rs index 5eacdf40248..2e76dcbfdef 100644 --- a/tools/xtask-llm-benchmark/src/llm/config.rs +++ b/tools/xtask-llm-benchmark/src/llm/config.rs @@ -6,7 +6,7 @@ use crate::llm::clients::{ AnthropicClient, DeepSeekClient, GoogleGeminiClient, MetaLlamaClient, OpenAiClient, OpenRouterClient, XaiGrokClient, }; use crate::llm::provider::{LlmProvider, RouterProvider}; -use crate::llm::types::Vendor; +use crate::llm::types::{ReasoningEffort, Vendor}; fn force_vendor_from_env() -> Option { match env::var("LLM_VENDOR").ok().as_deref() { @@ -34,7 +34,7 @@ fn force_vendor_from_env() -> Option { /// When OPENROUTER_API_KEY is set, it acts as a fallback for any vendor that doesn't /// have its own direct API key configured. This means you can set just OPENROUTER_API_KEY /// to run all models through OpenRouter, or mix direct keys with OpenRouter fallback. -pub fn make_provider_from_env() -> Result> { +pub fn make_provider_from_env(reasoning: ReasoningEffort) -> Result> { let http = HttpClient::new()?; // Filter out empty strings so an empty env var falls through to OpenRouter. @@ -84,6 +84,8 @@ pub fn make_provider_from_env() -> Result> { let openrouter = openrouter_key.map(|k| OpenRouterClient::new(http.clone(), k)); let force = force_vendor_from_env(); - let router = RouterProvider::new(openai, anthropic, google, xai, deepseek, meta, openrouter, force); + let router = RouterProvider::new( + openai, anthropic, google, xai, deepseek, meta, openrouter, force, reasoning, + ); Ok(Arc::new(router)) } diff --git a/tools/xtask-llm-benchmark/src/llm/mod.rs b/tools/xtask-llm-benchmark/src/llm/mod.rs index 4b72185c760..9e6548ec55a 100644 --- a/tools/xtask-llm-benchmark/src/llm/mod.rs +++ b/tools/xtask-llm-benchmark/src/llm/mod.rs @@ -10,4 +10,4 @@ pub use config::make_provider_from_env; pub use model_routes::{default_model_routes, ModelRoute}; pub use prompt::PromptBuilder; pub use provider::{LlmProvider, RouterProvider}; -pub use types::LlmOutput; +pub use types::{LlmOutput, ReasoningEffort}; diff --git a/tools/xtask-llm-benchmark/src/llm/provider.rs b/tools/xtask-llm-benchmark/src/llm/provider.rs index 355f2e19a3e..1dba17764b9 100644 --- a/tools/xtask-llm-benchmark/src/llm/provider.rs +++ b/tools/xtask-llm-benchmark/src/llm/provider.rs @@ -8,7 +8,7 @@ use crate::llm::clients::{ }; use crate::llm::model_routes::ModelRoute; use crate::llm::prompt::BuiltPrompt; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[async_trait] pub trait LlmProvider: Send + Sync { @@ -19,6 +19,7 @@ pub trait LlmProvider: Send + Sync { pub struct RouterProvider { clients: HashMap>, pub force: Option, + reasoning: ReasoningEffort, } impl RouterProvider { @@ -32,6 +33,7 @@ impl RouterProvider { meta: Option, openrouter: Option, force: Option, + reasoning: ReasoningEffort, ) -> Self { let mut clients: HashMap> = HashMap::new(); @@ -57,7 +59,11 @@ impl RouterProvider { clients.insert(Vendor::OpenRouter, Box::new(client)); } - Self { clients, force } + Self { + clients, + force, + reasoning, + } } } @@ -105,7 +111,7 @@ impl LlmProvider for RouterProvider { ); } - resolved.client.generate(&resolved.model, prompt).await + resolved.client.generate(&resolved.model, prompt, self.reasoning).await } } diff --git a/tools/xtask-llm-benchmark/src/llm/types.rs b/tools/xtask-llm-benchmark/src/llm/types.rs index 5ccfd14f339..f83a45b3d81 100644 --- a/tools/xtask-llm-benchmark/src/llm/types.rs +++ b/tools/xtask-llm-benchmark/src/llm/types.rs @@ -1,6 +1,16 @@ +use clap::ValueEnum; use serde::{Deserialize, Serialize}; use std::fmt; +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum ReasoningEffort { + Low, + #[default] + Medium, + High, +} + /// Output from an LLM generation call, including token usage. #[derive(Debug, Clone, Default)] pub struct LlmOutput { @@ -68,3 +78,13 @@ impl fmt::Display for Vendor { f.write_str(self.slug()) } } + +#[cfg(test)] +mod tests { + use super::ReasoningEffort; + + #[test] + fn reasoning_effort_uses_provider_wire_values() { + assert_eq!(serde_json::to_string(&ReasoningEffort::Medium).unwrap(), r#""medium""#); + } +}