From 650c5be2e7241c3df19793541571d6e7d480a2d7 Mon Sep 17 00:00:00 2001 From: SonAIengine Date: Sun, 16 Aug 2026 23:56:26 +0900 Subject: [PATCH] Add model-in-loop long-horizon benchmark --- Makefile | 22 +- benchmarks/arazzo_long_horizon/model_run.py | 429 ++ benchmarks/paper_model_loop/client.py | 31 +- ...ng_horizon_deepseek_v4_flash_20260816.json | 6281 +++++++++++++++++ docs/benchmarks.md | 49 + docs/research/long-horizon-goal-evaluation.md | 28 + ...est_arazzo_long_horizon_model_benchmark.py | 115 + tests/test_paper_model_loop.py | 3 + 8 files changed, 6954 insertions(+), 4 deletions(-) create mode 100644 benchmarks/arazzo_long_horizon/model_run.py create mode 100644 benchmarks/results/arazzo_long_horizon_deepseek_v4_flash_20260816.json create mode 100644 tests/test_arazzo_long_horizon_model_benchmark.py diff --git a/Makefile b/Makefile index bcfe55f..26f8861 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: quick lint test verify research-check research-check-unit research-check-deterministic research-check-smoke paper-corpus-check paper-corpus-internal-review-check paper-corpus-claim-check paper-adapter-conformance paper-baseline-run paper-graph-ablation paper-producer-coverage paper-output-promotion paper-candidate-admission paper-contract-projection paper-model-loop paper-llm-catalog-baseline paper-model-loop-analysis paper-toolinkos-parity paper-openapi-closure paper-harness-check goal-completion-benchmark arazzo-long-horizon-benchmark xgen-benchmark xgen-llm-benchmark xgen-scale-snapshot xgen-scale-snapshot-check xgen-scale-acceptance xgen-scale-sweep xgen-scale-gate-check xgen-scale-028-gate-check xgen-scale-contract-ablation bfcl-benchmark bfcl-llm-benchmark bfcl-sweep bfcl-027-gate bfcl-027-gate-check bfcl-028-gate bfcl-028-gate-check bfcl-failure-subset bfcl-inspect-failures bfcl-hard-cases release-check pypi-smoke public-smoke launch-evidence launch-evidence-check observability-evidence observability-evidence-check +.PHONY: quick lint test verify research-check research-check-unit research-check-deterministic research-check-smoke paper-corpus-check paper-corpus-internal-review-check paper-corpus-claim-check paper-adapter-conformance paper-baseline-run paper-graph-ablation paper-producer-coverage paper-output-promotion paper-candidate-admission paper-contract-projection paper-model-loop paper-llm-catalog-baseline paper-model-loop-analysis paper-toolinkos-parity paper-openapi-closure paper-harness-check goal-completion-benchmark arazzo-long-horizon-benchmark arazzo-long-horizon-model-benchmark xgen-benchmark xgen-llm-benchmark xgen-scale-snapshot xgen-scale-snapshot-check xgen-scale-acceptance xgen-scale-sweep xgen-scale-gate-check xgen-scale-028-gate-check xgen-scale-contract-ablation bfcl-benchmark bfcl-llm-benchmark bfcl-sweep bfcl-027-gate bfcl-027-gate-check bfcl-028-gate bfcl-028-gate-check bfcl-failure-subset bfcl-inspect-failures bfcl-hard-cases release-check pypi-smoke public-smoke launch-evidence launch-evidence-check observability-evidence observability-evidence-check quick: scripts/quick-check.sh @@ -110,6 +110,7 @@ paper-model-loop: --model "$$MODEL" \ --model-revision "$$MODEL_REVISION" \ --provider "$${PROVIDER:-openai-compatible}" \ + --provider-profile "$${PROVIDER_PROFILE:-default}" \ --llm-url "$${LLM_URL:-http://localhost:8000/v1}" \ --repeats "$${REPEATS:-1}" \ --seed "$${SEED:-17}" \ @@ -186,6 +187,25 @@ arazzo-long-horizon-benchmark: --token-budget "$${TOKEN_BUDGET:-2048}" \ --out "$${OUT:-/tmp/graph-tool-call-arazzo-long-horizon.json}" +arazzo-long-horizon-model-benchmark: + @test -n "$${MODEL:-}" || (echo "MODEL is required"; exit 2) + @test -n "$${MODEL_REVISION:-}" || (echo "MODEL_REVISION is required"; exit 2) + @test -n "$${LLM_URL:-}" || (echo "LLM_URL is required"; exit 2) + poetry run python -m benchmarks.arazzo_long_horizon.model_run \ + --model "$$MODEL" \ + --model-revision "$$MODEL_REVISION" \ + --provider "$${PROVIDER:-openai-compatible}" \ + --llm-url "$$LLM_URL" \ + --api-key-env "$${API_KEY_ENV:-OPENAI_API_KEY}" \ + --catalog-size "$${CATALOG_SIZE:-1000}" \ + --lengths "$${LENGTHS:-3,10,30}" \ + --top-k "$${TOP_K:-8}" \ + --token-budget "$${TOKEN_BUDGET:-2048}" \ + --repeats "$${REPEATS:-1}" \ + --seed "$${SEED:-17}" \ + --timeout "$${TIMEOUT:-180}" \ + --out "$${OUT:-/tmp/graph-tool-call-arazzo-model-long-horizon.json}" + xgen-benchmark: poetry run python -m benchmarks.xgen_tool_graph.run --suite all diff --git a/benchmarks/arazzo_long_horizon/model_run.py b/benchmarks/arazzo_long_horizon/model_run.py new file mode 100644 index 0000000..aeaf877 --- /dev/null +++ b/benchmarks/arazzo_long_horizon/model_run.py @@ -0,0 +1,429 @@ +"""Run the sealed Arazzo long-horizon gate with a real target-selection model.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import time +from pathlib import Path +from typing import Any + +from benchmarks.arazzo_long_horizon.fixtures import FAMILIES, WorkflowFamily, build_fixture +from benchmarks.arazzo_long_horizon.run import DEFAULT_LENGTHS, WorkflowSandbox +from benchmarks.paper_model_loop import ( + HTTPModelClient, + ModelClient, + build_llm_catalog_index, + final_selection_messages, + parse_selector_decision, + redacted_url, +) +from graph_tool_call import __version__ +from graph_tool_call.evaluation import GoalExecutionRecord, ScenarioSpec, evaluate_goal_execution +from graph_tool_call.graphify import ( + apply_arazzo_workflows, + build_candidate_set, + ingest_openapi_graphify, + retrieve_graphify, + select_target_candidate, +) +from graph_tool_call.ingest.openapi import ingest_openapi +from graph_tool_call.plan import PathSynthesizer, PlanRunner + +MODEL_GATE_REVISION = "arazzo-model-target-selection-v1" + + +def run_model_benchmark( + *, + model: str, + model_revision: str, + llm_url: str, + provider: str = "openai-compatible", + provider_profile: str = "default", + api_key_env: str = "OPENAI_API_KEY", + catalog_size: int = 1000, + workflow_lengths: tuple[int, ...] = DEFAULT_LENGTHS, + top_k: int = 8, + token_budget: int = 2048, + repeats: int = 1, + seed: int = 17, + timeout: int = 180, + max_selection_tokens: int = 256, + model_client: ModelClient | None = None, +) -> dict[str, Any]: + """Evaluate model selection and graph-backed execution as separate stages.""" + + if len(workflow_lengths) > len(FAMILIES): + raise ValueError("workflow_lengths cannot exceed available unseen families") + if repeats <= 0: + raise ValueError("repeats must be greater than zero") + if model_client is None: + extra_body = _provider_options(provider_profile) + model_client = HTTPModelClient( + model=model, + url=llm_url, + provider=provider, + api_key_env=api_key_env, + extra_body=extra_body, + ) + + rows: list[dict[str, Any]] = [] + for repeat in range(repeats): + for family_index, (family, length) in enumerate( + zip(FAMILIES, workflow_lengths, strict=False) + ): + fixture = build_fixture(family, workflow_length=length, catalog_size=catalog_size) + rows.append( + _run_model_case( + fixture, + repeat=repeat, + paired_seed=seed + repeat * len(workflow_lengths) + family_index, + top_k=top_k, + token_budget=token_budget, + timeout=timeout, + max_selection_tokens=max_selection_tokens, + model_client=model_client, + ) + ) + + summary = _summarize(rows) + return { + "benchmark": "Arazzo Long-Horizon Model-in-the-Loop Evaluation", + "methodology": "retrieve_model_select_guard_plan_execute_goal_state", + "methodology_revision": MODEL_GATE_REVISION, + "graph_tool_call_version": __version__, + "model": { + "name": model, + "revision": model_revision, + "provider": provider, + "provider_profile": provider_profile, + "endpoint": redacted_url(llm_url), + }, + "catalog_size": catalog_size, + "workflow_lengths": list(workflow_lengths), + "top_k": top_k, + "token_budget": token_budget, + "repeats": repeats, + "seed": seed, + "summary": summary, + "cases": rows, + "limitations": [ + ( + "The model selects the final target from graph-tool-call's retrieved Top-K; " + "it is not shown evaluator gold or the complete 1,000-tool catalog." + ), + ( + "Arazzo supplies the prerequisite order and runtime bindings after target " + "selection; the model does not independently enumerate all 30 calls." + ), + "Generated catalogs are contract-distinct synthetic fixtures, not production APIs.", + ], + } + + +def _run_model_case( + fixture: dict[str, Any], + *, + repeat: int, + paired_seed: int, + top_k: int, + token_budget: int, + timeout: int, + max_selection_tokens: int, + model_client: ModelClient, +) -> dict[str, Any]: + family: WorkflowFamily = fixture["family"] + scenario = ScenarioSpec.from_dict(fixture["scenario"]) + tools, _ = ingest_openapi(fixture["openapi"]) + graph, edge_stats = ingest_openapi_graphify(tools, promote_contract_signals=True) + workflow_summary = apply_arazzo_workflows(graph, fixture["arazzo"]) + graph_payload = { + "graph": graph.graph.to_dict(), + "tools": {name: tool.to_dict() for name, tool in graph.tools.items()}, + } + + started = time.perf_counter() + retrieval = retrieve_graphify( + graph, + scenario.query, + top_k=top_k, + depth=0, + token_budget=token_budget, + include_evidence=True, + ) + retrieval_latency_ms = (time.perf_counter() - started) * 1000 + retrieval_rows = list(retrieval.get("results") or []) + retrieved = [str(row["name"]) for row in retrieval_rows] + serialized_catalog = _serialize_candidate_catalog(graph.tools, retrieved) + + model_response = model_client.complete( + final_selection_messages( + scenario.query, + serialized_catalog, + max_selected_tools=1, + ), + seed=paired_seed, + timeout=timeout, + max_tokens=max_selection_tokens, + ) + decision = parse_selector_decision(model_response.content) + llm_target = decision.target_tool + reason_codes = list(decision.reason_codes) + if model_response.error: + reason_codes.append("selection_model_call_failed") + if llm_target and llm_target not in retrieved: + reason_codes.append("selector_tool_not_in_catalog") + + selector = select_target_candidate( + scenario.query, + retrieved, + graph_payload["tools"], + retrieval_results=retrieval_rows, + llm_target=llm_target or None, + ) + selected_target = str(selector.get("selected_target") or "") + expected = list(fixture["operations"]) + target = expected[-1] + + candidates: list[str] = [] + plan = None + trace = None + failure: dict[str, str] = {} + sandbox = WorkflowSandbox(fixture) + started = time.perf_counter() + if selected_target: + expanded = build_candidate_set( + retrieved, + graph_payload["tools"], + expansion_seed=[selected_target], + max_producers_per_field=1, + max_hops=len(expected) + 1, + ) + candidates = [str(name) for name in expanded.get("candidates") or []] + try: + plan = PathSynthesizer(graph_payload, max_depth=len(expected) + 1).synthesize( + target=selected_target, + entities=dict(fixture["entities"]), + goal=scenario.query, + ) + trace = PlanRunner(sandbox.call_tool, binding_recovery=True).run( + plan, + input_context=dict(fixture["entities"]), + ) + except Exception as exc: # noqa: BLE001 - benchmark records stage failures + failure = {"reason": type(exc).__name__, "message": str(exc)} + plan_execute_latency_ms = (time.perf_counter() - started) * 1000 + + planned_tools = tuple(str(step.tool) for step in (getattr(plan, "steps", ()) or ())) + if trace is None: + record = GoalExecutionRecord( + calls=(), + success=False, + retrieved_tools=tuple(retrieved), + candidate_tools=tuple(candidates), + planned_tools=planned_tools, + final_state=sandbox.snapshot(), + ) + else: + record = GoalExecutionRecord.from_execution_trace( + trace, + plan=plan, + retrieved_tools=retrieved, + candidate_tools=candidates, + final_state=sandbox.snapshot(), + schema_valid=True, + ) + evaluation = evaluate_goal_execution(scenario, record) + executed = [call.tool for call in record.calls] + return { + "case_id": f"{scenario.id}::repeat-{repeat}", + "family": family.id, + "workflow_length": len(expected), + "catalog_size": len(graph.tools), + "paired_seed": paired_seed, + "query": scenario.query, + "tool_count": len(graph.tools), + "edge_count": graph.graph.edge_count(), + "edge_stats": edge_stats, + "workflow_summary": workflow_summary, + "target_hit_at_k": float(target in retrieved), + "llm_target": llm_target, + "llm_target_exact": float(llm_target == target), + "llm_target_in_catalog": float(llm_target in retrieved), + "selected_target": selected_target, + "selected_target_exact": float(selected_target == target), + "selector_overrode_llm": bool(selector.get("overrode_llm")), + "selector_reason_codes": list(selector.get("reason_codes") or []), + "retrieved_tools": retrieved, + "candidate_catalog_sha256": hashlib.sha256(serialized_catalog.encode()).hexdigest(), + "model_response": model_response.to_dict(), + "model_reason_codes": list(dict.fromkeys(reason_codes)), + "candidate_count": len(candidates), + "planned_call_count": len(planned_tools), + "executed_call_count": len(executed), + "plan_order_exact": float(list(planned_tools) == expected), + "execution_order_exact": float(executed == expected), + "planned_tools": list(planned_tools), + "executed_tools": executed, + "runner_success": bool(record.success), + "failure": failure, + "evaluation": evaluation.to_dict(), + "latency_ms": { + "retrieve": round(retrieval_latency_ms, 3), + "model_select": round(model_response.latency_ms, 3), + "plan_execute": round(plan_execute_latency_ms, 3), + }, + "usage": { + "input_tokens": model_response.input_tokens, + "output_tokens": model_response.output_tokens, + "retrieval_token_budget_used": int( + retrieval.get("token_budget_used") + or (retrieval.get("stats") or {}).get("token_budget_used") + or 0 + ), + }, + } + + +def _serialize_candidate_catalog(tools_by_name: dict[str, Any], names: list[str]) -> str: + entries_by_name = {entry["name"]: entry for entry in build_llm_catalog_index(tools_by_name)} + entries = [entries_by_name[name] for name in names if name in entries_by_name] + return json.dumps(entries, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _provider_options(provider_profile: str) -> dict[str, Any]: + if provider_profile == "deepseek": + return { + "thinking": {"type": "disabled"}, + "response_format": {"type": "json_object"}, + } + if provider_profile == "default": + return {} + raise ValueError("provider_profile must be default or deepseek") + + +def _summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: + def average(path: tuple[str, ...]) -> float: + values: list[float] = [] + for row in rows: + value: Any = row + for key in path: + value = value.get(key) if isinstance(value, dict) else None + values.append(float(value or 0)) + return round(sum(values) / len(values), 6) if values else 0.0 + + metrics = { + "target_hit_at_k": average(("target_hit_at_k",)), + "llm_target_exact": average(("llm_target_exact",)), + "llm_target_in_catalog": average(("llm_target_in_catalog",)), + "selected_target_exact": average(("selected_target_exact",)), + "plan_required_tool_recall": average( + ("evaluation", "metrics", "plan_required_tool_recall") + ), + "execution_required_tool_recall": average( + ("evaluation", "metrics", "execution_required_tool_recall") + ), + "plan_order_exact": average(("plan_order_exact",)), + "execution_order_exact": average(("execution_order_exact",)), + "binding_accuracy": average(("evaluation", "metrics", "binding_accuracy")), + "goal_completion_rate": average(("evaluation", "metrics", "goal_completion")), + } + gates = {name: value == 1.0 for name, value in metrics.items()} + return { + "status": "pass" if all(gates.values()) else "fail", + "case_count": len(rows), + "metrics": metrics, + "gates": gates, + "selector_override_rate": average(("selector_overrode_llm",)), + "latency_ms": { + stage: _latency_percentiles(rows, stage) + for stage in ("retrieve", "model_select", "plan_execute") + }, + "usage": { + "input_tokens": sum(int(row["usage"]["input_tokens"]) for row in rows), + "output_tokens": sum(int(row["usage"]["output_tokens"]) for row in rows), + }, + } + + +def _latency_percentiles(rows: list[dict[str, Any]], stage: str) -> dict[str, float]: + values = sorted(float(row["latency_ms"][stage]) for row in rows) + return {"p50": round(_percentile(values, 0.50), 3), "p95": round(_percentile(values, 0.95), 3)} + + +def _percentile(values: list[float], quantile: float) -> float: + if not values: + return 0.0 + position = (len(values) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return values[lower] + return values[lower] + (values[upper] - values[lower]) * (position - lower) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--model-revision", required=True) + parser.add_argument("--provider", default="openai-compatible") + parser.add_argument("--provider-profile", choices=("default", "deepseek"), default="default") + parser.add_argument("--llm-url", required=True) + parser.add_argument("--api-key-env", default="OPENAI_API_KEY") + parser.add_argument("--catalog-size", type=int, default=1000) + parser.add_argument("--lengths", default="3,10,30") + parser.add_argument("--top-k", type=int, default=8) + parser.add_argument("--token-budget", type=int, default=2048) + parser.add_argument("--repeats", type=int, default=1) + parser.add_argument("--seed", type=int, default=17) + parser.add_argument("--timeout", type=int, default=180) + parser.add_argument("--max-selection-tokens", type=int, default=256) + parser.add_argument("--out", type=Path) + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + lengths = tuple(int(value) for value in args.lengths.split(",") if value.strip()) + report = run_model_benchmark( + model=args.model, + model_revision=args.model_revision, + llm_url=args.llm_url, + provider=args.provider, + provider_profile=args.provider_profile, + api_key_env=args.api_key_env, + catalog_size=args.catalog_size, + workflow_lengths=lengths, + top_k=args.top_k, + token_budget=args.token_budget, + repeats=args.repeats, + seed=args.seed, + timeout=args.timeout, + max_selection_tokens=args.max_selection_tokens, + ) + rendered = json.dumps(report, ensure_ascii=False, indent=2) + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(rendered + "\n", encoding="utf-8") + if args.json: + print(rendered) + else: + summary = report["summary"] + metrics = summary["metrics"] + print( + f"{report['benchmark']}: {summary['status']} " + f"model target={metrics['llm_target_exact']:.0%}, " + f"final target={metrics['selected_target_exact']:.0%}, " + f"goal={metrics['goal_completion_rate']:.0%}" + ) + for row in report["cases"]: + print( + f"- {row['family']} {row['workflow_length']} steps / " + f"{row['catalog_size']} tools: llm={row['llm_target_exact']:.0f}, " + f"final={row['selected_target_exact']:.0f}, " + f"goal={'pass' if row['evaluation']['goal_completed'] else 'fail'}" + ) + return 0 if report["summary"]["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/paper_model_loop/client.py b/benchmarks/paper_model_loop/client.py index dff7fb3..4d0432e 100644 --- a/benchmarks/paper_model_loop/client.py +++ b/benchmarks/paper_model_loop/client.py @@ -7,6 +7,7 @@ import re import time import urllib.error +import urllib.parse import urllib.request from dataclasses import asdict, dataclass from typing import Any, Protocol @@ -59,6 +60,7 @@ def __init__( disable_thinking: bool = True, include_seed: bool = True, api_key_env: str = "OPENAI_API_KEY", + extra_body: dict[str, Any] | None = None, ) -> None: if not model.strip() or not url.strip(): raise ValueError("model and url must be non-empty.") @@ -70,6 +72,7 @@ def __init__( self.disable_thinking = disable_thinking self.include_seed = include_seed self.api_key_env = api_key_env + self.extra_body = dict(extra_body or {}) def complete( self, @@ -111,8 +114,9 @@ def _complete_openai( } if self.include_seed: payload["seed"] = seed - if self.disable_thinking: + if self.disable_thinking and "thinking" not in self.extra_body: payload["chat_template_kwargs"] = {"enable_thinking": False} + payload.update(self.extra_body) headers = {"Content-Type": "application/json"} api_key = os.environ.get(self.api_key_env) if api_key: @@ -189,8 +193,29 @@ def _complete_ollama( def redacted_url(url: str) -> str: - """Remove embedded credentials before a URL enters an artifact.""" - return re.sub(r"://([^/@]+)@", "://***@", url) + """Remove embedded credentials and sensitive query values from artifact URLs.""" + try: + parsed = urllib.parse.urlsplit(url) + netloc = parsed.netloc + if "@" in netloc: + netloc = f"***@{netloc.rsplit('@', 1)[1]}" + query = urllib.parse.parse_qsl(parsed.query, keep_blank_values=True) + scrubbed_query = urllib.parse.urlencode( + [(key, "***" if _is_sensitive_query_key(key) else value) for key, value in query] + ) + return urllib.parse.urlunsplit( + (parsed.scheme, netloc, parsed.path, scrubbed_query, parsed.fragment) + ) + except ValueError: + return re.sub(r"://([^/@]+)@", "://***@", url) + + +def _is_sensitive_query_key(key: str) -> bool: + normalized = re.sub(r"[^a-z]", "", key.lower()) + return any( + marker in normalized + for marker in ("apikey", "token", "authorization", "password", "secret", "signature") + ) def _post_json( diff --git a/benchmarks/results/arazzo_long_horizon_deepseek_v4_flash_20260816.json b/benchmarks/results/arazzo_long_horizon_deepseek_v4_flash_20260816.json new file mode 100644 index 0000000..2c3c198 --- /dev/null +++ b/benchmarks/results/arazzo_long_horizon_deepseek_v4_flash_20260816.json @@ -0,0 +1,6281 @@ +{ + "benchmark": "Arazzo Long-Horizon Model-in-the-Loop Evaluation", + "methodology": "retrieve_model_select_guard_plan_execute_goal_state", + "methodology_revision": "arazzo-model-target-selection-v1", + "graph_tool_call_version": "0.42.0", + "model": { + "name": "deepseek-v4-flash", + "revision": "api-catalog-2026-08-16", + "provider": "openai-compatible", + "provider_profile": "deepseek", + "endpoint": "https://api.deepseek.com/v1" + }, + "catalog_size": 1000, + "workflow_lengths": [ + 3, + 10, + 30 + ], + "top_k": 8, + "token_budget": 2048, + "repeats": 3, + "seed": 17, + "summary": { + "status": "pass", + "case_count": 9, + "metrics": { + "target_hit_at_k": 1.0, + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target_exact": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "binding_accuracy": 1.0, + "goal_completion_rate": 1.0 + }, + "gates": { + "target_hit_at_k": true, + "llm_target_exact": true, + "llm_target_in_catalog": true, + "selected_target_exact": true, + "plan_required_tool_recall": true, + "execution_required_tool_recall": true, + "plan_order_exact": true, + "execution_order_exact": true, + "binding_accuracy": true, + "goal_completion_rate": true + }, + "selector_override_rate": 0.0, + "latency_ms": { + "retrieve": { + "p50": 96.618, + "p95": 116.312 + }, + "model_select": { + "p50": 1184.876, + "p95": 1252.382 + }, + "plan_execute": { + "p50": 6.256, + "p95": 11.277 + } + }, + "usage": { + "input_tokens": 8223, + "output_tokens": 168 + } + }, + "cases": [ + { + "case_id": "supplier_onboarding_3::repeat-0", + "family": "supplier_onboarding", + "workflow_length": 3, + "catalog_size": 1000, + "paired_seed": 17, + "query": "공급업체 등록 절차를 모두 완료하고 활성화 인증서를 발급해줘", + "tool_count": 1000, + "edge_count": 2, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 998, + "produces_consumer_aligned": 0, + "consumes_added": 3, + "produces_skipped": 3, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 3, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 2, + "create": 1, + "search": 997 + }, + "resource_counts": { + "archive": 997, + "supplier_onboarding": 3 + }, + "module_counts": { + "archive_references": 997, + "supplier_onboarding_stages": 3 + }, + "result_shape_counts": { + "list": 997, + "mutation": 3 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 997, + "rate": 0.997 + }, + { + "module": "supplier_onboarding_stages", + "count": 3, + "rate": 0.003 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 3, + "relation_count": 2, + "by_dependency_kind": { + "runtime_reference": 2 + }, + "edge_stats": { + "added": 2, + "merged": 0, + "binding_aliases_added": 2 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "08929f2c7d36ff7b0036617930b6636dcfda90ca514a2f1207564c01acfd317a", + "bytes": 793, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "llm_target": "issueSupplierActivationCertificate", + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target": "issueSupplierActivationCertificate", + "selected_target_exact": 1.0, + "selector_overrode_llm": false, + "selector_reason_codes": [ + "ambiguous_target" + ], + "retrieved_tools": [ + "issueSupplierActivationCertificate", + "completeSupplierOnboardingStage02", + "resolveSupplierTenant" + ], + "candidate_catalog_sha256": "80a5bfd6a1d5bb9d6015baba15a6e9cc10f5d67dfd0a6e817fbda1d4a6b01d36", + "model_response": { + "content": "{\"target_tool\":\"issueSupplierActivationCertificate\",\"supporting_tools\":[]}", + "input_tokens": 671, + "output_tokens": 18, + "latency_ms": 1184.8757937550545, + "status_code": 200, + "finish_reason": "stop", + "error": "" + }, + "model_reason_codes": [], + "candidate_count": 3, + "planned_call_count": 3, + "executed_call_count": 3, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "resolveSupplierTenant", + "completeSupplierOnboardingStage02", + "issueSupplierActivationCertificate" + ], + "executed_tools": [ + "resolveSupplierTenant", + "completeSupplierOnboardingStage02", + "issueSupplierActivationCertificate" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "supplier_onboarding_3", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 3, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "resolveSupplierTenant" + ], + "observed": "resolveSupplierTenant" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completeSupplierOnboardingStage02" + ], + "observed": "completeSupplierOnboardingStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "issueSupplierActivationCertificate" + ], + "observed": "issueSupplierActivationCertificate" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-supplier_onboarding-01", + "observed": "EVIDENCE-supplier_onboarding-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-supplier_onboarding-02", + "observed": "EVIDENCE-supplier_onboarding-02" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.activationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-supplier_onboarding" + }, + "observed": "CERT-supplier_onboarding" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 3, + "observed": 3 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 3, + "observed": 3 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "retrieve": 96.618, + "model_select": 1184.876, + "plan_execute": 5.635 + }, + "usage": { + "input_tokens": 671, + "output_tokens": 18, + "retrieval_token_budget_used": 190 + } + }, + { + "case_id": "incident_recovery_10::repeat-0", + "family": "incident_recovery", + "workflow_length": 10, + "catalog_size": 1000, + "paired_seed": 18, + "query": "서비스 장애 복구 절차를 모두 수행하고 복구 검증 보고서를 발급해줘", + "tool_count": 1000, + "edge_count": 9, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 991, + "produces_consumer_aligned": 0, + "consumes_added": 10, + "produces_skipped": 10, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 10, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 10, + "search": 990 + }, + "resource_counts": { + "archive": 990, + "incident_recovery": 10 + }, + "module_counts": { + "archive_references": 990, + "incident_recovery_stages": 10 + }, + "result_shape_counts": { + "list": 990, + "mutation": 10 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 990, + "rate": 0.99 + }, + { + "module": "incident_recovery_stages", + "count": 10, + "rate": 0.01 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 10, + "relation_count": 9, + "by_dependency_kind": { + "runtime_reference": 9 + }, + "edge_stats": { + "added": 9, + "merged": 0, + "binding_aliases_added": 9 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "42cbf08002ff711f27fda523ebfe3974c14eaa8e1ab48f8a8857e3d0ca8b31fc", + "bytes": 2308, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "llm_target": "issueIncidentRecoveryReport", + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target": "issueIncidentRecoveryReport", + "selected_target_exact": 1.0, + "selector_overrode_llm": false, + "selector_reason_codes": [ + "ambiguous_target" + ], + "retrieved_tools": [ + "issueIncidentRecoveryReport", + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04" + ], + "candidate_catalog_sha256": "d6a18ec50814891ca2010066e044da9a1a06f83d007bd11fe04a331dc234d3e0", + "model_response": { + "content": "{\"target_tool\":\"issueIncidentRecoveryReport\",\"supporting_tools\":[]}", + "input_tokens": 1030, + "output_tokens": 20, + "latency_ms": 1180.9020768851042, + "status_code": 200, + "finish_reason": "stop", + "error": "" + }, + "model_reason_codes": [], + "candidate_count": 10, + "planned_call_count": 10, + "executed_call_count": 10, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04", + "completeIncidentRecoveryStage05", + "completeIncidentRecoveryStage06", + "completeIncidentRecoveryStage07", + "completeIncidentRecoveryStage08", + "completeIncidentRecoveryStage09", + "issueIncidentRecoveryReport" + ], + "executed_tools": [ + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04", + "completeIncidentRecoveryStage05", + "completeIncidentRecoveryStage06", + "completeIncidentRecoveryStage07", + "completeIncidentRecoveryStage08", + "completeIncidentRecoveryStage09", + "issueIncidentRecoveryReport" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "incident_recovery_10", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 10, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3, + "stage_04": 4, + "stage_05": 5, + "stage_06": 6, + "stage_07": 7, + "stage_08": 8, + "stage_09": 9, + "stage_10": 10 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openIncidentInvestigation" + ], + "observed": "openIncidentInvestigation" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completeIncidentRecoveryStage02" + ], + "observed": "completeIncidentRecoveryStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "completeIncidentRecoveryStage03" + ], + "observed": "completeIncidentRecoveryStage03" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_04", + "expected": [ + "completeIncidentRecoveryStage04" + ], + "observed": "completeIncidentRecoveryStage04" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_05", + "expected": [ + "completeIncidentRecoveryStage05" + ], + "observed": "completeIncidentRecoveryStage05" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_06", + "expected": [ + "completeIncidentRecoveryStage06" + ], + "observed": "completeIncidentRecoveryStage06" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_07", + "expected": [ + "completeIncidentRecoveryStage07" + ], + "observed": "completeIncidentRecoveryStage07" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_08", + "expected": [ + "completeIncidentRecoveryStage08" + ], + "observed": "completeIncidentRecoveryStage08" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_09", + "expected": [ + "completeIncidentRecoveryStage09" + ], + "observed": "completeIncidentRecoveryStage09" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_10", + "expected": [ + "issueIncidentRecoveryReport" + ], + "observed": "issueIncidentRecoveryReport" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": 3, + "after_sequence": 4 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": 4, + "after_sequence": 5 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": 5, + "after_sequence": 6 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": 6, + "after_sequence": 7 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": 7, + "after_sequence": 8 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": 8, + "after_sequence": 9 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": 9, + "after_sequence": 10 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-incident_recovery-01", + "observed": "EVIDENCE-incident_recovery-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-incident_recovery-02", + "observed": "EVIDENCE-incident_recovery-02" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "EVIDENCE-incident_recovery-03", + "observed": "EVIDENCE-incident_recovery-03" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "EVIDENCE-incident_recovery-04", + "observed": "EVIDENCE-incident_recovery-04" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "EVIDENCE-incident_recovery-05", + "observed": "EVIDENCE-incident_recovery-05" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "EVIDENCE-incident_recovery-06", + "observed": "EVIDENCE-incident_recovery-06" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "EVIDENCE-incident_recovery-07", + "observed": "EVIDENCE-incident_recovery-07" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "EVIDENCE-incident_recovery-08", + "observed": "EVIDENCE-incident_recovery-08" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "EVIDENCE-incident_recovery-09", + "observed": "EVIDENCE-incident_recovery-09" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.recoveryReportId", + "expected": { + "operator": "eq", + "value": "CERT-incident_recovery" + }, + "observed": "CERT-incident_recovery" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 10, + "observed": 10 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 10, + "observed": 10 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "retrieve": 94.422, + "model_select": 1180.902, + "plan_execute": 8.155 + }, + "usage": { + "input_tokens": 1030, + "output_tokens": 20, + "retrieval_token_budget_used": 291 + } + }, + { + "case_id": "publication_approval_30::repeat-0", + "family": "publication_approval", + "workflow_length": 30, + "catalog_size": 1000, + "paired_seed": 19, + "query": "콘텐츠 검토 절차를 모두 완료하고 게시 승인 증명서를 발급해줘", + "tool_count": 1000, + "edge_count": 29, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 971, + "produces_consumer_aligned": 0, + "consumes_added": 30, + "produces_skipped": 30, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 30, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 30, + "search": 970 + }, + "resource_counts": { + "archive": 970, + "publication_approval": 30 + }, + "module_counts": { + "archive_references": 970, + "publication_approval_stages": 30 + }, + "result_shape_counts": { + "list": 970, + "mutation": 30 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 970, + "rate": 0.97 + }, + { + "module": "publication_approval_stages", + "count": 30, + "rate": 0.03 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 30, + "relation_count": 29, + "by_dependency_kind": { + "runtime_reference": 29 + }, + "edge_stats": { + "added": 29, + "merged": 0, + "binding_aliases_added": 29 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "21a3ad4765fcfda8aea532b05c84a753829039e9d45653270a918c2deda939e1", + "bytes": 6709, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "llm_target": "issuePublicationApprovalCertificate", + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target": "issuePublicationApprovalCertificate", + "selected_target_exact": 1.0, + "selector_overrode_llm": false, + "selector_reason_codes": [ + "ambiguous_target" + ], + "retrieved_tools": [ + "issuePublicationApprovalCertificate", + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04" + ], + "candidate_catalog_sha256": "0a64e7719a81b3885f0837c2987c98460066571e16195dc411cebdead3c4cd41", + "model_response": { + "content": "{\"target_tool\":\"issuePublicationApprovalCertificate\",\"supporting_tools\":[]}", + "input_tokens": 1040, + "output_tokens": 18, + "latency_ms": 1189.5391196012497, + "status_code": 200, + "finish_reason": "stop", + "error": "" + }, + "model_reason_codes": [], + "candidate_count": 30, + "planned_call_count": 30, + "executed_call_count": 30, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04", + "completePublicationReviewStage05", + "completePublicationReviewStage06", + "completePublicationReviewStage07", + "completePublicationReviewStage08", + "completePublicationReviewStage09", + "completePublicationReviewStage10", + "completePublicationReviewStage11", + "completePublicationReviewStage12", + "completePublicationReviewStage13", + "completePublicationReviewStage14", + "completePublicationReviewStage15", + "completePublicationReviewStage16", + "completePublicationReviewStage17", + "completePublicationReviewStage18", + "completePublicationReviewStage19", + "completePublicationReviewStage20", + "completePublicationReviewStage21", + "completePublicationReviewStage22", + "completePublicationReviewStage23", + "completePublicationReviewStage24", + "completePublicationReviewStage25", + "completePublicationReviewStage26", + "completePublicationReviewStage27", + "completePublicationReviewStage28", + "completePublicationReviewStage29", + "issuePublicationApprovalCertificate" + ], + "executed_tools": [ + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04", + "completePublicationReviewStage05", + "completePublicationReviewStage06", + "completePublicationReviewStage07", + "completePublicationReviewStage08", + "completePublicationReviewStage09", + "completePublicationReviewStage10", + "completePublicationReviewStage11", + "completePublicationReviewStage12", + "completePublicationReviewStage13", + "completePublicationReviewStage14", + "completePublicationReviewStage15", + "completePublicationReviewStage16", + "completePublicationReviewStage17", + "completePublicationReviewStage18", + "completePublicationReviewStage19", + "completePublicationReviewStage20", + "completePublicationReviewStage21", + "completePublicationReviewStage22", + "completePublicationReviewStage23", + "completePublicationReviewStage24", + "completePublicationReviewStage25", + "completePublicationReviewStage26", + "completePublicationReviewStage27", + "completePublicationReviewStage28", + "completePublicationReviewStage29", + "issuePublicationApprovalCertificate" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "publication_approval_30", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 30, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3, + "stage_04": 4, + "stage_05": 5, + "stage_06": 6, + "stage_07": 7, + "stage_08": 8, + "stage_09": 9, + "stage_10": 10, + "stage_11": 11, + "stage_12": 12, + "stage_13": 13, + "stage_14": 14, + "stage_15": 15, + "stage_16": 16, + "stage_17": 17, + "stage_18": 18, + "stage_19": 19, + "stage_20": 20, + "stage_21": 21, + "stage_22": 22, + "stage_23": 23, + "stage_24": 24, + "stage_25": 25, + "stage_26": 26, + "stage_27": 27, + "stage_28": 28, + "stage_29": 29, + "stage_30": 30 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openPublicationReview" + ], + "observed": "openPublicationReview" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completePublicationReviewStage02" + ], + "observed": "completePublicationReviewStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "completePublicationReviewStage03" + ], + "observed": "completePublicationReviewStage03" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_04", + "expected": [ + "completePublicationReviewStage04" + ], + "observed": "completePublicationReviewStage04" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_05", + "expected": [ + "completePublicationReviewStage05" + ], + "observed": "completePublicationReviewStage05" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_06", + "expected": [ + "completePublicationReviewStage06" + ], + "observed": "completePublicationReviewStage06" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_07", + "expected": [ + "completePublicationReviewStage07" + ], + "observed": "completePublicationReviewStage07" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_08", + "expected": [ + "completePublicationReviewStage08" + ], + "observed": "completePublicationReviewStage08" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_09", + "expected": [ + "completePublicationReviewStage09" + ], + "observed": "completePublicationReviewStage09" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_10", + "expected": [ + "completePublicationReviewStage10" + ], + "observed": "completePublicationReviewStage10" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_11", + "expected": [ + "completePublicationReviewStage11" + ], + "observed": "completePublicationReviewStage11" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_12", + "expected": [ + "completePublicationReviewStage12" + ], + "observed": "completePublicationReviewStage12" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_13", + "expected": [ + "completePublicationReviewStage13" + ], + "observed": "completePublicationReviewStage13" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_14", + "expected": [ + "completePublicationReviewStage14" + ], + "observed": "completePublicationReviewStage14" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_15", + "expected": [ + "completePublicationReviewStage15" + ], + "observed": "completePublicationReviewStage15" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_16", + "expected": [ + "completePublicationReviewStage16" + ], + "observed": "completePublicationReviewStage16" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_17", + "expected": [ + "completePublicationReviewStage17" + ], + "observed": "completePublicationReviewStage17" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_18", + "expected": [ + "completePublicationReviewStage18" + ], + "observed": "completePublicationReviewStage18" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_19", + "expected": [ + "completePublicationReviewStage19" + ], + "observed": "completePublicationReviewStage19" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_20", + "expected": [ + "completePublicationReviewStage20" + ], + "observed": "completePublicationReviewStage20" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_21", + "expected": [ + "completePublicationReviewStage21" + ], + "observed": "completePublicationReviewStage21" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_22", + "expected": [ + "completePublicationReviewStage22" + ], + "observed": "completePublicationReviewStage22" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_23", + "expected": [ + "completePublicationReviewStage23" + ], + "observed": "completePublicationReviewStage23" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_24", + "expected": [ + "completePublicationReviewStage24" + ], + "observed": "completePublicationReviewStage24" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_25", + "expected": [ + "completePublicationReviewStage25" + ], + "observed": "completePublicationReviewStage25" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_26", + "expected": [ + "completePublicationReviewStage26" + ], + "observed": "completePublicationReviewStage26" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_27", + "expected": [ + "completePublicationReviewStage27" + ], + "observed": "completePublicationReviewStage27" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_28", + "expected": [ + "completePublicationReviewStage28" + ], + "observed": "completePublicationReviewStage28" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_29", + "expected": [ + "completePublicationReviewStage29" + ], + "observed": "completePublicationReviewStage29" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_30", + "expected": [ + "issuePublicationApprovalCertificate" + ], + "observed": "issuePublicationApprovalCertificate" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": 3, + "after_sequence": 4 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": 4, + "after_sequence": 5 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": 5, + "after_sequence": 6 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": 6, + "after_sequence": 7 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": 7, + "after_sequence": 8 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": 8, + "after_sequence": 9 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": 9, + "after_sequence": 10 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_10->stage_11", + "expected": "before", + "observed": { + "before_sequence": 10, + "after_sequence": 11 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_11->stage_12", + "expected": "before", + "observed": { + "before_sequence": 11, + "after_sequence": 12 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_12->stage_13", + "expected": "before", + "observed": { + "before_sequence": 12, + "after_sequence": 13 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_13->stage_14", + "expected": "before", + "observed": { + "before_sequence": 13, + "after_sequence": 14 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_14->stage_15", + "expected": "before", + "observed": { + "before_sequence": 14, + "after_sequence": 15 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_15->stage_16", + "expected": "before", + "observed": { + "before_sequence": 15, + "after_sequence": 16 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_16->stage_17", + "expected": "before", + "observed": { + "before_sequence": 16, + "after_sequence": 17 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_17->stage_18", + "expected": "before", + "observed": { + "before_sequence": 17, + "after_sequence": 18 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_18->stage_19", + "expected": "before", + "observed": { + "before_sequence": 18, + "after_sequence": 19 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_19->stage_20", + "expected": "before", + "observed": { + "before_sequence": 19, + "after_sequence": 20 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_20->stage_21", + "expected": "before", + "observed": { + "before_sequence": 20, + "after_sequence": 21 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_21->stage_22", + "expected": "before", + "observed": { + "before_sequence": 21, + "after_sequence": 22 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_22->stage_23", + "expected": "before", + "observed": { + "before_sequence": 22, + "after_sequence": 23 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_23->stage_24", + "expected": "before", + "observed": { + "before_sequence": 23, + "after_sequence": 24 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_24->stage_25", + "expected": "before", + "observed": { + "before_sequence": 24, + "after_sequence": 25 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_25->stage_26", + "expected": "before", + "observed": { + "before_sequence": 25, + "after_sequence": 26 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_26->stage_27", + "expected": "before", + "observed": { + "before_sequence": 26, + "after_sequence": 27 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_27->stage_28", + "expected": "before", + "observed": { + "before_sequence": 27, + "after_sequence": 28 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_28->stage_29", + "expected": "before", + "observed": { + "before_sequence": 28, + "after_sequence": 29 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_29->stage_30", + "expected": "before", + "observed": { + "before_sequence": 29, + "after_sequence": 30 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-publication_approval-01", + "observed": "EVIDENCE-publication_approval-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-publication_approval-02", + "observed": "EVIDENCE-publication_approval-02" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "EVIDENCE-publication_approval-03", + "observed": "EVIDENCE-publication_approval-03" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "EVIDENCE-publication_approval-04", + "observed": "EVIDENCE-publication_approval-04" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "EVIDENCE-publication_approval-05", + "observed": "EVIDENCE-publication_approval-05" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "EVIDENCE-publication_approval-06", + "observed": "EVIDENCE-publication_approval-06" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "EVIDENCE-publication_approval-07", + "observed": "EVIDENCE-publication_approval-07" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "EVIDENCE-publication_approval-08", + "observed": "EVIDENCE-publication_approval-08" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "EVIDENCE-publication_approval-09", + "observed": "EVIDENCE-publication_approval-09" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_10.stageEvidence10->stage_11.handoffToken10", + "expected": "EVIDENCE-publication_approval-10", + "observed": "EVIDENCE-publication_approval-10" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_11.stageEvidence11->stage_12.handoffToken11", + "expected": "EVIDENCE-publication_approval-11", + "observed": "EVIDENCE-publication_approval-11" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_12.stageEvidence12->stage_13.handoffToken12", + "expected": "EVIDENCE-publication_approval-12", + "observed": "EVIDENCE-publication_approval-12" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_13.stageEvidence13->stage_14.handoffToken13", + "expected": "EVIDENCE-publication_approval-13", + "observed": "EVIDENCE-publication_approval-13" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_14.stageEvidence14->stage_15.handoffToken14", + "expected": "EVIDENCE-publication_approval-14", + "observed": "EVIDENCE-publication_approval-14" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_15.stageEvidence15->stage_16.handoffToken15", + "expected": "EVIDENCE-publication_approval-15", + "observed": "EVIDENCE-publication_approval-15" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_16.stageEvidence16->stage_17.handoffToken16", + "expected": "EVIDENCE-publication_approval-16", + "observed": "EVIDENCE-publication_approval-16" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_17.stageEvidence17->stage_18.handoffToken17", + "expected": "EVIDENCE-publication_approval-17", + "observed": "EVIDENCE-publication_approval-17" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_18.stageEvidence18->stage_19.handoffToken18", + "expected": "EVIDENCE-publication_approval-18", + "observed": "EVIDENCE-publication_approval-18" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_19.stageEvidence19->stage_20.handoffToken19", + "expected": "EVIDENCE-publication_approval-19", + "observed": "EVIDENCE-publication_approval-19" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_20.stageEvidence20->stage_21.handoffToken20", + "expected": "EVIDENCE-publication_approval-20", + "observed": "EVIDENCE-publication_approval-20" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_21.stageEvidence21->stage_22.handoffToken21", + "expected": "EVIDENCE-publication_approval-21", + "observed": "EVIDENCE-publication_approval-21" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_22.stageEvidence22->stage_23.handoffToken22", + "expected": "EVIDENCE-publication_approval-22", + "observed": "EVIDENCE-publication_approval-22" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_23.stageEvidence23->stage_24.handoffToken23", + "expected": "EVIDENCE-publication_approval-23", + "observed": "EVIDENCE-publication_approval-23" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_24.stageEvidence24->stage_25.handoffToken24", + "expected": "EVIDENCE-publication_approval-24", + "observed": "EVIDENCE-publication_approval-24" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_25.stageEvidence25->stage_26.handoffToken25", + "expected": "EVIDENCE-publication_approval-25", + "observed": "EVIDENCE-publication_approval-25" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_26.stageEvidence26->stage_27.handoffToken26", + "expected": "EVIDENCE-publication_approval-26", + "observed": "EVIDENCE-publication_approval-26" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_27.stageEvidence27->stage_28.handoffToken27", + "expected": "EVIDENCE-publication_approval-27", + "observed": "EVIDENCE-publication_approval-27" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_28.stageEvidence28->stage_29.handoffToken28", + "expected": "EVIDENCE-publication_approval-28", + "observed": "EVIDENCE-publication_approval-28" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_29.stageEvidence29->stage_30.handoffToken29", + "expected": "EVIDENCE-publication_approval-29", + "observed": "EVIDENCE-publication_approval-29" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.publicationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-publication_approval" + }, + "observed": "CERT-publication_approval" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 30, + "observed": 30 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 30, + "observed": 30 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "retrieve": 97.12, + "model_select": 1189.539, + "plan_execute": 13.358 + }, + "usage": { + "input_tokens": 1040, + "output_tokens": 18, + "retrieval_token_budget_used": 301 + } + }, + { + "case_id": "supplier_onboarding_3::repeat-1", + "family": "supplier_onboarding", + "workflow_length": 3, + "catalog_size": 1000, + "paired_seed": 20, + "query": "공급업체 등록 절차를 모두 완료하고 활성화 인증서를 발급해줘", + "tool_count": 1000, + "edge_count": 2, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 998, + "produces_consumer_aligned": 0, + "consumes_added": 3, + "produces_skipped": 3, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 3, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 2, + "create": 1, + "search": 997 + }, + "resource_counts": { + "archive": 997, + "supplier_onboarding": 3 + }, + "module_counts": { + "archive_references": 997, + "supplier_onboarding_stages": 3 + }, + "result_shape_counts": { + "list": 997, + "mutation": 3 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 997, + "rate": 0.997 + }, + { + "module": "supplier_onboarding_stages", + "count": 3, + "rate": 0.003 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 3, + "relation_count": 2, + "by_dependency_kind": { + "runtime_reference": 2 + }, + "edge_stats": { + "added": 2, + "merged": 0, + "binding_aliases_added": 2 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "08929f2c7d36ff7b0036617930b6636dcfda90ca514a2f1207564c01acfd317a", + "bytes": 793, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "llm_target": "issueSupplierActivationCertificate", + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target": "issueSupplierActivationCertificate", + "selected_target_exact": 1.0, + "selector_overrode_llm": false, + "selector_reason_codes": [ + "ambiguous_target" + ], + "retrieved_tools": [ + "issueSupplierActivationCertificate", + "completeSupplierOnboardingStage02", + "resolveSupplierTenant" + ], + "candidate_catalog_sha256": "80a5bfd6a1d5bb9d6015baba15a6e9cc10f5d67dfd0a6e817fbda1d4a6b01d36", + "model_response": { + "content": "{\"target_tool\":\"issueSupplierActivationCertificate\",\"supporting_tools\":[]}", + "input_tokens": 671, + "output_tokens": 18, + "latency_ms": 896.0726372897625, + "status_code": 200, + "finish_reason": "stop", + "error": "" + }, + "model_reason_codes": [], + "candidate_count": 3, + "planned_call_count": 3, + "executed_call_count": 3, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "resolveSupplierTenant", + "completeSupplierOnboardingStage02", + "issueSupplierActivationCertificate" + ], + "executed_tools": [ + "resolveSupplierTenant", + "completeSupplierOnboardingStage02", + "issueSupplierActivationCertificate" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "supplier_onboarding_3", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 3, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "resolveSupplierTenant" + ], + "observed": "resolveSupplierTenant" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completeSupplierOnboardingStage02" + ], + "observed": "completeSupplierOnboardingStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "issueSupplierActivationCertificate" + ], + "observed": "issueSupplierActivationCertificate" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-supplier_onboarding-01", + "observed": "EVIDENCE-supplier_onboarding-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-supplier_onboarding-02", + "observed": "EVIDENCE-supplier_onboarding-02" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.activationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-supplier_onboarding" + }, + "observed": "CERT-supplier_onboarding" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 3, + "observed": 3 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 3, + "observed": 3 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "retrieve": 93.581, + "model_select": 896.073, + "plan_execute": 5.589 + }, + "usage": { + "input_tokens": 671, + "output_tokens": 18, + "retrieval_token_budget_used": 190 + } + }, + { + "case_id": "incident_recovery_10::repeat-1", + "family": "incident_recovery", + "workflow_length": 10, + "catalog_size": 1000, + "paired_seed": 21, + "query": "서비스 장애 복구 절차를 모두 수행하고 복구 검증 보고서를 발급해줘", + "tool_count": 1000, + "edge_count": 9, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 991, + "produces_consumer_aligned": 0, + "consumes_added": 10, + "produces_skipped": 10, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 10, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 10, + "search": 990 + }, + "resource_counts": { + "archive": 990, + "incident_recovery": 10 + }, + "module_counts": { + "archive_references": 990, + "incident_recovery_stages": 10 + }, + "result_shape_counts": { + "list": 990, + "mutation": 10 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 990, + "rate": 0.99 + }, + { + "module": "incident_recovery_stages", + "count": 10, + "rate": 0.01 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 10, + "relation_count": 9, + "by_dependency_kind": { + "runtime_reference": 9 + }, + "edge_stats": { + "added": 9, + "merged": 0, + "binding_aliases_added": 9 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "42cbf08002ff711f27fda523ebfe3974c14eaa8e1ab48f8a8857e3d0ca8b31fc", + "bytes": 2308, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "llm_target": "issueIncidentRecoveryReport", + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target": "issueIncidentRecoveryReport", + "selected_target_exact": 1.0, + "selector_overrode_llm": false, + "selector_reason_codes": [ + "ambiguous_target" + ], + "retrieved_tools": [ + "issueIncidentRecoveryReport", + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04" + ], + "candidate_catalog_sha256": "d6a18ec50814891ca2010066e044da9a1a06f83d007bd11fe04a331dc234d3e0", + "model_response": { + "content": "{\"target_tool\":\"issueIncidentRecoveryReport\",\"supporting_tools\":[]}", + "input_tokens": 1030, + "output_tokens": 20, + "latency_ms": 1273.809753358364, + "status_code": 200, + "finish_reason": "stop", + "error": "" + }, + "model_reason_codes": [], + "candidate_count": 10, + "planned_call_count": 10, + "executed_call_count": 10, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04", + "completeIncidentRecoveryStage05", + "completeIncidentRecoveryStage06", + "completeIncidentRecoveryStage07", + "completeIncidentRecoveryStage08", + "completeIncidentRecoveryStage09", + "issueIncidentRecoveryReport" + ], + "executed_tools": [ + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04", + "completeIncidentRecoveryStage05", + "completeIncidentRecoveryStage06", + "completeIncidentRecoveryStage07", + "completeIncidentRecoveryStage08", + "completeIncidentRecoveryStage09", + "issueIncidentRecoveryReport" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "incident_recovery_10", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 10, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3, + "stage_04": 4, + "stage_05": 5, + "stage_06": 6, + "stage_07": 7, + "stage_08": 8, + "stage_09": 9, + "stage_10": 10 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openIncidentInvestigation" + ], + "observed": "openIncidentInvestigation" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completeIncidentRecoveryStage02" + ], + "observed": "completeIncidentRecoveryStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "completeIncidentRecoveryStage03" + ], + "observed": "completeIncidentRecoveryStage03" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_04", + "expected": [ + "completeIncidentRecoveryStage04" + ], + "observed": "completeIncidentRecoveryStage04" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_05", + "expected": [ + "completeIncidentRecoveryStage05" + ], + "observed": "completeIncidentRecoveryStage05" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_06", + "expected": [ + "completeIncidentRecoveryStage06" + ], + "observed": "completeIncidentRecoveryStage06" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_07", + "expected": [ + "completeIncidentRecoveryStage07" + ], + "observed": "completeIncidentRecoveryStage07" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_08", + "expected": [ + "completeIncidentRecoveryStage08" + ], + "observed": "completeIncidentRecoveryStage08" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_09", + "expected": [ + "completeIncidentRecoveryStage09" + ], + "observed": "completeIncidentRecoveryStage09" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_10", + "expected": [ + "issueIncidentRecoveryReport" + ], + "observed": "issueIncidentRecoveryReport" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": 3, + "after_sequence": 4 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": 4, + "after_sequence": 5 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": 5, + "after_sequence": 6 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": 6, + "after_sequence": 7 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": 7, + "after_sequence": 8 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": 8, + "after_sequence": 9 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": 9, + "after_sequence": 10 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-incident_recovery-01", + "observed": "EVIDENCE-incident_recovery-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-incident_recovery-02", + "observed": "EVIDENCE-incident_recovery-02" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "EVIDENCE-incident_recovery-03", + "observed": "EVIDENCE-incident_recovery-03" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "EVIDENCE-incident_recovery-04", + "observed": "EVIDENCE-incident_recovery-04" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "EVIDENCE-incident_recovery-05", + "observed": "EVIDENCE-incident_recovery-05" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "EVIDENCE-incident_recovery-06", + "observed": "EVIDENCE-incident_recovery-06" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "EVIDENCE-incident_recovery-07", + "observed": "EVIDENCE-incident_recovery-07" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "EVIDENCE-incident_recovery-08", + "observed": "EVIDENCE-incident_recovery-08" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "EVIDENCE-incident_recovery-09", + "observed": "EVIDENCE-incident_recovery-09" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.recoveryReportId", + "expected": { + "operator": "eq", + "value": "CERT-incident_recovery" + }, + "observed": "CERT-incident_recovery" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 10, + "observed": 10 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 10, + "observed": 10 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "retrieve": 99.252, + "model_select": 1273.81, + "plan_execute": 6.105 + }, + "usage": { + "input_tokens": 1030, + "output_tokens": 20, + "retrieval_token_budget_used": 291 + } + }, + { + "case_id": "publication_approval_30::repeat-1", + "family": "publication_approval", + "workflow_length": 30, + "catalog_size": 1000, + "paired_seed": 22, + "query": "콘텐츠 검토 절차를 모두 완료하고 게시 승인 증명서를 발급해줘", + "tool_count": 1000, + "edge_count": 29, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 971, + "produces_consumer_aligned": 0, + "consumes_added": 30, + "produces_skipped": 30, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 30, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 30, + "search": 970 + }, + "resource_counts": { + "archive": 970, + "publication_approval": 30 + }, + "module_counts": { + "archive_references": 970, + "publication_approval_stages": 30 + }, + "result_shape_counts": { + "list": 970, + "mutation": 30 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 970, + "rate": 0.97 + }, + { + "module": "publication_approval_stages", + "count": 30, + "rate": 0.03 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 30, + "relation_count": 29, + "by_dependency_kind": { + "runtime_reference": 29 + }, + "edge_stats": { + "added": 29, + "merged": 0, + "binding_aliases_added": 29 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "21a3ad4765fcfda8aea532b05c84a753829039e9d45653270a918c2deda939e1", + "bytes": 6709, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "llm_target": "issuePublicationApprovalCertificate", + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target": "issuePublicationApprovalCertificate", + "selected_target_exact": 1.0, + "selector_overrode_llm": false, + "selector_reason_codes": [ + "ambiguous_target" + ], + "retrieved_tools": [ + "issuePublicationApprovalCertificate", + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04" + ], + "candidate_catalog_sha256": "0a64e7719a81b3885f0837c2987c98460066571e16195dc411cebdead3c4cd41", + "model_response": { + "content": "{\"target_tool\":\"issuePublicationApprovalCertificate\",\"supporting_tools\":[]}", + "input_tokens": 1040, + "output_tokens": 18, + "latency_ms": 1174.284292384982, + "status_code": 200, + "finish_reason": "stop", + "error": "" + }, + "model_reason_codes": [], + "candidate_count": 30, + "planned_call_count": 30, + "executed_call_count": 30, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04", + "completePublicationReviewStage05", + "completePublicationReviewStage06", + "completePublicationReviewStage07", + "completePublicationReviewStage08", + "completePublicationReviewStage09", + "completePublicationReviewStage10", + "completePublicationReviewStage11", + "completePublicationReviewStage12", + "completePublicationReviewStage13", + "completePublicationReviewStage14", + "completePublicationReviewStage15", + "completePublicationReviewStage16", + "completePublicationReviewStage17", + "completePublicationReviewStage18", + "completePublicationReviewStage19", + "completePublicationReviewStage20", + "completePublicationReviewStage21", + "completePublicationReviewStage22", + "completePublicationReviewStage23", + "completePublicationReviewStage24", + "completePublicationReviewStage25", + "completePublicationReviewStage26", + "completePublicationReviewStage27", + "completePublicationReviewStage28", + "completePublicationReviewStage29", + "issuePublicationApprovalCertificate" + ], + "executed_tools": [ + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04", + "completePublicationReviewStage05", + "completePublicationReviewStage06", + "completePublicationReviewStage07", + "completePublicationReviewStage08", + "completePublicationReviewStage09", + "completePublicationReviewStage10", + "completePublicationReviewStage11", + "completePublicationReviewStage12", + "completePublicationReviewStage13", + "completePublicationReviewStage14", + "completePublicationReviewStage15", + "completePublicationReviewStage16", + "completePublicationReviewStage17", + "completePublicationReviewStage18", + "completePublicationReviewStage19", + "completePublicationReviewStage20", + "completePublicationReviewStage21", + "completePublicationReviewStage22", + "completePublicationReviewStage23", + "completePublicationReviewStage24", + "completePublicationReviewStage25", + "completePublicationReviewStage26", + "completePublicationReviewStage27", + "completePublicationReviewStage28", + "completePublicationReviewStage29", + "issuePublicationApprovalCertificate" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "publication_approval_30", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 30, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3, + "stage_04": 4, + "stage_05": 5, + "stage_06": 6, + "stage_07": 7, + "stage_08": 8, + "stage_09": 9, + "stage_10": 10, + "stage_11": 11, + "stage_12": 12, + "stage_13": 13, + "stage_14": 14, + "stage_15": 15, + "stage_16": 16, + "stage_17": 17, + "stage_18": 18, + "stage_19": 19, + "stage_20": 20, + "stage_21": 21, + "stage_22": 22, + "stage_23": 23, + "stage_24": 24, + "stage_25": 25, + "stage_26": 26, + "stage_27": 27, + "stage_28": 28, + "stage_29": 29, + "stage_30": 30 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openPublicationReview" + ], + "observed": "openPublicationReview" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completePublicationReviewStage02" + ], + "observed": "completePublicationReviewStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "completePublicationReviewStage03" + ], + "observed": "completePublicationReviewStage03" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_04", + "expected": [ + "completePublicationReviewStage04" + ], + "observed": "completePublicationReviewStage04" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_05", + "expected": [ + "completePublicationReviewStage05" + ], + "observed": "completePublicationReviewStage05" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_06", + "expected": [ + "completePublicationReviewStage06" + ], + "observed": "completePublicationReviewStage06" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_07", + "expected": [ + "completePublicationReviewStage07" + ], + "observed": "completePublicationReviewStage07" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_08", + "expected": [ + "completePublicationReviewStage08" + ], + "observed": "completePublicationReviewStage08" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_09", + "expected": [ + "completePublicationReviewStage09" + ], + "observed": "completePublicationReviewStage09" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_10", + "expected": [ + "completePublicationReviewStage10" + ], + "observed": "completePublicationReviewStage10" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_11", + "expected": [ + "completePublicationReviewStage11" + ], + "observed": "completePublicationReviewStage11" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_12", + "expected": [ + "completePublicationReviewStage12" + ], + "observed": "completePublicationReviewStage12" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_13", + "expected": [ + "completePublicationReviewStage13" + ], + "observed": "completePublicationReviewStage13" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_14", + "expected": [ + "completePublicationReviewStage14" + ], + "observed": "completePublicationReviewStage14" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_15", + "expected": [ + "completePublicationReviewStage15" + ], + "observed": "completePublicationReviewStage15" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_16", + "expected": [ + "completePublicationReviewStage16" + ], + "observed": "completePublicationReviewStage16" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_17", + "expected": [ + "completePublicationReviewStage17" + ], + "observed": "completePublicationReviewStage17" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_18", + "expected": [ + "completePublicationReviewStage18" + ], + "observed": "completePublicationReviewStage18" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_19", + "expected": [ + "completePublicationReviewStage19" + ], + "observed": "completePublicationReviewStage19" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_20", + "expected": [ + "completePublicationReviewStage20" + ], + "observed": "completePublicationReviewStage20" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_21", + "expected": [ + "completePublicationReviewStage21" + ], + "observed": "completePublicationReviewStage21" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_22", + "expected": [ + "completePublicationReviewStage22" + ], + "observed": "completePublicationReviewStage22" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_23", + "expected": [ + "completePublicationReviewStage23" + ], + "observed": "completePublicationReviewStage23" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_24", + "expected": [ + "completePublicationReviewStage24" + ], + "observed": "completePublicationReviewStage24" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_25", + "expected": [ + "completePublicationReviewStage25" + ], + "observed": "completePublicationReviewStage25" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_26", + "expected": [ + "completePublicationReviewStage26" + ], + "observed": "completePublicationReviewStage26" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_27", + "expected": [ + "completePublicationReviewStage27" + ], + "observed": "completePublicationReviewStage27" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_28", + "expected": [ + "completePublicationReviewStage28" + ], + "observed": "completePublicationReviewStage28" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_29", + "expected": [ + "completePublicationReviewStage29" + ], + "observed": "completePublicationReviewStage29" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_30", + "expected": [ + "issuePublicationApprovalCertificate" + ], + "observed": "issuePublicationApprovalCertificate" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": 3, + "after_sequence": 4 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": 4, + "after_sequence": 5 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": 5, + "after_sequence": 6 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": 6, + "after_sequence": 7 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": 7, + "after_sequence": 8 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": 8, + "after_sequence": 9 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": 9, + "after_sequence": 10 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_10->stage_11", + "expected": "before", + "observed": { + "before_sequence": 10, + "after_sequence": 11 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_11->stage_12", + "expected": "before", + "observed": { + "before_sequence": 11, + "after_sequence": 12 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_12->stage_13", + "expected": "before", + "observed": { + "before_sequence": 12, + "after_sequence": 13 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_13->stage_14", + "expected": "before", + "observed": { + "before_sequence": 13, + "after_sequence": 14 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_14->stage_15", + "expected": "before", + "observed": { + "before_sequence": 14, + "after_sequence": 15 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_15->stage_16", + "expected": "before", + "observed": { + "before_sequence": 15, + "after_sequence": 16 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_16->stage_17", + "expected": "before", + "observed": { + "before_sequence": 16, + "after_sequence": 17 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_17->stage_18", + "expected": "before", + "observed": { + "before_sequence": 17, + "after_sequence": 18 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_18->stage_19", + "expected": "before", + "observed": { + "before_sequence": 18, + "after_sequence": 19 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_19->stage_20", + "expected": "before", + "observed": { + "before_sequence": 19, + "after_sequence": 20 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_20->stage_21", + "expected": "before", + "observed": { + "before_sequence": 20, + "after_sequence": 21 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_21->stage_22", + "expected": "before", + "observed": { + "before_sequence": 21, + "after_sequence": 22 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_22->stage_23", + "expected": "before", + "observed": { + "before_sequence": 22, + "after_sequence": 23 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_23->stage_24", + "expected": "before", + "observed": { + "before_sequence": 23, + "after_sequence": 24 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_24->stage_25", + "expected": "before", + "observed": { + "before_sequence": 24, + "after_sequence": 25 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_25->stage_26", + "expected": "before", + "observed": { + "before_sequence": 25, + "after_sequence": 26 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_26->stage_27", + "expected": "before", + "observed": { + "before_sequence": 26, + "after_sequence": 27 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_27->stage_28", + "expected": "before", + "observed": { + "before_sequence": 27, + "after_sequence": 28 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_28->stage_29", + "expected": "before", + "observed": { + "before_sequence": 28, + "after_sequence": 29 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_29->stage_30", + "expected": "before", + "observed": { + "before_sequence": 29, + "after_sequence": 30 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-publication_approval-01", + "observed": "EVIDENCE-publication_approval-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-publication_approval-02", + "observed": "EVIDENCE-publication_approval-02" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "EVIDENCE-publication_approval-03", + "observed": "EVIDENCE-publication_approval-03" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "EVIDENCE-publication_approval-04", + "observed": "EVIDENCE-publication_approval-04" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "EVIDENCE-publication_approval-05", + "observed": "EVIDENCE-publication_approval-05" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "EVIDENCE-publication_approval-06", + "observed": "EVIDENCE-publication_approval-06" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "EVIDENCE-publication_approval-07", + "observed": "EVIDENCE-publication_approval-07" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "EVIDENCE-publication_approval-08", + "observed": "EVIDENCE-publication_approval-08" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "EVIDENCE-publication_approval-09", + "observed": "EVIDENCE-publication_approval-09" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_10.stageEvidence10->stage_11.handoffToken10", + "expected": "EVIDENCE-publication_approval-10", + "observed": "EVIDENCE-publication_approval-10" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_11.stageEvidence11->stage_12.handoffToken11", + "expected": "EVIDENCE-publication_approval-11", + "observed": "EVIDENCE-publication_approval-11" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_12.stageEvidence12->stage_13.handoffToken12", + "expected": "EVIDENCE-publication_approval-12", + "observed": "EVIDENCE-publication_approval-12" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_13.stageEvidence13->stage_14.handoffToken13", + "expected": "EVIDENCE-publication_approval-13", + "observed": "EVIDENCE-publication_approval-13" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_14.stageEvidence14->stage_15.handoffToken14", + "expected": "EVIDENCE-publication_approval-14", + "observed": "EVIDENCE-publication_approval-14" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_15.stageEvidence15->stage_16.handoffToken15", + "expected": "EVIDENCE-publication_approval-15", + "observed": "EVIDENCE-publication_approval-15" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_16.stageEvidence16->stage_17.handoffToken16", + "expected": "EVIDENCE-publication_approval-16", + "observed": "EVIDENCE-publication_approval-16" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_17.stageEvidence17->stage_18.handoffToken17", + "expected": "EVIDENCE-publication_approval-17", + "observed": "EVIDENCE-publication_approval-17" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_18.stageEvidence18->stage_19.handoffToken18", + "expected": "EVIDENCE-publication_approval-18", + "observed": "EVIDENCE-publication_approval-18" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_19.stageEvidence19->stage_20.handoffToken19", + "expected": "EVIDENCE-publication_approval-19", + "observed": "EVIDENCE-publication_approval-19" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_20.stageEvidence20->stage_21.handoffToken20", + "expected": "EVIDENCE-publication_approval-20", + "observed": "EVIDENCE-publication_approval-20" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_21.stageEvidence21->stage_22.handoffToken21", + "expected": "EVIDENCE-publication_approval-21", + "observed": "EVIDENCE-publication_approval-21" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_22.stageEvidence22->stage_23.handoffToken22", + "expected": "EVIDENCE-publication_approval-22", + "observed": "EVIDENCE-publication_approval-22" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_23.stageEvidence23->stage_24.handoffToken23", + "expected": "EVIDENCE-publication_approval-23", + "observed": "EVIDENCE-publication_approval-23" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_24.stageEvidence24->stage_25.handoffToken24", + "expected": "EVIDENCE-publication_approval-24", + "observed": "EVIDENCE-publication_approval-24" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_25.stageEvidence25->stage_26.handoffToken25", + "expected": "EVIDENCE-publication_approval-25", + "observed": "EVIDENCE-publication_approval-25" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_26.stageEvidence26->stage_27.handoffToken26", + "expected": "EVIDENCE-publication_approval-26", + "observed": "EVIDENCE-publication_approval-26" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_27.stageEvidence27->stage_28.handoffToken27", + "expected": "EVIDENCE-publication_approval-27", + "observed": "EVIDENCE-publication_approval-27" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_28.stageEvidence28->stage_29.handoffToken28", + "expected": "EVIDENCE-publication_approval-28", + "observed": "EVIDENCE-publication_approval-28" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_29.stageEvidence29->stage_30.handoffToken29", + "expected": "EVIDENCE-publication_approval-29", + "observed": "EVIDENCE-publication_approval-29" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.publicationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-publication_approval" + }, + "observed": "CERT-publication_approval" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 30, + "observed": 30 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 30, + "observed": 30 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "retrieve": 116.226, + "model_select": 1174.284, + "plan_execute": 7.357 + }, + "usage": { + "input_tokens": 1040, + "output_tokens": 18, + "retrieval_token_budget_used": 301 + } + }, + { + "case_id": "supplier_onboarding_3::repeat-2", + "family": "supplier_onboarding", + "workflow_length": 3, + "catalog_size": 1000, + "paired_seed": 23, + "query": "공급업체 등록 절차를 모두 완료하고 활성화 인증서를 발급해줘", + "tool_count": 1000, + "edge_count": 2, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 998, + "produces_consumer_aligned": 0, + "consumes_added": 3, + "produces_skipped": 3, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 3, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 2, + "create": 1, + "search": 997 + }, + "resource_counts": { + "archive": 997, + "supplier_onboarding": 3 + }, + "module_counts": { + "archive_references": 997, + "supplier_onboarding_stages": 3 + }, + "result_shape_counts": { + "list": 997, + "mutation": 3 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 997, + "rate": 0.997 + }, + { + "module": "supplier_onboarding_stages", + "count": 3, + "rate": 0.003 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 3, + "relation_count": 2, + "by_dependency_kind": { + "runtime_reference": 2 + }, + "edge_stats": { + "added": 2, + "merged": 0, + "binding_aliases_added": 2 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "08929f2c7d36ff7b0036617930b6636dcfda90ca514a2f1207564c01acfd317a", + "bytes": 793, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "llm_target": "issueSupplierActivationCertificate", + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target": "issueSupplierActivationCertificate", + "selected_target_exact": 1.0, + "selector_overrode_llm": false, + "selector_reason_codes": [ + "ambiguous_target" + ], + "retrieved_tools": [ + "issueSupplierActivationCertificate", + "completeSupplierOnboardingStage02", + "resolveSupplierTenant" + ], + "candidate_catalog_sha256": "80a5bfd6a1d5bb9d6015baba15a6e9cc10f5d67dfd0a6e817fbda1d4a6b01d36", + "model_response": { + "content": "{\"target_tool\":\"issueSupplierActivationCertificate\",\"supporting_tools\":[]}", + "input_tokens": 671, + "output_tokens": 18, + "latency_ms": 1208.362327888608, + "status_code": 200, + "finish_reason": "stop", + "error": "" + }, + "model_reason_codes": [], + "candidate_count": 3, + "planned_call_count": 3, + "executed_call_count": 3, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "resolveSupplierTenant", + "completeSupplierOnboardingStage02", + "issueSupplierActivationCertificate" + ], + "executed_tools": [ + "resolveSupplierTenant", + "completeSupplierOnboardingStage02", + "issueSupplierActivationCertificate" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "supplier_onboarding_3", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 3, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "resolveSupplierTenant" + ], + "observed": "resolveSupplierTenant" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completeSupplierOnboardingStage02" + ], + "observed": "completeSupplierOnboardingStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "issueSupplierActivationCertificate" + ], + "observed": "issueSupplierActivationCertificate" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-supplier_onboarding-01", + "observed": "EVIDENCE-supplier_onboarding-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-supplier_onboarding-02", + "observed": "EVIDENCE-supplier_onboarding-02" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.activationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-supplier_onboarding" + }, + "observed": "CERT-supplier_onboarding" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 3, + "observed": 3 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 3, + "observed": 3 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "retrieve": 92.453, + "model_select": 1208.362, + "plan_execute": 6.067 + }, + "usage": { + "input_tokens": 671, + "output_tokens": 18, + "retrieval_token_budget_used": 190 + } + }, + { + "case_id": "incident_recovery_10::repeat-2", + "family": "incident_recovery", + "workflow_length": 10, + "catalog_size": 1000, + "paired_seed": 24, + "query": "서비스 장애 복구 절차를 모두 수행하고 복구 검증 보고서를 발급해줘", + "tool_count": 1000, + "edge_count": 9, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 991, + "produces_consumer_aligned": 0, + "consumes_added": 10, + "produces_skipped": 10, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 10, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 10, + "search": 990 + }, + "resource_counts": { + "archive": 990, + "incident_recovery": 10 + }, + "module_counts": { + "archive_references": 990, + "incident_recovery_stages": 10 + }, + "result_shape_counts": { + "list": 990, + "mutation": 10 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 990, + "rate": 0.99 + }, + { + "module": "incident_recovery_stages", + "count": 10, + "rate": 0.01 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 10, + "relation_count": 9, + "by_dependency_kind": { + "runtime_reference": 9 + }, + "edge_stats": { + "added": 9, + "merged": 0, + "binding_aliases_added": 9 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "42cbf08002ff711f27fda523ebfe3974c14eaa8e1ab48f8a8857e3d0ca8b31fc", + "bytes": 2308, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "llm_target": "issueIncidentRecoveryReport", + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target": "issueIncidentRecoveryReport", + "selected_target_exact": 1.0, + "selector_overrode_llm": false, + "selector_reason_codes": [ + "ambiguous_target" + ], + "retrieved_tools": [ + "issueIncidentRecoveryReport", + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04" + ], + "candidate_catalog_sha256": "d6a18ec50814891ca2010066e044da9a1a06f83d007bd11fe04a331dc234d3e0", + "model_response": { + "content": "{\"target_tool\":\"issueIncidentRecoveryReport\",\"supporting_tools\":[]}", + "input_tokens": 1030, + "output_tokens": 20, + "latency_ms": 1144.292825832963, + "status_code": 200, + "finish_reason": "stop", + "error": "" + }, + "model_reason_codes": [], + "candidate_count": 10, + "planned_call_count": 10, + "executed_call_count": 10, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04", + "completeIncidentRecoveryStage05", + "completeIncidentRecoveryStage06", + "completeIncidentRecoveryStage07", + "completeIncidentRecoveryStage08", + "completeIncidentRecoveryStage09", + "issueIncidentRecoveryReport" + ], + "executed_tools": [ + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04", + "completeIncidentRecoveryStage05", + "completeIncidentRecoveryStage06", + "completeIncidentRecoveryStage07", + "completeIncidentRecoveryStage08", + "completeIncidentRecoveryStage09", + "issueIncidentRecoveryReport" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "incident_recovery_10", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 10, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3, + "stage_04": 4, + "stage_05": 5, + "stage_06": 6, + "stage_07": 7, + "stage_08": 8, + "stage_09": 9, + "stage_10": 10 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openIncidentInvestigation" + ], + "observed": "openIncidentInvestigation" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completeIncidentRecoveryStage02" + ], + "observed": "completeIncidentRecoveryStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "completeIncidentRecoveryStage03" + ], + "observed": "completeIncidentRecoveryStage03" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_04", + "expected": [ + "completeIncidentRecoveryStage04" + ], + "observed": "completeIncidentRecoveryStage04" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_05", + "expected": [ + "completeIncidentRecoveryStage05" + ], + "observed": "completeIncidentRecoveryStage05" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_06", + "expected": [ + "completeIncidentRecoveryStage06" + ], + "observed": "completeIncidentRecoveryStage06" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_07", + "expected": [ + "completeIncidentRecoveryStage07" + ], + "observed": "completeIncidentRecoveryStage07" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_08", + "expected": [ + "completeIncidentRecoveryStage08" + ], + "observed": "completeIncidentRecoveryStage08" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_09", + "expected": [ + "completeIncidentRecoveryStage09" + ], + "observed": "completeIncidentRecoveryStage09" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_10", + "expected": [ + "issueIncidentRecoveryReport" + ], + "observed": "issueIncidentRecoveryReport" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": 3, + "after_sequence": 4 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": 4, + "after_sequence": 5 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": 5, + "after_sequence": 6 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": 6, + "after_sequence": 7 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": 7, + "after_sequence": 8 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": 8, + "after_sequence": 9 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": 9, + "after_sequence": 10 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-incident_recovery-01", + "observed": "EVIDENCE-incident_recovery-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-incident_recovery-02", + "observed": "EVIDENCE-incident_recovery-02" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "EVIDENCE-incident_recovery-03", + "observed": "EVIDENCE-incident_recovery-03" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "EVIDENCE-incident_recovery-04", + "observed": "EVIDENCE-incident_recovery-04" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "EVIDENCE-incident_recovery-05", + "observed": "EVIDENCE-incident_recovery-05" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "EVIDENCE-incident_recovery-06", + "observed": "EVIDENCE-incident_recovery-06" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "EVIDENCE-incident_recovery-07", + "observed": "EVIDENCE-incident_recovery-07" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "EVIDENCE-incident_recovery-08", + "observed": "EVIDENCE-incident_recovery-08" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "EVIDENCE-incident_recovery-09", + "observed": "EVIDENCE-incident_recovery-09" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.recoveryReportId", + "expected": { + "operator": "eq", + "value": "CERT-incident_recovery" + }, + "observed": "CERT-incident_recovery" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 10, + "observed": 10 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 10, + "observed": 10 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "retrieve": 94.529, + "model_select": 1144.293, + "plan_execute": 6.256 + }, + "usage": { + "input_tokens": 1030, + "output_tokens": 20, + "retrieval_token_budget_used": 291 + } + }, + { + "case_id": "publication_approval_30::repeat-2", + "family": "publication_approval", + "workflow_length": 30, + "catalog_size": 1000, + "paired_seed": 25, + "query": "콘텐츠 검토 절차를 모두 완료하고 게시 승인 증명서를 발급해줘", + "tool_count": 1000, + "edge_count": 29, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 971, + "produces_consumer_aligned": 0, + "consumes_added": 30, + "produces_skipped": 30, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 30, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 30, + "search": 970 + }, + "resource_counts": { + "archive": 970, + "publication_approval": 30 + }, + "module_counts": { + "archive_references": 970, + "publication_approval_stages": 30 + }, + "result_shape_counts": { + "list": 970, + "mutation": 30 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 970, + "rate": 0.97 + }, + { + "module": "publication_approval_stages", + "count": 30, + "rate": 0.03 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 30, + "relation_count": 29, + "by_dependency_kind": { + "runtime_reference": 29 + }, + "edge_stats": { + "added": 29, + "merged": 0, + "binding_aliases_added": 29 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "21a3ad4765fcfda8aea532b05c84a753829039e9d45653270a918c2deda939e1", + "bytes": 6709, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "llm_target": "issuePublicationApprovalCertificate", + "llm_target_exact": 1.0, + "llm_target_in_catalog": 1.0, + "selected_target": "issuePublicationApprovalCertificate", + "selected_target_exact": 1.0, + "selector_overrode_llm": false, + "selector_reason_codes": [ + "ambiguous_target" + ], + "retrieved_tools": [ + "issuePublicationApprovalCertificate", + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04" + ], + "candidate_catalog_sha256": "0a64e7719a81b3885f0837c2987c98460066571e16195dc411cebdead3c4cd41", + "model_response": { + "content": "{\"target_tool\":\"issuePublicationApprovalCertificate\",\"supporting_tools\":[]}", + "input_tokens": 1040, + "output_tokens": 18, + "latency_ms": 1220.2402967959642, + "status_code": 200, + "finish_reason": "stop", + "error": "" + }, + "model_reason_codes": [], + "candidate_count": 30, + "planned_call_count": 30, + "executed_call_count": 30, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04", + "completePublicationReviewStage05", + "completePublicationReviewStage06", + "completePublicationReviewStage07", + "completePublicationReviewStage08", + "completePublicationReviewStage09", + "completePublicationReviewStage10", + "completePublicationReviewStage11", + "completePublicationReviewStage12", + "completePublicationReviewStage13", + "completePublicationReviewStage14", + "completePublicationReviewStage15", + "completePublicationReviewStage16", + "completePublicationReviewStage17", + "completePublicationReviewStage18", + "completePublicationReviewStage19", + "completePublicationReviewStage20", + "completePublicationReviewStage21", + "completePublicationReviewStage22", + "completePublicationReviewStage23", + "completePublicationReviewStage24", + "completePublicationReviewStage25", + "completePublicationReviewStage26", + "completePublicationReviewStage27", + "completePublicationReviewStage28", + "completePublicationReviewStage29", + "issuePublicationApprovalCertificate" + ], + "executed_tools": [ + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04", + "completePublicationReviewStage05", + "completePublicationReviewStage06", + "completePublicationReviewStage07", + "completePublicationReviewStage08", + "completePublicationReviewStage09", + "completePublicationReviewStage10", + "completePublicationReviewStage11", + "completePublicationReviewStage12", + "completePublicationReviewStage13", + "completePublicationReviewStage14", + "completePublicationReviewStage15", + "completePublicationReviewStage16", + "completePublicationReviewStage17", + "completePublicationReviewStage18", + "completePublicationReviewStage19", + "completePublicationReviewStage20", + "completePublicationReviewStage21", + "completePublicationReviewStage22", + "completePublicationReviewStage23", + "completePublicationReviewStage24", + "completePublicationReviewStage25", + "completePublicationReviewStage26", + "completePublicationReviewStage27", + "completePublicationReviewStage28", + "completePublicationReviewStage29", + "issuePublicationApprovalCertificate" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "publication_approval_30", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 30, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3, + "stage_04": 4, + "stage_05": 5, + "stage_06": 6, + "stage_07": 7, + "stage_08": 8, + "stage_09": 9, + "stage_10": 10, + "stage_11": 11, + "stage_12": 12, + "stage_13": 13, + "stage_14": 14, + "stage_15": 15, + "stage_16": 16, + "stage_17": 17, + "stage_18": 18, + "stage_19": 19, + "stage_20": 20, + "stage_21": 21, + "stage_22": 22, + "stage_23": 23, + "stage_24": 24, + "stage_25": 25, + "stage_26": 26, + "stage_27": 27, + "stage_28": 28, + "stage_29": 29, + "stage_30": 30 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openPublicationReview" + ], + "observed": "openPublicationReview" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completePublicationReviewStage02" + ], + "observed": "completePublicationReviewStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "completePublicationReviewStage03" + ], + "observed": "completePublicationReviewStage03" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_04", + "expected": [ + "completePublicationReviewStage04" + ], + "observed": "completePublicationReviewStage04" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_05", + "expected": [ + "completePublicationReviewStage05" + ], + "observed": "completePublicationReviewStage05" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_06", + "expected": [ + "completePublicationReviewStage06" + ], + "observed": "completePublicationReviewStage06" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_07", + "expected": [ + "completePublicationReviewStage07" + ], + "observed": "completePublicationReviewStage07" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_08", + "expected": [ + "completePublicationReviewStage08" + ], + "observed": "completePublicationReviewStage08" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_09", + "expected": [ + "completePublicationReviewStage09" + ], + "observed": "completePublicationReviewStage09" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_10", + "expected": [ + "completePublicationReviewStage10" + ], + "observed": "completePublicationReviewStage10" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_11", + "expected": [ + "completePublicationReviewStage11" + ], + "observed": "completePublicationReviewStage11" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_12", + "expected": [ + "completePublicationReviewStage12" + ], + "observed": "completePublicationReviewStage12" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_13", + "expected": [ + "completePublicationReviewStage13" + ], + "observed": "completePublicationReviewStage13" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_14", + "expected": [ + "completePublicationReviewStage14" + ], + "observed": "completePublicationReviewStage14" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_15", + "expected": [ + "completePublicationReviewStage15" + ], + "observed": "completePublicationReviewStage15" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_16", + "expected": [ + "completePublicationReviewStage16" + ], + "observed": "completePublicationReviewStage16" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_17", + "expected": [ + "completePublicationReviewStage17" + ], + "observed": "completePublicationReviewStage17" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_18", + "expected": [ + "completePublicationReviewStage18" + ], + "observed": "completePublicationReviewStage18" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_19", + "expected": [ + "completePublicationReviewStage19" + ], + "observed": "completePublicationReviewStage19" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_20", + "expected": [ + "completePublicationReviewStage20" + ], + "observed": "completePublicationReviewStage20" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_21", + "expected": [ + "completePublicationReviewStage21" + ], + "observed": "completePublicationReviewStage21" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_22", + "expected": [ + "completePublicationReviewStage22" + ], + "observed": "completePublicationReviewStage22" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_23", + "expected": [ + "completePublicationReviewStage23" + ], + "observed": "completePublicationReviewStage23" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_24", + "expected": [ + "completePublicationReviewStage24" + ], + "observed": "completePublicationReviewStage24" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_25", + "expected": [ + "completePublicationReviewStage25" + ], + "observed": "completePublicationReviewStage25" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_26", + "expected": [ + "completePublicationReviewStage26" + ], + "observed": "completePublicationReviewStage26" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_27", + "expected": [ + "completePublicationReviewStage27" + ], + "observed": "completePublicationReviewStage27" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_28", + "expected": [ + "completePublicationReviewStage28" + ], + "observed": "completePublicationReviewStage28" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_29", + "expected": [ + "completePublicationReviewStage29" + ], + "observed": "completePublicationReviewStage29" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_30", + "expected": [ + "issuePublicationApprovalCertificate" + ], + "observed": "issuePublicationApprovalCertificate" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": 3, + "after_sequence": 4 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": 4, + "after_sequence": 5 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": 5, + "after_sequence": 6 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": 6, + "after_sequence": 7 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": 7, + "after_sequence": 8 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": 8, + "after_sequence": 9 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": 9, + "after_sequence": 10 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_10->stage_11", + "expected": "before", + "observed": { + "before_sequence": 10, + "after_sequence": 11 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_11->stage_12", + "expected": "before", + "observed": { + "before_sequence": 11, + "after_sequence": 12 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_12->stage_13", + "expected": "before", + "observed": { + "before_sequence": 12, + "after_sequence": 13 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_13->stage_14", + "expected": "before", + "observed": { + "before_sequence": 13, + "after_sequence": 14 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_14->stage_15", + "expected": "before", + "observed": { + "before_sequence": 14, + "after_sequence": 15 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_15->stage_16", + "expected": "before", + "observed": { + "before_sequence": 15, + "after_sequence": 16 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_16->stage_17", + "expected": "before", + "observed": { + "before_sequence": 16, + "after_sequence": 17 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_17->stage_18", + "expected": "before", + "observed": { + "before_sequence": 17, + "after_sequence": 18 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_18->stage_19", + "expected": "before", + "observed": { + "before_sequence": 18, + "after_sequence": 19 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_19->stage_20", + "expected": "before", + "observed": { + "before_sequence": 19, + "after_sequence": 20 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_20->stage_21", + "expected": "before", + "observed": { + "before_sequence": 20, + "after_sequence": 21 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_21->stage_22", + "expected": "before", + "observed": { + "before_sequence": 21, + "after_sequence": 22 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_22->stage_23", + "expected": "before", + "observed": { + "before_sequence": 22, + "after_sequence": 23 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_23->stage_24", + "expected": "before", + "observed": { + "before_sequence": 23, + "after_sequence": 24 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_24->stage_25", + "expected": "before", + "observed": { + "before_sequence": 24, + "after_sequence": 25 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_25->stage_26", + "expected": "before", + "observed": { + "before_sequence": 25, + "after_sequence": 26 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_26->stage_27", + "expected": "before", + "observed": { + "before_sequence": 26, + "after_sequence": 27 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_27->stage_28", + "expected": "before", + "observed": { + "before_sequence": 27, + "after_sequence": 28 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_28->stage_29", + "expected": "before", + "observed": { + "before_sequence": 28, + "after_sequence": 29 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_29->stage_30", + "expected": "before", + "observed": { + "before_sequence": 29, + "after_sequence": 30 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-publication_approval-01", + "observed": "EVIDENCE-publication_approval-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-publication_approval-02", + "observed": "EVIDENCE-publication_approval-02" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "EVIDENCE-publication_approval-03", + "observed": "EVIDENCE-publication_approval-03" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "EVIDENCE-publication_approval-04", + "observed": "EVIDENCE-publication_approval-04" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "EVIDENCE-publication_approval-05", + "observed": "EVIDENCE-publication_approval-05" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "EVIDENCE-publication_approval-06", + "observed": "EVIDENCE-publication_approval-06" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "EVIDENCE-publication_approval-07", + "observed": "EVIDENCE-publication_approval-07" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "EVIDENCE-publication_approval-08", + "observed": "EVIDENCE-publication_approval-08" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "EVIDENCE-publication_approval-09", + "observed": "EVIDENCE-publication_approval-09" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_10.stageEvidence10->stage_11.handoffToken10", + "expected": "EVIDENCE-publication_approval-10", + "observed": "EVIDENCE-publication_approval-10" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_11.stageEvidence11->stage_12.handoffToken11", + "expected": "EVIDENCE-publication_approval-11", + "observed": "EVIDENCE-publication_approval-11" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_12.stageEvidence12->stage_13.handoffToken12", + "expected": "EVIDENCE-publication_approval-12", + "observed": "EVIDENCE-publication_approval-12" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_13.stageEvidence13->stage_14.handoffToken13", + "expected": "EVIDENCE-publication_approval-13", + "observed": "EVIDENCE-publication_approval-13" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_14.stageEvidence14->stage_15.handoffToken14", + "expected": "EVIDENCE-publication_approval-14", + "observed": "EVIDENCE-publication_approval-14" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_15.stageEvidence15->stage_16.handoffToken15", + "expected": "EVIDENCE-publication_approval-15", + "observed": "EVIDENCE-publication_approval-15" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_16.stageEvidence16->stage_17.handoffToken16", + "expected": "EVIDENCE-publication_approval-16", + "observed": "EVIDENCE-publication_approval-16" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_17.stageEvidence17->stage_18.handoffToken17", + "expected": "EVIDENCE-publication_approval-17", + "observed": "EVIDENCE-publication_approval-17" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_18.stageEvidence18->stage_19.handoffToken18", + "expected": "EVIDENCE-publication_approval-18", + "observed": "EVIDENCE-publication_approval-18" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_19.stageEvidence19->stage_20.handoffToken19", + "expected": "EVIDENCE-publication_approval-19", + "observed": "EVIDENCE-publication_approval-19" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_20.stageEvidence20->stage_21.handoffToken20", + "expected": "EVIDENCE-publication_approval-20", + "observed": "EVIDENCE-publication_approval-20" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_21.stageEvidence21->stage_22.handoffToken21", + "expected": "EVIDENCE-publication_approval-21", + "observed": "EVIDENCE-publication_approval-21" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_22.stageEvidence22->stage_23.handoffToken22", + "expected": "EVIDENCE-publication_approval-22", + "observed": "EVIDENCE-publication_approval-22" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_23.stageEvidence23->stage_24.handoffToken23", + "expected": "EVIDENCE-publication_approval-23", + "observed": "EVIDENCE-publication_approval-23" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_24.stageEvidence24->stage_25.handoffToken24", + "expected": "EVIDENCE-publication_approval-24", + "observed": "EVIDENCE-publication_approval-24" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_25.stageEvidence25->stage_26.handoffToken25", + "expected": "EVIDENCE-publication_approval-25", + "observed": "EVIDENCE-publication_approval-25" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_26.stageEvidence26->stage_27.handoffToken26", + "expected": "EVIDENCE-publication_approval-26", + "observed": "EVIDENCE-publication_approval-26" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_27.stageEvidence27->stage_28.handoffToken27", + "expected": "EVIDENCE-publication_approval-27", + "observed": "EVIDENCE-publication_approval-27" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_28.stageEvidence28->stage_29.handoffToken28", + "expected": "EVIDENCE-publication_approval-28", + "observed": "EVIDENCE-publication_approval-28" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_29.stageEvidence29->stage_30.handoffToken29", + "expected": "EVIDENCE-publication_approval-29", + "observed": "EVIDENCE-publication_approval-29" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.publicationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-publication_approval" + }, + "observed": "CERT-publication_approval" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 30, + "observed": 30 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 30, + "observed": 30 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "retrieve": 116.369, + "model_select": 1220.24, + "plan_execute": 7.215 + }, + "usage": { + "input_tokens": 1040, + "output_tokens": 18, + "retrieval_token_budget_used": 301 + } + } + ], + "limitations": [ + "The model selects the final target from graph-tool-call's retrieved Top-K; it is not shown evaluator gold or the complete 1,000-tool catalog.", + "Arazzo supplies the prerequisite order and runtime bindings after target selection; the model does not independently enumerate all 30 calls.", + "Generated catalogs are contract-distinct synthetic fixtures, not production APIs." + ] +} diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 41d1ebf..4d3bd78 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1176,3 +1176,52 @@ binding accuracy, and final goal completion. The result is recorded in This benchmark measures graph-tool-call's deterministic middleware. It does not claim that a model independently discovered the 30-step workflow. + +### Model-in-the-loop 3/10/30-step evaluation + +The model gate adds one real chat-model decision above the same sealed +catalogs. graph-tool-call retrieves eight candidates from 1,000 tools, the +model selects the final target from that frozen candidate catalog, the +strong-evidence selector guard records any override, and Arazzo-backed +planning executes the complete 3, 10, or 30-call workflow in the sandbox. + +```bash +MODEL=deepseek-chat \ +MODEL_REVISION=provider-reported-revision \ +LLM_URL=https://api.deepseek.com/v1 \ +API_KEY_ENV=DEEPSEEK_API_KEY \ +PROVIDER_PROFILE=deepseek \ +make arazzo-long-horizon-model-benchmark +``` + +The artifact reports `llm_target_exact` separately from +`selected_target_exact`, so deterministic guard recovery is never counted as +model accuracy. It also records exact plan/execution order, binding accuracy, +final-state goal completion, latency, token usage, and model/provider +provenance without storing API credentials. + +This is a target-selection plus graph-execution benchmark. Arazzo supplies the +prerequisite order and runtime bindings after target selection; it does not +claim that the model independently enumerated all 30 calls. + +The 2026-08-16 DeepSeek V4 Flash run used three repeats (nine executions): + +| Metric | Result | +|---|---:| +| Retrieval target hit@8 | `1.00` | +| Raw LLM target exact | `1.00` | +| Final selected target exact | `1.00` | +| Plan/execution required-tool recall | `1.00 / 1.00` | +| Exact plan/execution order | `1.00 / 1.00` | +| Binding accuracy | `1.00` | +| Goal completion | `1.00` | +| Retrieval p95 | `116.312 ms` | +| Model selection p95 | `1252.382 ms` | +| Plan and sandbox execution p95 | `11.277 ms` | + +The case-level artifact is +[`arazzo_long_horizon_deepseek_v4_flash_20260816.json`](../benchmarks/results/arazzo_long_horizon_deepseek_v4_flash_20260816.json). +DeepSeek thinking was explicitly disabled with the provider's documented +[`thinking.type=disabled`](https://api-docs.deepseek.com/guides/thinking_mode) +control so the bounded JSON selector output was not consumed by hidden +reasoning tokens. diff --git a/docs/research/long-horizon-goal-evaluation.md b/docs/research/long-horizon-goal-evaluation.md index 10b58f0..7cd8ad4 100644 --- a/docs/research/long-horizon-goal-evaluation.md +++ b/docs/research/long-horizon-goal-evaluation.md @@ -120,3 +120,31 @@ claim about arbitrary production workflows. The next release-candidate gate runs model target selection above the same sealed catalogs, followed by a read-only XGEN dev replay. Production API names and payloads remain outside the public artifact. + +## Model target-selection gate + +The release-candidate model gate is now executable with any OpenAI-compatible +or Ollama endpoint: + +```bash +MODEL=deepseek-chat \ +MODEL_REVISION=provider-reported-revision \ +LLM_URL=https://api.deepseek.com/v1 \ +API_KEY_ENV=DEEPSEEK_API_KEY \ +PROVIDER_PROFILE=deepseek \ +REPEATS=3 \ +make arazzo-long-horizon-model-benchmark +``` + +The model sees the natural-language request and only the retrieved Top-8 +catalog. Evaluator gold remains sealed until execution finishes. The report +separates raw model target accuracy, selector-guard accuracy, exact plan and +execution order, runtime binding accuracy, and final-state goal completion. +API keys are read only from the named environment variable and are not written +to the result artifact. + +The first frozen model artifact uses DeepSeek V4 Flash with three repeats over +all three horizons. All nine executions achieved exact raw model target +selection, exact graph plan and execution order, valid cross-step bindings, +and final-state goal completion. See +[`arazzo_long_horizon_deepseek_v4_flash_20260816.json`](../../benchmarks/results/arazzo_long_horizon_deepseek_v4_flash_20260816.json). diff --git a/tests/test_arazzo_long_horizon_model_benchmark.py b/tests/test_arazzo_long_horizon_model_benchmark.py new file mode 100644 index 0000000..65af2c1 --- /dev/null +++ b/tests/test_arazzo_long_horizon_model_benchmark.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import re + +import benchmarks.paper_model_loop.client as client_module +from benchmarks.arazzo_long_horizon.model_run import _provider_options, run_model_benchmark +from benchmarks.paper_model_loop import HTTPModelClient, ModelResponse + + +class ExactTargetClient: + provider = "test" + + def complete(self, messages, *, seed, timeout, max_tokens): + query = messages[-1]["content"] + if "공급업체" in query: + target = "issueSupplierActivationCertificate" + elif "장애 복구" in query: + target = "issueIncidentRecoveryReport" + else: + target = "issuePublicationApprovalCertificate" + assert re.search(rf'"name":"{target}"', query) + return ModelResponse( + content=f'{{"target_tool":"{target}","supporting_tools":[]}}', + input_tokens=100, + output_tokens=12, + latency_ms=5.0, + status_code=200, + finish_reason="stop", + ) + + +def test_model_gate_separates_llm_selection_from_graph_execution(): + report = run_model_benchmark( + model="fixture-model", + model_revision="test", + llm_url="https://user:secret@example.invalid/v1", + catalog_size=60, + workflow_lengths=(3, 10, 30), + model_client=ExactTargetClient(), + ) + + assert report["summary"]["status"] == "pass" + assert report["summary"]["metrics"]["llm_target_exact"] == 1.0 + assert report["summary"]["metrics"]["goal_completion_rate"] == 1.0 + assert [case["executed_call_count"] for case in report["cases"]] == [3, 10, 30] + assert report["model"]["endpoint"] == "https://***@example.invalid/v1" + assert report["summary"]["usage"] == {"input_tokens": 300, "output_tokens": 36} + + +class HallucinatingClient: + provider = "test" + + def complete(self, messages, *, seed, timeout, max_tokens): + return ModelResponse(content='{"target_tool":"notInCatalog"}', status_code=200) + + +def test_model_gate_records_hallucination_without_confusing_it_with_retrieval(): + report = run_model_benchmark( + model="fixture-model", + model_revision="test", + llm_url="https://example.invalid/v1", + catalog_size=40, + workflow_lengths=(3,), + model_client=HallucinatingClient(), + ) + case = report["cases"][0] + + assert case["target_hit_at_k"] == 1.0 + assert case["llm_target_exact"] == 0.0 + assert case["llm_target_in_catalog"] == 0.0 + assert "selector_tool_not_in_catalog" in case["model_reason_codes"] + assert report["summary"]["status"] == "fail" + + +def test_deepseek_provider_options_disable_thinking_and_require_json(): + assert _provider_options("deepseek") == { + "thinking": {"type": "disabled"}, + "response_format": {"type": "json_object"}, + } + assert _provider_options("default") == {} + + +def test_http_model_client_forwards_provider_options(monkeypatch): + captured = {} + + def fake_post(url, payload, *, headers, timeout): + captured.update({"url": url, "payload": payload, "headers": headers, "timeout": timeout}) + return ( + { + "choices": [{"message": {"content": "{}"}, "finish_reason": "stop"}], + "usage": {}, + }, + 200, + "", + ) + + monkeypatch.setattr(client_module, "_post_json", fake_post) + client = HTTPModelClient( + model="deepseek-v4-flash", + url="https://api.deepseek.com/v1", + provider="openai-compatible", + extra_body=_provider_options("deepseek"), + ) + + response = client.complete( + [{"role": "user", "content": "select"}], + seed=17, + timeout=30, + max_tokens=64, + ) + + assert response.status_code == 200 + assert captured["payload"]["thinking"] == {"type": "disabled"} + assert captured["payload"]["response_format"] == {"type": "json_object"} + assert "chat_template_kwargs" not in captured["payload"] diff --git a/tests/test_paper_model_loop.py b/tests/test_paper_model_loop.py index a7ac34d..5da7a03 100644 --- a/tests/test_paper_model_loop.py +++ b/tests/test_paper_model_loop.py @@ -476,6 +476,9 @@ def test_pairing_rejects_duplicate_model_loop_conditions() -> None: def test_redacted_url_removes_embedded_credentials() -> None: assert redacted_url("https://user:secret@example.com/v1") == "https://***@example.com/v1" + assert redacted_url("https://example.com/v1?api_key=secret®ion=kr") == ( + "https://example.com/v1?api_key=%2A%2A%2A®ion=kr" + ) def _analysis_cases(deltas_by_case: dict[str, list[float]]) -> list[dict[str, Any]]: