From 59985251a8212980900845d7097bb55ea68d86d8 Mon Sep 17 00:00:00 2001 From: SonAIengine Date: Sat, 15 Aug 2026 23:55:39 +0900 Subject: [PATCH 1/2] Add goal-completion evaluation harness --- Makefile | 7 +- benchmarks/goal_completion/__init__.py | 1 + benchmarks/goal_completion/run.py | 349 ++++++ benchmarks/goal_completion/scenarios.json | 267 +++++ .../goal_completion_baseline_0.40.json | 1045 +++++++++++++++++ docs/benchmarks.md | 46 + docs/research/long-horizon-goal-evaluation.md | 83 ++ graph_tool_call/__init__.py | 11 + graph_tool_call/evaluation/__init__.py | 27 + graph_tool_call/evaluation/evaluator.py | 400 +++++++ graph_tool_call/evaluation/schema.py | 377 ++++++ tests/test_goal_completion_benchmark.py | 38 + tests/test_goal_evaluation.py | 300 +++++ 13 files changed, 2950 insertions(+), 1 deletion(-) create mode 100644 benchmarks/goal_completion/__init__.py create mode 100644 benchmarks/goal_completion/run.py create mode 100644 benchmarks/goal_completion/scenarios.json create mode 100644 benchmarks/results/goal_completion_baseline_0.40.json create mode 100644 docs/research/long-horizon-goal-evaluation.md create mode 100644 graph_tool_call/evaluation/__init__.py create mode 100644 graph_tool_call/evaluation/evaluator.py create mode 100644 graph_tool_call/evaluation/schema.py create mode 100644 tests/test_goal_completion_benchmark.py create mode 100644 tests/test_goal_evaluation.py diff --git a/Makefile b/Makefile index 23db7a4..c1c4b29 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: quick lint test verify research-check research-check-unit research-check-deterministic research-check-smoke paper-corpus-check paper-corpus-internal-review-check paper-corpus-claim-check paper-adapter-conformance paper-baseline-run paper-graph-ablation paper-producer-coverage paper-output-promotion paper-candidate-admission paper-contract-projection paper-model-loop paper-llm-catalog-baseline paper-model-loop-analysis paper-toolinkos-parity paper-openapi-closure paper-harness-check xgen-benchmark xgen-llm-benchmark xgen-scale-snapshot xgen-scale-snapshot-check xgen-scale-acceptance xgen-scale-sweep xgen-scale-gate-check xgen-scale-028-gate-check xgen-scale-contract-ablation bfcl-benchmark bfcl-llm-benchmark bfcl-sweep bfcl-027-gate bfcl-027-gate-check bfcl-028-gate bfcl-028-gate-check bfcl-failure-subset bfcl-inspect-failures bfcl-hard-cases release-check pypi-smoke public-smoke launch-evidence launch-evidence-check observability-evidence observability-evidence-check +.PHONY: quick lint test verify research-check research-check-unit research-check-deterministic research-check-smoke paper-corpus-check paper-corpus-internal-review-check paper-corpus-claim-check paper-adapter-conformance paper-baseline-run paper-graph-ablation paper-producer-coverage paper-output-promotion paper-candidate-admission paper-contract-projection paper-model-loop paper-llm-catalog-baseline paper-model-loop-analysis paper-toolinkos-parity paper-openapi-closure paper-harness-check goal-completion-benchmark xgen-benchmark xgen-llm-benchmark xgen-scale-snapshot xgen-scale-snapshot-check xgen-scale-acceptance xgen-scale-sweep xgen-scale-gate-check xgen-scale-028-gate-check xgen-scale-contract-ablation bfcl-benchmark bfcl-llm-benchmark bfcl-sweep bfcl-027-gate bfcl-027-gate-check bfcl-028-gate bfcl-028-gate-check bfcl-failure-subset bfcl-inspect-failures bfcl-hard-cases release-check pypi-smoke public-smoke launch-evidence launch-evidence-check observability-evidence observability-evidence-check quick: scripts/quick-check.sh @@ -173,6 +173,11 @@ paper-harness-check: tests/test_openapi_dependency_closure_benchmark.py \ -q +goal-completion-benchmark: + poetry run python -m benchmarks.goal_completion.run \ + --scenarios "$${SCENARIOS:-benchmarks/goal_completion/scenarios.json}" \ + --output "$${OUT:-/tmp/graph-tool-call-goal-completion.json}" + xgen-benchmark: poetry run python -m benchmarks.xgen_tool_graph.run --suite all diff --git a/benchmarks/goal_completion/__init__.py b/benchmarks/goal_completion/__init__.py new file mode 100644 index 0000000..a509e19 --- /dev/null +++ b/benchmarks/goal_completion/__init__.py @@ -0,0 +1 @@ +"""Natural-language to goal-completion benchmark.""" diff --git a/benchmarks/goal_completion/run.py b/benchmarks/goal_completion/run.py new file mode 100644 index 0000000..87c6972 --- /dev/null +++ b/benchmarks/goal_completion/run.py @@ -0,0 +1,349 @@ +"""Run natural-language requests through retrieval, planning, and execution. + +Gold milestones are used only after execution by the goal evaluator. They are +never passed to retrieval, target selection, PathSynthesizer, or PlanRunner. +""" + +from __future__ import annotations + +import argparse +import copy +import json +from pathlib import Path +from typing import Any + +from benchmarks.xgen_tool_graph.run import DEFAULT_SPEC_PATH, build_benchmark_graph, load_json +from graph_tool_call import __version__ +from graph_tool_call.evaluation import GoalExecutionRecord, ScenarioSpec, evaluate_goal_execution +from graph_tool_call.graphify import ( + build_candidate_set, + retrieve_graphify, + target_action_priority_for_query, +) +from graph_tool_call.plan import PathSynthesizer, PlanRunner + +ROOT = Path(__file__).resolve().parent +DEFAULT_SCENARIOS_PATH = ROOT / "scenarios.json" + + +class CommerceSandbox: + """Resettable deterministic API world for fast goal-completion checks.""" + + def __init__(self, initial_state: dict[str, Any]) -> None: + self.state = copy.deepcopy(initial_state) + self._handlers = { + "searchProducts": self._call_search_products, + "getProductDetail": self._call_get_product_detail, + "getInventory": self._call_get_inventory, + "getCart": self._call_get_cart, + "addCartItem": self._call_add_cart_item, + "validateCoupon": self._call_validate_coupon, + "checkoutCart": self._call_checkout_cart, + "findOrders": self._call_find_orders, + "getOrderDetail": self._call_get_order_detail, + "getShipmentTracking": self._call_get_shipment_tracking, + "createProductReview": self._call_create_product_review, + } + + def call_tool(self, tool: str, args: dict[str, Any]) -> dict[str, Any]: + handler = self._handlers.get(tool) + if handler is None: + raise RuntimeError(f"unsupported sandbox tool: {tool}") + return handler(dict(args)) + + def snapshot(self) -> dict[str, Any]: + return copy.deepcopy(self.state) + + def _call_search_products(self, args: dict[str, Any]) -> dict[str, Any]: + _require(args, "q") + return { + "items": [ + { + "productId": "P100", + "productName": str(args["q"]), + "skuId": "SKU100", + "price": 100.0, + } + ] + } + + def _call_get_product_detail(self, args: dict[str, Any]) -> dict[str, Any]: + _require_value(args, "productId", "P100") + return { + "productId": "P100", + "productName": "fixture product", + "skuOptions": [{"skuId": "SKU100", "stockQty": 12}], + } + + def _call_get_inventory(self, args: dict[str, Any]) -> dict[str, Any]: + _require_value(args, "skuId", "SKU100") + return {"skuId": "SKU100", "stockQty": 12, "available": True} + + def _call_get_cart(self, args: dict[str, Any]) -> dict[str, Any]: + _require(args, "userId") + return copy.deepcopy(self.state["cart"]) + + def _call_add_cart_item(self, args: dict[str, Any]) -> dict[str, Any]: + _require(args, "userId", "skuId", "quantity") + _require_value(args, "skuId", "SKU100") + if not isinstance(args["quantity"], int) or args["quantity"] < 1: + raise ValueError("quantity must be a positive integer") + self.state["cart"]["items"].append({"skuId": args["skuId"], "quantity": args["quantity"]}) + return {"cartItemId": "CI100", "skuId": args["skuId"]} + + def _call_validate_coupon(self, args: dict[str, Any]) -> dict[str, Any]: + _require_value(args, "couponCode", "WELCOME20") + return { + "couponValidationId": "CV100", + "couponCode": "WELCOME20", + "discountAmount": 20.0, + "eligible": True, + } + + def _call_checkout_cart(self, args: dict[str, Any]) -> dict[str, Any]: + _require_value(args, "cartId", "cart-1") + _require_value(args, "couponValidationId", "CV100") + _require(args, "paymentMethod") + self.state["orders"]["O200"] = { + "orderNo": "O200", + "status": "paid", + "paymentMethod": args["paymentMethod"], + } + return {"orderNo": "O200", "paymentId": "PAY100"} + + def _call_find_orders(self, args: dict[str, Any]) -> dict[str, Any]: + _require_value(args, "customerEmail", "buyer@example.com") + return {"orders": [copy.deepcopy(self.state["orders"]["O100"])]} + + def _call_get_order_detail(self, args: dict[str, Any]) -> dict[str, Any]: + _require(args, "orderNo") + order = self.state["orders"].get(str(args["orderNo"])) + if not order: + raise ValueError("unknown orderNo") + return copy.deepcopy(order) + + def _call_get_shipment_tracking(self, args: dict[str, Any]) -> dict[str, Any]: + _require_value(args, "shipmentId", "S100") + return {"shipmentId": "S100", "trackingStatus": "in_transit", "eta": "2026-08-18"} + + def _call_create_product_review(self, args: dict[str, Any]) -> dict[str, Any]: + _require(args, "userId", "productId", "rating", "comment") + _require_value(args, "productId", "P100") + review = { + "reviewId": "R100", + "productId": args["productId"], + "rating": args["rating"], + "comment": args["comment"], + } + self.state["reviews"].append(review) + return {"reviewId": review["reviewId"], "productId": review["productId"]} + + +def run_benchmark( + *, + scenarios_path: Path = DEFAULT_SCENARIOS_PATH, + spec_path: Path = DEFAULT_SPEC_PATH, +) -> dict[str, Any]: + document = load_json(scenarios_path) + graph, graph_payload, _spec = build_benchmark_graph(spec_path=spec_path) + top_k = int(document.get("top_k") or 5) + token_budget = int(document.get("token_budget") or 2048) + context_defaults = dict(document.get("context_defaults") or {}) + initial_state = dict(document.get("initial_state") or {}) + rows = [ + _run_case( + case, + graph=graph, + graph_payload=graph_payload, + top_k=top_k, + token_budget=token_budget, + context_defaults=context_defaults, + initial_state=initial_state, + ) + for case in document.get("cases") or [] + ] + completed = sum(bool(row["evaluation"]["goal_completed"]) for row in rows) + return { + "benchmark": document.get("name"), + "description": document.get("description"), + "methodology": "natural_language_retrieve_plan_execute_goal_state", + "model": "none", + "graph_tool_call_version": __version__, + "scenario_count": len(rows), + "tool_count": len(graph.tools), + "edge_count": graph.graph.edge_count(), + "summary": _summarize(rows, completed=completed), + "cases": rows, + } + + +def _run_case( + case: dict[str, Any], + *, + graph: Any, + graph_payload: dict[str, Any], + top_k: int, + token_budget: int, + context_defaults: dict[str, Any], + initial_state: dict[str, Any], +) -> dict[str, Any]: + scenario_value = {**case, "initial_state": initial_state} + scenario = ScenarioSpec.from_dict(scenario_value) + retrieval = retrieve_graphify( + graph, + scenario.query, + top_k=top_k, + depth=0, + token_budget=token_budget, + include_evidence=True, + ) + retrieved = [str(item["name"]) for item in retrieval.get("results") or []] + selector = build_candidate_set( + retrieved, + graph_payload["tools"], + target_action_priority=target_action_priority_for_query(scenario.query), + max_hops=0, + ) + targets = [str(item) for item in selector.get("target_candidates") or []] + selected_target = targets[0] if targets else "" + candidates: list[str] = [] + plan = None + trace = None + failure: dict[str, Any] = {} + sandbox = CommerceSandbox(initial_state) + if selected_target: + expanded = build_candidate_set( + retrieved, + graph_payload["tools"], + expansion_seed=[selected_target], + max_producers_per_field=3, + max_hops=5, + ) + candidates = [str(item) for item in expanded.get("candidates") or []] + try: + plan = PathSynthesizer( + graph_payload, + max_depth=5, + context_defaults=context_defaults, + ).synthesize( + target=selected_target, + entities=dict(case.get("entities") or {}), + goal=scenario.query, + ) + trace = PlanRunner(sandbox.call_tool, binding_recovery=True).run( + plan, + input_context=dict(case.get("entities") or {}), + ) + except Exception as exc: # noqa: BLE001 - benchmark must report stage failures + failure = {"reason": type(exc).__name__, "message": str(exc)} + + if trace is None: + record = GoalExecutionRecord( + calls=(), + success=False, + retrieved_tools=tuple(retrieved), + candidate_tools=tuple(candidates), + planned_tools=tuple(str(step.tool) for step in (getattr(plan, "steps", ()) or ())), + final_state=sandbox.snapshot(), + ) + else: + record = GoalExecutionRecord.from_execution_trace( + trace, + plan=plan, + retrieved_tools=retrieved, + candidate_tools=candidates, + final_state=sandbox.snapshot(), + schema_valid=True, + ) + evaluation = evaluate_goal_execution(scenario, record) + expected_targets = [ + tool for milestone in scenario.milestones if milestone.target for tool in milestone.tools + ] + return { + "id": scenario.id, + "query": scenario.query, + "expected_targets": expected_targets, + "retrieved_tools": retrieved, + "selected_target": selected_target, + "candidate_tools": candidates, + "planned_tools": list(record.planned_tools), + "executed_tools": [call.tool for call in record.calls], + "runner_success": record.success, + "failure": failure, + "evaluation": evaluation.to_dict(), + } + + +def _summarize(rows: list[dict[str, Any]], *, completed: int) -> dict[str, Any]: + metric_names = ( + "candidate_required_tool_recall", + "plan_required_tool_recall", + "execution_required_tool_recall", + "dependency_order_accuracy", + "binding_accuracy", + "final_state_accuracy", + "schema_valid_call_rate", + "extraneous_call_rate", + ) + metrics: dict[str, Any] = {} + for name in metric_names: + values = [ + row["evaluation"]["metrics"].get(name) + for row in rows + if row["evaluation"]["metrics"].get(name) is not None + ] + metrics[name] = round(sum(values) / len(values), 6) if values else None + return { + "status": "pass" if completed == len(rows) else "fail", + "goal_completion_rate": round(completed / len(rows), 6) if rows else 0.0, + "completed": completed, + "cases": len(rows), + "uncaught_error_count": sum(bool(row["failure"]) for row in rows), + **metrics, + } + + +def _require(args: dict[str, Any], *names: str) -> None: + missing = [name for name in names if name not in args or args[name] in (None, "")] + if missing: + raise ValueError(f"missing required arguments: {', '.join(missing)}") + + +def _require_value(args: dict[str, Any], name: str, expected: Any) -> None: + _require(args, name) + if args[name] != expected: + raise ValueError(f"invalid {name}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scenarios", type=Path, default=DEFAULT_SCENARIOS_PATH) + parser.add_argument("--spec", type=Path, default=DEFAULT_SPEC_PATH) + parser.add_argument("--output", type=Path) + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + report = run_benchmark(scenarios_path=args.scenarios, spec_path=args.spec) + rendered = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n", encoding="utf-8") + if args.json: + print(rendered) + else: + summary = report["summary"] + print( + f"{report['benchmark']}: {summary['completed']}/{summary['cases']} goals " + f"({summary['goal_completion_rate']:.1%}), status={summary['status']}" + ) + for row in report["cases"]: + result = row["evaluation"] + print( + f"- {row['id']}: selected={row['selected_target'] or '-'} " + f"plan={','.join(row['planned_tools']) or '-'} " + f"goal={'pass' if result['goal_completed'] else 'fail'}" + ) + return 0 if report["summary"]["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/goal_completion/scenarios.json b/benchmarks/goal_completion/scenarios.json new file mode 100644 index 0000000..6c9d58b --- /dev/null +++ b/benchmarks/goal_completion/scenarios.json @@ -0,0 +1,267 @@ +{ + "name": "Goal Completion Commerce Baseline", + "description": "Natural-language retrieval, planning, binding, execution, and goal-state checks.", + "top_k": 5, + "token_budget": 2048, + "context_defaults": { + "siteNo": "100", + "userId": "user-1" + }, + "initial_state": { + "cart": { + "cartId": "cart-1", + "items": [] + }, + "orders": { + "O100": { + "orderNo": "O100", + "shipmentId": "S100", + "status": "shipped", + "totalAmount": 120.0 + } + }, + "reviews": [] + }, + "cases": [ + { + "id": "product_detail", + "query": "셔츠를 검색해서 상품 상세를 보여줘", + "target": "getProductDetail", + "entities": {"q": "셔츠"}, + "milestones": [ + {"id": "find_product", "tools": ["searchProducts"]}, + {"id": "read_product", "tools": ["getProductDetail"], "target": true} + ], + "dependency_constraints": [ + {"before": "find_product", "after": "read_product"} + ], + "binding_constraints": [ + { + "source_milestone": "find_product", + "source_path": "items[0].productId", + "target_milestone": "read_product", + "target_arg": "productId" + } + ], + "final_state_assertions": [ + {"scope": "output", "path": "productId", "operator": "eq", "value": "P100"} + ], + "forbidden_tools": ["checkoutCart", "createProductReview"], + "max_calls": 3, + "max_replans": 1, + "timeout_sec": 5 + }, + { + "id": "inventory_lookup", + "query": "후드 상품을 찾아서 실제 SKU 재고와 구매 가능 여부를 확인해줘", + "target": "getInventory", + "entities": {"q": "후드"}, + "milestones": [ + {"id": "find_product", "tools": ["searchProducts"]}, + {"id": "read_product", "tools": ["getProductDetail"]}, + {"id": "read_inventory", "tools": ["getInventory"], "target": true} + ], + "dependency_constraints": [ + {"before": "find_product", "after": "read_product"}, + {"before": "read_product", "after": "read_inventory"} + ], + "binding_constraints": [ + { + "source_milestone": "find_product", + "source_path": "items[0].productId", + "target_milestone": "read_product", + "target_arg": "productId" + }, + { + "source_milestone": "read_product", + "source_path": "skuOptions[0].skuId", + "target_milestone": "read_inventory", + "target_arg": "skuId" + } + ], + "final_state_assertions": [ + {"scope": "output", "path": "stockQty", "operator": "gte", "value": 1}, + {"scope": "output", "path": "available", "operator": "eq", "value": true} + ], + "forbidden_tools": ["addCartItem", "checkoutCart"], + "max_calls": 4, + "max_replans": 1, + "timeout_sec": 5 + }, + { + "id": "add_cart_item", + "query": "운동화를 찾아서 첫 번째 옵션 한 개를 내 장바구니에 담아줘", + "target": "addCartItem", + "entities": {"q": "운동화", "quantity": 1}, + "milestones": [ + {"id": "find_product", "tools": ["searchProducts"]}, + {"id": "read_product", "tools": ["getProductDetail"]}, + {"id": "add_item", "tools": ["addCartItem"], "target": true} + ], + "dependency_constraints": [ + {"before": "find_product", "after": "read_product"}, + {"before": "read_product", "after": "add_item"} + ], + "binding_constraints": [ + { + "source_milestone": "find_product", + "source_path": "items[0].productId", + "target_milestone": "read_product", + "target_arg": "productId" + }, + { + "source_milestone": "read_product", + "source_path": "skuOptions[0].skuId", + "target_milestone": "add_item", + "target_arg": "skuId" + } + ], + "final_state_assertions": [ + { + "scope": "final_state", + "path": "cart.items[0].skuId", + "operator": "eq", + "value": "SKU100" + }, + { + "scope": "final_state", + "path": "cart.items[0].quantity", + "operator": "eq", + "value": 1 + } + ], + "forbidden_tools": ["checkoutCart"], + "max_calls": 4, + "max_replans": 1, + "timeout_sec": 5 + }, + { + "id": "coupon_checkout", + "query": "현재 장바구니에 WELCOME20 쿠폰을 검증해서 카드로 결제해줘", + "target": "checkoutCart", + "entities": {"couponCode": "WELCOME20", "paymentMethod": "card"}, + "milestones": [ + {"id": "read_cart", "tools": ["getCart"]}, + {"id": "validate_coupon", "tools": ["validateCoupon"]}, + {"id": "checkout", "tools": ["checkoutCart"], "target": true} + ], + "dependency_constraints": [ + {"before": "read_cart", "after": "checkout"}, + {"before": "validate_coupon", "after": "checkout"} + ], + "binding_constraints": [ + { + "source_milestone": "read_cart", + "source_path": "cartId", + "target_milestone": "checkout", + "target_arg": "cartId" + }, + { + "source_milestone": "validate_coupon", + "source_path": "couponValidationId", + "target_milestone": "checkout", + "target_arg": "couponValidationId" + } + ], + "final_state_assertions": [ + { + "scope": "final_state", + "path": "orders.O200.status", + "operator": "eq", + "value": "paid" + }, + { + "scope": "final_state", + "path": "orders.O200.paymentMethod", + "operator": "eq", + "value": "card" + } + ], + "forbidden_tools": ["createProductReview"], + "max_calls": 4, + "max_replans": 1, + "timeout_sec": 5 + }, + { + "id": "shipment_tracking", + "query": "buyer@example.com 고객 주문을 찾아 배송 추적 상태와 도착 예정일을 알려줘", + "target": "getShipmentTracking", + "entities": {"customerEmail": "buyer@example.com"}, + "milestones": [ + {"id": "find_order", "tools": ["findOrders"]}, + {"id": "read_order", "tools": ["getOrderDetail"]}, + {"id": "track_shipment", "tools": ["getShipmentTracking"], "target": true} + ], + "dependency_constraints": [ + {"before": "find_order", "after": "read_order"}, + {"before": "read_order", "after": "track_shipment"} + ], + "binding_constraints": [ + { + "source_milestone": "find_order", + "source_path": "orders[0].orderNo", + "target_milestone": "read_order", + "target_arg": "orderNo" + }, + { + "source_milestone": "read_order", + "source_path": "shipmentId", + "target_milestone": "track_shipment", + "target_arg": "shipmentId" + } + ], + "final_state_assertions": [ + { + "scope": "output", + "path": "trackingStatus", + "operator": "eq", + "value": "in_transit" + }, + {"scope": "output", "path": "eta", "operator": "exists"} + ], + "forbidden_tools": ["checkoutCart"], + "max_calls": 4, + "max_replans": 1, + "timeout_sec": 5 + }, + { + "id": "create_review", + "query": "백팩을 찾아서 별점 5점과 '튼튼해요'라는 리뷰를 작성해줘", + "target": "createProductReview", + "entities": {"q": "백팩", "rating": 5, "comment": "튼튼해요"}, + "milestones": [ + {"id": "find_product", "tools": ["searchProducts"]}, + {"id": "create_review", "tools": ["createProductReview"], "target": true} + ], + "dependency_constraints": [ + {"before": "find_product", "after": "create_review"} + ], + "binding_constraints": [ + { + "source_milestone": "find_product", + "source_path": "items[0].productId", + "target_milestone": "create_review", + "target_arg": "productId" + } + ], + "final_state_assertions": [ + { + "scope": "final_state", + "path": "reviews[0].rating", + "operator": "eq", + "value": 5 + }, + { + "scope": "final_state", + "path": "reviews[0].comment", + "operator": "eq", + "value": "튼튼해요" + } + ], + "forbidden_tools": ["checkoutCart"], + "max_calls": 3, + "max_replans": 1, + "timeout_sec": 5 + } + ] +} diff --git a/benchmarks/results/goal_completion_baseline_0.40.json b/benchmarks/results/goal_completion_baseline_0.40.json new file mode 100644 index 0000000..30290a3 --- /dev/null +++ b/benchmarks/results/goal_completion_baseline_0.40.json @@ -0,0 +1,1045 @@ +{ + "benchmark": "Goal Completion Commerce Baseline", + "description": "Natural-language retrieval, planning, binding, execution, and goal-state checks.", + "methodology": "natural_language_retrieve_plan_execute_goal_state", + "model": "none", + "graph_tool_call_version": "0.40.0", + "scenario_count": 6, + "tool_count": 11, + "edge_count": 13, + "summary": { + "status": "fail", + "goal_completion_rate": 0.5, + "completed": 3, + "cases": 6, + "uncaught_error_count": 0, + "candidate_required_tool_recall": 0.611111, + "plan_required_tool_recall": 0.611111, + "execution_required_tool_recall": 0.611111, + "dependency_order_accuracy": 0.5, + "binding_accuracy": 0.5, + "final_state_accuracy": 0.5, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.166667 + }, + "cases": [ + { + "id": "product_detail", + "query": "셔츠를 검색해서 상품 상세를 보여줘", + "expected_targets": [ + "getProductDetail" + ], + "retrieved_tools": [ + "searchProducts", + "getProductDetail", + "addCartItem", + "createProductReview", + "findOrders" + ], + "selected_target": "getProductDetail", + "candidate_tools": [ + "getProductDetail", + "searchProducts" + ], + "planned_tools": [ + "searchProducts", + "getProductDetail" + ], + "executed_tools": [ + "searchProducts", + "getProductDetail" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "product_detail", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 2, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "find_product": 1, + "read_product": 2 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "find_product", + "expected": [ + "searchProducts" + ], + "observed": "searchProducts" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "read_product", + "expected": [ + "getProductDetail" + ], + "observed": "getProductDetail" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "find_product->read_product", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "find_product.items[0].productId->read_product.productId", + "expected": "P100", + "observed": "P100" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.productId", + "expected": { + "operator": "eq", + "value": "P100" + }, + "observed": "P100" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 3, + "observed": 2 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 1, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 5.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 2, + "observed": 2 + } + ], + "failure_reason_codes": [] + } + }, + { + "id": "inventory_lookup", + "query": "후드 상품을 찾아서 실제 SKU 재고와 구매 가능 여부를 확인해줘", + "expected_targets": [ + "getInventory" + ], + "retrieved_tools": [ + "getInventory", + "addCartItem", + "searchProducts", + "getCart", + "getProductDetail" + ], + "selected_target": "searchProducts", + "candidate_tools": [ + "searchProducts" + ], + "planned_tools": [ + "searchProducts" + ], + "executed_tools": [ + "searchProducts" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "inventory_lookup", + "goal_completed": false, + "metrics": { + "candidate_required_tool_recall": 0.333333, + "plan_required_tool_recall": 0.333333, + "execution_required_tool_recall": 0.333333, + "required_tool_recall": 0.333333, + "milestone_completion": 0.333333, + "dependency_order_accuracy": 0.0, + "binding_accuracy": 0.0, + "final_state_accuracy": 0.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 1, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 0.0 + }, + "matched_milestones": { + "find_product": 1 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "find_product", + "expected": [ + "searchProducts" + ], + "observed": "searchProducts" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "read_product", + "expected": [ + "getProductDetail" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "read_inventory", + "expected": [ + "getInventory" + ], + "observed": "" + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "find_product->read_product", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "read_product->read_inventory", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "find_product.items[0].productId->read_product.productId", + "expected": "P100", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "read_product.skuOptions[0].skuId->read_inventory.skuId", + "expected": "", + "observed": "" + }, + { + "category": "state", + "code": "goal_state_mismatch", + "passed": false, + "subject": "output.stockQty", + "expected": { + "operator": "gte", + "value": 1 + }, + "observed": "" + }, + { + "category": "state", + "code": "goal_state_mismatch", + "passed": false, + "subject": "output.available", + "expected": { + "operator": "eq", + "value": true + }, + "observed": "" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 4, + "observed": 1 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 1, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 5.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 1, + "observed": 1 + } + ], + "failure_reason_codes": [ + "missing_milestone", + "invalid_dependency_order", + "binding_mismatch", + "goal_state_mismatch" + ] + } + }, + { + "id": "add_cart_item", + "query": "운동화를 찾아서 첫 번째 옵션 한 개를 내 장바구니에 담아줘", + "expected_targets": [ + "addCartItem" + ], + "retrieved_tools": [ + "addCartItem", + "getCart", + "checkoutCart", + "getProductDetail", + "getInventory" + ], + "selected_target": "getCart", + "candidate_tools": [ + "getCart" + ], + "planned_tools": [ + "getCart" + ], + "executed_tools": [ + "getCart" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "add_cart_item", + "goal_completed": false, + "metrics": { + "candidate_required_tool_recall": 0.0, + "plan_required_tool_recall": 0.0, + "execution_required_tool_recall": 0.0, + "required_tool_recall": 0.0, + "milestone_completion": 0.0, + "dependency_order_accuracy": 0.0, + "binding_accuracy": 0.0, + "final_state_accuracy": 0.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 1.0, + "policy_violation_count": 0, + "call_count": 1, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 0.0 + }, + "matched_milestones": {}, + "checks": [ + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "find_product", + "expected": [ + "searchProducts" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "read_product", + "expected": [ + "getProductDetail" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "add_item", + "expected": [ + "addCartItem" + ], + "observed": "" + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "find_product->read_product", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "read_product->add_item", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "find_product.items[0].productId->read_product.productId", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "read_product.skuOptions[0].skuId->add_item.skuId", + "expected": "", + "observed": "" + }, + { + "category": "state", + "code": "goal_state_mismatch", + "passed": false, + "subject": "final_state.cart.items[0].skuId", + "expected": { + "operator": "eq", + "value": "SKU100" + }, + "observed": "" + }, + { + "category": "state", + "code": "goal_state_mismatch", + "passed": false, + "subject": "final_state.cart.items[0].quantity", + "expected": { + "operator": "eq", + "value": 1 + }, + "observed": "" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 4, + "observed": 1 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 1, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 5.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 1, + "observed": 1 + } + ], + "failure_reason_codes": [ + "missing_milestone", + "invalid_dependency_order", + "binding_mismatch", + "goal_state_mismatch" + ] + } + }, + { + "id": "coupon_checkout", + "query": "현재 장바구니에 WELCOME20 쿠폰을 검증해서 카드로 결제해줘", + "expected_targets": [ + "checkoutCart" + ], + "retrieved_tools": [ + "checkoutCart", + "getCart", + "validateCoupon", + "addCartItem", + "searchProducts" + ], + "selected_target": "checkoutCart", + "candidate_tools": [ + "checkoutCart", + "getCart", + "validateCoupon" + ], + "planned_tools": [ + "getCart", + "validateCoupon", + "checkoutCart" + ], + "executed_tools": [ + "getCart", + "validateCoupon", + "checkoutCart" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "coupon_checkout", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 3, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "read_cart": 1, + "validate_coupon": 2, + "checkout": 3 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "read_cart", + "expected": [ + "getCart" + ], + "observed": "getCart" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "validate_coupon", + "expected": [ + "validateCoupon" + ], + "observed": "validateCoupon" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "checkout", + "expected": [ + "checkoutCart" + ], + "observed": "checkoutCart" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "read_cart->checkout", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 3 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "validate_coupon->checkout", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "read_cart.cartId->checkout.cartId", + "expected": "cart-1", + "observed": "cart-1" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "validate_coupon.couponValidationId->checkout.couponValidationId", + "expected": "CV100", + "observed": "CV100" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "final_state.orders.O200.status", + "expected": { + "operator": "eq", + "value": "paid" + }, + "observed": "paid" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "final_state.orders.O200.paymentMethod", + "expected": { + "operator": "eq", + "value": "card" + }, + "observed": "card" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 4, + "observed": 3 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 1, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 5.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 3, + "observed": 3 + } + ], + "failure_reason_codes": [] + } + }, + { + "id": "shipment_tracking", + "query": "buyer@example.com 고객 주문을 찾아 배송 추적 상태와 도착 예정일을 알려줘", + "expected_targets": [ + "getShipmentTracking" + ], + "retrieved_tools": [ + "getShipmentTracking", + "getOrderDetail", + "findOrders", + "createProductReview", + "getCart" + ], + "selected_target": "findOrders", + "candidate_tools": [ + "findOrders" + ], + "planned_tools": [ + "findOrders" + ], + "executed_tools": [ + "findOrders" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "shipment_tracking", + "goal_completed": false, + "metrics": { + "candidate_required_tool_recall": 0.333333, + "plan_required_tool_recall": 0.333333, + "execution_required_tool_recall": 0.333333, + "required_tool_recall": 0.333333, + "milestone_completion": 0.333333, + "dependency_order_accuracy": 0.0, + "binding_accuracy": 0.0, + "final_state_accuracy": 0.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 1, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 0.0 + }, + "matched_milestones": { + "find_order": 1 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "find_order", + "expected": [ + "findOrders" + ], + "observed": "findOrders" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "read_order", + "expected": [ + "getOrderDetail" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "track_shipment", + "expected": [ + "getShipmentTracking" + ], + "observed": "" + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "find_order->read_order", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "read_order->track_shipment", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "find_order.orders[0].orderNo->read_order.orderNo", + "expected": "O100", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "read_order.shipmentId->track_shipment.shipmentId", + "expected": "", + "observed": "" + }, + { + "category": "state", + "code": "goal_state_mismatch", + "passed": false, + "subject": "output.trackingStatus", + "expected": { + "operator": "eq", + "value": "in_transit" + }, + "observed": "" + }, + { + "category": "state", + "code": "goal_state_mismatch", + "passed": false, + "subject": "output.eta", + "expected": { + "operator": "exists", + "value": null + }, + "observed": "" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 4, + "observed": 1 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 1, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 5.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 1, + "observed": 1 + } + ], + "failure_reason_codes": [ + "missing_milestone", + "invalid_dependency_order", + "binding_mismatch", + "goal_state_mismatch" + ] + } + }, + { + "id": "create_review", + "query": "백팩을 찾아서 별점 5점과 '튼튼해요'라는 리뷰를 작성해줘", + "expected_targets": [ + "createProductReview" + ], + "retrieved_tools": [ + "createProductReview" + ], + "selected_target": "createProductReview", + "candidate_tools": [ + "createProductReview", + "searchProducts" + ], + "planned_tools": [ + "searchProducts", + "createProductReview" + ], + "executed_tools": [ + "searchProducts", + "createProductReview" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "create_review", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 2, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "find_product": 1, + "create_review": 2 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "find_product", + "expected": [ + "searchProducts" + ], + "observed": "searchProducts" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "create_review", + "expected": [ + "createProductReview" + ], + "observed": "createProductReview" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "find_product->create_review", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "find_product.items[0].productId->create_review.productId", + "expected": "P100", + "observed": "P100" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "final_state.reviews[0].rating", + "expected": { + "operator": "eq", + "value": 5 + }, + "observed": 5 + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "final_state.reviews[0].comment", + "expected": { + "operator": "eq", + "value": "튼튼해요" + }, + "observed": "튼튼해요" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 3, + "observed": 2 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 1, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 5.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 2, + "observed": 2 + } + ], + "failure_reason_codes": [] + } + } + ] +} diff --git a/docs/benchmarks.md b/docs/benchmarks.md index f4b1aa9..f9e6ccb 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1110,3 +1110,49 @@ One case (`inventory_chain_ko`) selects the correct final target but omits upstream producer steps because the wording can be read as "the SKU is already known." This is counted in `final_plan_exact_match`, while graph-tool-call's own retrieval and candidate-plan coverage remain `1.00`. + +## Goal-completion benchmark + +Target retrieval alone does not prove that an agent can finish a multi-API +request. The goal-completion harness runs this complete deterministic path: + +```text +natural language -> retrieval -> target selection -> dependency plan + -> resolved calls -> sandbox execution -> goal-state checks +``` + +Run it with: + +```bash +make goal-completion-benchmark +``` + +The scenario contract does not require one exact plan. It declares allowed +tool alternatives, required milestones, dependency constraints, output-to-arg +bindings, forbidden tools, budgets, and assertions against the final state or +final output. Gold constraints are only read by the evaluator after execution; +they are never passed to retrieval or planning. + +The first graph-tool-call `0.40.0` baseline contains six 2-3 step commerce +requests. It is intentionally recorded as a failing baseline: + +| Metric | Baseline | +|---|---:| +| Goal completion | `3/6` (`0.50`) | +| Candidate required-tool recall | `0.611111` | +| Plan required-tool recall | `0.611111` | +| Dependency order accuracy | `0.50` | +| Binding accuracy | `0.50` | +| Final-state accuracy | `0.50` | +| Schema-valid call rate | `1.00` | + +All three failures retrieved the intended target inside Top-K, but target +selection chose an upstream search/read tool and produced a one-step plan. +This demonstrates why `hit@K` and schema-valid calls are insufficient evidence +for end-to-end tool-use quality. The saved replayable report is +[`benchmarks/results/goal_completion_baseline_0.40.json`](../benchmarks/results/goal_completion_baseline_0.40.json). + +This first fixture validates the harness, not long-horizon or model quality. +The next gates add 5-8, 9-15, and 16-30 step scenarios, repeated model runs, +failure recovery, and XGEN dev API assertions. See +[`docs/research/long-horizon-goal-evaluation.md`](research/long-horizon-goal-evaluation.md). diff --git a/docs/research/long-horizon-goal-evaluation.md b/docs/research/long-horizon-goal-evaluation.md new file mode 100644 index 0000000..11bc609 --- /dev/null +++ b/docs/research/long-horizon-goal-evaluation.md @@ -0,0 +1,83 @@ +# Long-horizon tool-use goal evaluation + +## Question + +The benchmark answers one product question: + +> Given only a natural-language request and a large tool catalog, did the +> system find every necessary capability, execute a valid dependency order, +> pass the right values between calls, recover safely, and reach the requested +> final state? + +An exact tool array is not the primary ground truth. Real APIs can expose +equivalent tools and independent branches can run in different valid orders. + +## Stable scenario contract + +`graph_tool_call.evaluation.ScenarioSpec` contains: + +- `milestones`: semantic steps with one or more allowed tools +- `dependency_constraints`: partial-order requirements between milestones +- `binding_constraints`: source response path to target argument equality +- `final_state_assertions`: deterministic checks on state or final output +- `forbidden_tools`: policy and mutation safety boundaries +- `max_calls`, `max_replans`, `timeout_sec`: bounded execution budgets + +The evaluator accepts a transport-neutral `GoalExecutionRecord`. XGEN can +adapt Quality Lab traces to this record without moving DB, auth, HTTP, or SSE +logic into graph-tool-call. + +## Metrics + +The report keeps stages separate so a high retrieval score cannot hide an +execution failure: + +1. candidate required-tool recall +2. plan required-tool recall +3. execution required-tool recall and milestone completion +4. dependency-order accuracy +5. binding accuracy +6. schema-valid call rate +7. final-state accuracy +8. extraneous-call and policy-violation rates +9. recovery attempt/success, calls, replans, latency +10. strict final goal completion + +`goal_completion=1` requires runner success and every hard scenario check to +pass. Partial metrics remain diagnostic only. + +## Evaluation tiers + +| Tier | Required calls | Purpose | Routine | +|---|---:|---|---| +| L1 | 2-4 | target, basic order, binding | every PR | +| L2 | 5-8 | multiple producers and branches | every PR/nightly | +| L3 | 9-15 | pagination, retries, re-planning | nightly | +| L4 | 16-30 | long context and sustained execution | release candidate | + +The deterministic sandbox remains resettable and safe for every commit. Real +XGEN dev APIs are a separate release-candidate gate with read-only assertions +or explicit mutation cleanup. + +## Baseline and next bottleneck + +The initial six-case L1 fixture completes 3/6 goals. Product detail, checkout, +and review pass. Inventory, add-to-cart, and shipment tracking fail because the +intended target is retrieved but an upstream tool is selected as the final +target. The runner then executes a valid but incomplete one-step plan. + +The next engine change therefore targets final-target inference for compound +requests. It must improve this sealed baseline without reading milestone gold +data and without adding fixture-specific operation names or domain aliases. + +After the L1 selector gate passes, work proceeds in this order: + +1. add L2 scenarios with alternative tools and independent dependency branches +2. add explicit retry/re-plan trajectories and recovery metrics +3. add state-reset and cleanup adapters for XGEN Quality Lab +4. run the same sealed cases with the XGEN default model at three repeats +5. add L3/L4 datasets and compare full catalog, Top-K, graph closure, and the + complete graph-tool-call pipeline under the same model and prompt + +README or paper claims may use only versioned scenario files and saved reports +that can be reproduced by the documented command. diff --git a/graph_tool_call/__init__.py b/graph_tool_call/__init__.py index 4bfd5fa..e8dbe84 100644 --- a/graph_tool_call/__init__.py +++ b/graph_tool_call/__init__.py @@ -17,6 +17,8 @@ "IngestCapabilities", "IngestConformanceError", "GraphQLIntrospectionIngestAdapter", + "GoalEvaluation", + "GoalExecutionRecord", "IngestIssue", "IngestResult", "UnknownIngestAdapterError", @@ -32,6 +34,7 @@ "RelationType", "RetrievalResult", "SearchMode", + "ScenarioSpec", "ToolCallAssessment", "ToolCallDecision", "ToolCallPolicy", @@ -41,6 +44,7 @@ "TraceRecorder", "OpenTelemetryTraceExporter", "filter_tools", + "evaluate_goal_execution", "detect_ingest_adapter", "get_default_ingest_registry", "ingest_source", @@ -106,6 +110,13 @@ "scrub_trace_payload": ("graph_tool_call.learning", "scrub_trace_payload"), "TraceEnvelope": ("graph_tool_call.observability", "TraceEnvelope"), "TraceRecorder": ("graph_tool_call.observability", "TraceRecorder"), + "ScenarioSpec": ("graph_tool_call.evaluation", "ScenarioSpec"), + "GoalExecutionRecord": ("graph_tool_call.evaluation", "GoalExecutionRecord"), + "GoalEvaluation": ("graph_tool_call.evaluation", "GoalEvaluation"), + "evaluate_goal_execution": ( + "graph_tool_call.evaluation", + "evaluate_goal_execution", + ), "OpenTelemetryTraceExporter": ( "graph_tool_call.observability", "OpenTelemetryTraceExporter", diff --git a/graph_tool_call/evaluation/__init__.py b/graph_tool_call/evaluation/__init__.py new file mode 100644 index 0000000..a132f2e --- /dev/null +++ b/graph_tool_call/evaluation/__init__.py @@ -0,0 +1,27 @@ +"""Outcome-based evaluation for multi-tool plans and executions.""" + +from .evaluator import evaluate_goal_execution +from .schema import ( + BindingConstraint, + DependencyConstraint, + EvaluationCheck, + GoalEvaluation, + GoalExecutionRecord, + MilestoneSpec, + ObservedToolCall, + ScenarioSpec, + StateAssertion, +) + +__all__ = [ + "BindingConstraint", + "DependencyConstraint", + "EvaluationCheck", + "GoalEvaluation", + "GoalExecutionRecord", + "MilestoneSpec", + "ObservedToolCall", + "ScenarioSpec", + "StateAssertion", + "evaluate_goal_execution", +] diff --git a/graph_tool_call/evaluation/evaluator.py b/graph_tool_call/evaluation/evaluator.py new file mode 100644 index 0000000..e36c145 --- /dev/null +++ b/graph_tool_call/evaluation/evaluator.py @@ -0,0 +1,400 @@ +"""Deterministic goal-state evaluator for multi-tool trajectories.""" + +from __future__ import annotations + +import re +from typing import Any + +from .schema import ( + EvaluationCheck, + GoalEvaluation, + GoalExecutionRecord, + MilestoneSpec, + ObservedToolCall, + ScenarioSpec, + StateAssertion, +) + +_MISSING = object() +_PATH_TOKEN = re.compile(r"(?:^|\.)([^.\[\]]+)|\[([0-9]+)\]") + + +def evaluate_goal_execution( + scenario: ScenarioSpec | dict[str, Any], + record: GoalExecutionRecord, +) -> GoalEvaluation: + """Evaluate a trajectory without requiring one brittle exact tool list. + + Required milestones are matched to successful calls, then dependency, + binding, state, schema, budget, and safety checks are evaluated. The goal + passes only when every hard check passes and the runner completed. + """ + + spec = scenario if isinstance(scenario, ScenarioSpec) else ScenarioSpec.from_dict(scenario) + checks: list[EvaluationCheck] = [] + matched = _match_milestones(spec.milestones, record.calls) + required = [item for item in spec.milestones if item.required] + + for milestone in required: + sequence = matched.get(milestone.id) + checks.append( + EvaluationCheck( + category="milestone", + code=("milestone_completed" if sequence is not None else "missing_milestone"), + passed=sequence is not None, + subject=milestone.id, + expected=list(milestone.tools), + observed=_tool_at_sequence(record.calls, sequence), + ) + ) + + for constraint in spec.dependency_constraints: + before = matched.get(constraint.before) + after = matched.get(constraint.after) + passed = before is not None and after is not None and before < after + checks.append( + EvaluationCheck( + category="dependency", + code=("dependency_order_valid" if passed else "invalid_dependency_order"), + passed=passed, + subject=f"{constraint.before}->{constraint.after}", + expected="before", + observed={"before_sequence": before, "after_sequence": after}, + ) + ) + + for constraint in spec.binding_constraints: + source = _call_at_sequence(record.calls, matched.get(constraint.source_milestone)) + target = _call_at_sequence(record.calls, matched.get(constraint.target_milestone)) + source_value = _read_path(source.output, constraint.source_path) if source else _MISSING + target_value = _read_path(target.args, constraint.target_arg) if target else _MISSING + passed = source_value is not _MISSING and source_value == target_value + checks.append( + EvaluationCheck( + category="binding", + code=("binding_valid" if passed else "binding_mismatch"), + passed=passed, + subject=( + f"{constraint.source_milestone}.{constraint.source_path}" + f"->{constraint.target_milestone}.{constraint.target_arg}" + ), + expected=source_value if source_value is not _MISSING else "", + observed=target_value if target_value is not _MISSING else "", + ) + ) + + for assertion in spec.final_state_assertions: + scope = _assertion_scope(assertion, spec=spec, record=record) + observed = _read_path(scope, assertion.path) + passed = _evaluate_operator(observed, assertion.operator, assertion.value) + checks.append( + EvaluationCheck( + category="state", + code=("state_assertion_valid" if passed else "goal_state_mismatch"), + passed=passed, + subject=f"{assertion.scope}.{assertion.path}", + expected={"operator": assertion.operator, "value": assertion.value}, + observed=observed if observed is not _MISSING else "", + ) + ) + + forbidden = set(spec.forbidden_tools) + for call in record.calls: + if call.tool in forbidden: + checks.append( + EvaluationCheck( + category="policy", + code="forbidden_tool_called", + passed=False, + subject=call.tool, + expected="not called", + observed=call.sequence, + ) + ) + + if spec.max_calls is not None: + checks.append( + EvaluationCheck( + category="budget", + code=( + "call_budget_valid" + if len(record.calls) <= spec.max_calls + else "max_calls_exceeded" + ), + passed=len(record.calls) <= spec.max_calls, + subject="max_calls", + expected=spec.max_calls, + observed=len(record.calls), + ) + ) + if spec.max_replans is not None: + checks.append( + EvaluationCheck( + category="budget", + code=( + "replan_budget_valid" + if record.replans <= spec.max_replans + else "max_replans_exceeded" + ), + passed=record.replans <= spec.max_replans, + subject="max_replans", + expected=spec.max_replans, + observed=record.replans, + ) + ) + if spec.timeout_sec is not None: + timeout_ms = int(spec.timeout_sec * 1000) + checks.append( + EvaluationCheck( + category="budget", + code=( + "latency_budget_valid" + if record.latency_ms <= timeout_ms + else "timeout_exceeded" + ), + passed=record.latency_ms <= timeout_ms, + subject="timeout_sec", + expected=spec.timeout_sec, + observed=record.latency_ms / 1000, + ) + ) + + schema_values = [call.schema_valid for call in record.calls if call.schema_valid is not None] + if schema_values: + schema_passed = all(schema_values) + checks.append( + EvaluationCheck( + category="schema", + code=("schema_valid" if schema_passed else "schema_invalid"), + passed=schema_passed, + subject="tool_calls", + expected=len(schema_values), + observed=sum(bool(value) for value in schema_values), + ) + ) + + if not record.success: + checks.append( + EvaluationCheck( + category="execution", + code="execution_failed", + passed=False, + subject="runner", + expected="completed", + observed="failed", + ) + ) + + metrics = _metrics(spec, record, matched, checks) + goal_completed = record.success and all(check.passed for check in checks) + metrics["goal_completion"] = float(goal_completed) + failures = tuple(dict.fromkeys(check.code for check in checks if not check.passed)) + return GoalEvaluation( + scenario_id=spec.id, + goal_completed=goal_completed, + metrics=metrics, + matched_milestones=dict(matched), + checks=tuple(checks), + failure_reason_codes=failures, + ) + + +def _match_milestones( + milestones: tuple[MilestoneSpec, ...], + calls: tuple[ObservedToolCall, ...], +) -> dict[str, int]: + eligible: dict[int, list[int]] = {} + for milestone_index, milestone in enumerate(milestones): + eligible[milestone_index] = [ + call.sequence + for call in sorted(calls, key=lambda item: item.sequence) + if call.success + and call.tool in milestone.tools + and _args_match(call.args, milestone.match_args) + ] + + owner_by_sequence: dict[int, int] = {} + + def assign(milestone_index: int, visited: set[int]) -> bool: + for sequence in eligible[milestone_index]: + if sequence in visited: + continue + visited.add(sequence) + owner = owner_by_sequence.get(sequence) + if owner is not None and not assign(owner, visited): + continue + owner_by_sequence[sequence] = milestone_index + return True + return False + + order = sorted( + range(len(milestones)), + key=lambda index: ( + not milestones[index].required, + -len(milestones[index].match_args), + len(milestones[index].tools), + index, + ), + ) + for milestone_index in order: + assign(milestone_index, set()) + return { + milestones[milestone_index].id: sequence + for sequence, milestone_index in owner_by_sequence.items() + } + + +def _args_match(args: dict[str, Any], expected: dict[str, Any]) -> bool: + return all(_read_path(args, path) == value for path, value in expected.items()) + + +def _metrics( + spec: ScenarioSpec, + record: GoalExecutionRecord, + matched: dict[str, int], + checks: list[EvaluationCheck], +) -> dict[str, float | int | None]: + required = [item for item in spec.milestones if item.required] + required_ids = {item.id for item in required} + completed = len(required_ids & set(matched)) + matched_sequences = set(matched.values()) + extraneous = sum(1 for call in record.calls if call.sequence not in matched_sequences) + dependency = [item for item in checks if item.category == "dependency"] + binding = [item for item in checks if item.category == "binding"] + state = [item for item in checks if item.category == "state"] + schema_values = [call.schema_valid for call in record.calls if call.schema_valid is not None] + failed_calls = [call for call in record.calls if not call.success] + return { + "candidate_required_tool_recall": _tool_set_recall(required, record.candidate_tools), + "plan_required_tool_recall": _tool_set_recall(required, record.planned_tools), + "execution_required_tool_recall": _ratio(completed, len(required)), + "required_tool_recall": _ratio(completed, len(required)), + "milestone_completion": _ratio(completed, len(required)), + "dependency_order_accuracy": _check_ratio(dependency), + "binding_accuracy": _check_ratio(binding), + "final_state_accuracy": _check_ratio(state), + "schema_valid_call_rate": ( + _ratio(sum(bool(value) for value in schema_values), len(schema_values)) + if schema_values + else None + ), + "extraneous_call_rate": _ratio(extraneous, len(record.calls)), + "policy_violation_count": sum( + 1 for item in checks if item.category == "policy" and not item.passed + ), + "call_count": len(record.calls), + "replan_count": record.replans, + "latency_ms": record.latency_ms, + "recovery_attempted": int(bool(failed_calls or record.replans)), + "recovery_success": (int(record.success) if failed_calls or record.replans else None), + "goal_completion": 0.0, + } + + +def _tool_set_recall( + milestones: list[MilestoneSpec], + observed_tools: tuple[str, ...], +) -> float | None: + if not observed_tools: + return None + observed = set(observed_tools) + hits = sum(1 for item in milestones if observed.intersection(item.tools)) + return _ratio(hits, len(milestones)) + + +def _check_ratio(checks: list[EvaluationCheck]) -> float: + return _ratio(sum(item.passed for item in checks), len(checks)) if checks else 1.0 + + +def _ratio(numerator: int, denominator: int) -> float: + return round(numerator / denominator, 6) if denominator else 1.0 + + +def _assertion_scope( + assertion: StateAssertion, + *, + spec: ScenarioSpec, + record: GoalExecutionRecord, +) -> Any: + if assertion.scope == "final_state": + return record.final_state + if assertion.scope == "initial_state": + return spec.initial_state + if assertion.scope == "output": + return record.output + raise ValueError(f"unsupported assertion scope: {assertion.scope!r}") + + +def _evaluate_operator(observed: Any, operator: str, expected: Any) -> bool: + if operator == "exists": + return observed is not _MISSING + if operator == "not_exists": + return observed is _MISSING + if observed is _MISSING: + return False + if operator == "eq": + return observed == expected + if operator == "ne": + return observed != expected + if operator == "contains": + try: + return expected in observed + except TypeError: + return False + if operator == "in": + try: + return observed in expected + except TypeError: + return False + try: + if operator == "gt": + return bool(observed > expected) + if operator == "gte": + return bool(observed >= expected) + if operator == "lt": + return bool(observed < expected) + if operator == "lte": + return bool(observed <= expected) + except TypeError: + return False + raise ValueError(f"unsupported assertion operator: {operator!r}") + + +def _read_path(value: Any, path: str) -> Any: + current = value + normalized = str(path or "").strip() + if normalized in ("", "$"): + return current + if normalized.startswith("$"): + normalized = normalized[1:] + if normalized.startswith("."): + normalized = normalized[1:] + position = 0 + for match in _PATH_TOKEN.finditer(normalized): + if match.start() != position: + return _MISSING + key, index = match.groups() + if key is not None: + if not isinstance(current, dict) or key not in current: + return _MISSING + current = current[key] + else: + idx = int(index) + if not isinstance(current, (list, tuple)) or idx >= len(current): + return _MISSING + current = current[idx] + position = match.end() + return current if position == len(normalized) else _MISSING + + +def _call_at_sequence( + calls: tuple[ObservedToolCall, ...], sequence: int | None +) -> ObservedToolCall | None: + if sequence is None: + return None + return next((call for call in calls if call.sequence == sequence), None) + + +def _tool_at_sequence(calls: tuple[ObservedToolCall, ...], sequence: int | None) -> str: + call = _call_at_sequence(calls, sequence) + return call.tool if call else "" diff --git a/graph_tool_call/evaluation/schema.py b/graph_tool_call/evaluation/schema.py new file mode 100644 index 0000000..1325259 --- /dev/null +++ b/graph_tool_call/evaluation/schema.py @@ -0,0 +1,377 @@ +"""Stable contracts for end-to-end tool-use goal evaluation. + +The evaluator deliberately describes outcomes instead of one exact plan. A +scenario can allow alternative tools while still requiring milestones, +dependency order, value bindings, safety constraints, and final state. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class MilestoneSpec: + """One required or optional semantic step in a tool-use trajectory.""" + + id: str + tools: tuple[str, ...] + required: bool = True + target: bool = False + match_args: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> MilestoneSpec: + return cls( + id=str(value.get("id") or ""), + tools=tuple(str(item) for item in (value.get("tools") or []) if item), + required=bool(value.get("required", True)), + target=bool(value.get("target", False)), + match_args=dict(value.get("match_args") or {}), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "tools": list(self.tools), + "required": self.required, + "target": self.target, + "match_args": dict(self.match_args), + } + + +@dataclass(frozen=True) +class DependencyConstraint: + """Require one milestone to execute before another milestone.""" + + before: str + after: str + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> DependencyConstraint: + return cls(before=str(value.get("before") or ""), after=str(value.get("after") or "")) + + def to_dict(self) -> dict[str, str]: + return {"before": self.before, "after": self.after} + + +@dataclass(frozen=True) +class BindingConstraint: + """Require a source output value to be passed into a later tool argument.""" + + source_milestone: str + source_path: str + target_milestone: str + target_arg: str + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> BindingConstraint: + return cls( + source_milestone=str(value.get("source_milestone") or ""), + source_path=str(value.get("source_path") or ""), + target_milestone=str(value.get("target_milestone") or ""), + target_arg=str(value.get("target_arg") or ""), + ) + + def to_dict(self) -> dict[str, str]: + return { + "source_milestone": self.source_milestone, + "source_path": self.source_path, + "target_milestone": self.target_milestone, + "target_arg": self.target_arg, + } + + +@dataclass(frozen=True) +class StateAssertion: + """A deterministic assertion against initial state, final state, or output.""" + + path: str + operator: str = "eq" + value: Any = None + scope: str = "final_state" + + def __post_init__(self) -> None: + if self.scope not in {"initial_state", "final_state", "output"}: + raise ValueError(f"unsupported assertion scope: {self.scope!r}") + if self.operator not in { + "eq", + "ne", + "exists", + "not_exists", + "contains", + "in", + "gt", + "gte", + "lt", + "lte", + }: + raise ValueError(f"unsupported assertion operator: {self.operator!r}") + if not self.path.strip() and self.path != "$": + raise ValueError("state assertion path must be non-empty") + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> StateAssertion: + return cls( + path=str(value.get("path") or ""), + operator=str(value.get("operator") or "eq"), + value=value.get("value"), + scope=str(value.get("scope") or "final_state"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "scope": self.scope, + "path": self.path, + "operator": self.operator, + "value": self.value, + } + + +@dataclass(frozen=True) +class ScenarioSpec: + """Versioned goal contract used by deterministic and live benchmarks.""" + + id: str + query: str + milestones: tuple[MilestoneSpec, ...] + dependency_constraints: tuple[DependencyConstraint, ...] = () + binding_constraints: tuple[BindingConstraint, ...] = () + final_state_assertions: tuple[StateAssertion, ...] = () + initial_state: dict[str, Any] = field(default_factory=dict) + user_context: dict[str, Any] = field(default_factory=dict) + forbidden_tools: tuple[str, ...] = () + max_calls: int | None = None + max_replans: int | None = None + timeout_sec: float | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.id.strip(): + raise ValueError("scenario id must be non-empty") + if not self.query.strip(): + raise ValueError("scenario query must be non-empty") + if not self.milestones: + raise ValueError("scenario must define at least one milestone") + milestone_ids = [item.id for item in self.milestones] + if any(not item.id.strip() or not item.tools for item in self.milestones): + raise ValueError("every milestone requires a non-empty id and at least one tool") + if len(milestone_ids) != len(set(milestone_ids)): + raise ValueError("milestone ids must be unique") + known = set(milestone_ids) + for item in self.dependency_constraints: + if item.before not in known or item.after not in known: + raise ValueError("dependency constraints must reference known milestones") + if item.before == item.after: + raise ValueError("dependency constraints cannot reference the same milestone") + for item in self.binding_constraints: + if item.source_milestone not in known or item.target_milestone not in known: + raise ValueError("binding constraints must reference known milestones") + if not item.source_path or not item.target_arg: + raise ValueError("binding constraints require source_path and target_arg") + if self.max_calls is not None and self.max_calls < 1: + raise ValueError("max_calls must be positive") + if self.max_replans is not None and self.max_replans < 0: + raise ValueError("max_replans cannot be negative") + if self.timeout_sec is not None and self.timeout_sec <= 0: + raise ValueError("timeout_sec must be positive") + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> ScenarioSpec: + assertions = value.get("final_state_assertions") + if assertions is None: + assertions = value.get("assertions") or [] + return cls( + id=str(value.get("id") or ""), + query=str(value.get("query") or ""), + milestones=tuple( + MilestoneSpec.from_dict(item) + for item in (value.get("milestones") or []) + if isinstance(item, dict) + ), + dependency_constraints=tuple( + DependencyConstraint.from_dict(item) + for item in (value.get("dependency_constraints") or []) + if isinstance(item, dict) + ), + binding_constraints=tuple( + BindingConstraint.from_dict(item) + for item in (value.get("binding_constraints") or []) + if isinstance(item, dict) + ), + final_state_assertions=tuple( + StateAssertion.from_dict(item) for item in assertions if isinstance(item, dict) + ), + initial_state=dict(value.get("initial_state") or {}), + user_context=dict(value.get("user_context") or {}), + forbidden_tools=tuple( + str(item) for item in (value.get("forbidden_tools") or []) if item + ), + max_calls=(int(value["max_calls"]) if value.get("max_calls") is not None else None), + max_replans=( + int(value["max_replans"]) if value.get("max_replans") is not None else None + ), + timeout_sec=( + float(value["timeout_sec"]) if value.get("timeout_sec") is not None else None + ), + metadata=dict(value.get("metadata") or {}), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "query": self.query, + "milestones": [item.to_dict() for item in self.milestones], + "dependency_constraints": [item.to_dict() for item in self.dependency_constraints], + "binding_constraints": [item.to_dict() for item in self.binding_constraints], + "final_state_assertions": [item.to_dict() for item in self.final_state_assertions], + "initial_state": dict(self.initial_state), + "user_context": dict(self.user_context), + "forbidden_tools": list(self.forbidden_tools), + "max_calls": self.max_calls, + "max_replans": self.max_replans, + "timeout_sec": self.timeout_sec, + "metadata": dict(self.metadata), + } + + +@dataclass(frozen=True) +class ObservedToolCall: + """Compact, transport-neutral record of one resolved tool invocation.""" + + sequence: int + tool: str + args: dict[str, Any] = field(default_factory=dict) + output: Any = None + success: bool = True + schema_valid: bool | None = None + duration_ms: int = 0 + error_kind: str = "" + + def to_dict(self) -> dict[str, Any]: + from graph_tool_call.learning import scrub_trace_payload + + return { + "sequence": self.sequence, + "tool": self.tool, + "args": scrub_trace_payload(dict(self.args)), + "output": scrub_trace_payload(self.output), + "success": self.success, + "schema_valid": self.schema_valid, + "duration_ms": self.duration_ms, + "error_kind": self.error_kind, + } + + +@dataclass(frozen=True) +class GoalExecutionRecord: + """Everything the evaluator needs; raw API payload persistence is optional.""" + + calls: tuple[ObservedToolCall, ...] + success: bool + retrieved_tools: tuple[str, ...] = () + candidate_tools: tuple[str, ...] = () + planned_tools: tuple[str, ...] = () + final_state: dict[str, Any] = field(default_factory=dict) + output: Any = None + latency_ms: int = 0 + replans: int = 0 + metadata: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_execution_trace( + cls, + trace: Any, + *, + plan: Any | None = None, + retrieved_tools: list[str] | tuple[str, ...] = (), + candidate_tools: list[str] | tuple[str, ...] = (), + final_state: dict[str, Any] | None = None, + replans: int = 0, + schema_valid: bool | None = None, + metadata: dict[str, Any] | None = None, + ) -> GoalExecutionRecord: + calls = [] + for sequence, step in enumerate(getattr(trace, "steps", ()) or (), start=1): + error = getattr(step, "error", None) + error_kind = str((error or {}).get("kind") or "") + calls.append( + ObservedToolCall( + sequence=sequence, + tool=str(getattr(step, "tool", "")), + args=dict(getattr(step, "args_resolved", {}) or {}), + output=getattr(step, "output", None), + success=error is None, + schema_valid=( + False + if error_kind in {"schema", "validation"} + else schema_valid + if error is None + else None + ), + duration_ms=int(getattr(step, "duration_ms", 0) or 0), + error_kind=error_kind, + ) + ) + planned_tools = tuple( + str(getattr(step, "tool", "")) for step in (getattr(plan, "steps", ()) or ()) + ) + return cls( + calls=tuple(calls), + success=bool(getattr(trace, "success", False)), + retrieved_tools=tuple(str(item) for item in retrieved_tools), + candidate_tools=tuple(str(item) for item in candidate_tools), + planned_tools=planned_tools, + final_state=dict(final_state or {}), + output=getattr(trace, "output", None), + latency_ms=int(getattr(trace, "total_duration_ms", 0) or 0), + replans=int(replans), + metadata=dict(metadata or {}), + ) + + +@dataclass(frozen=True) +class EvaluationCheck: + """One explainable pass/fail item in a goal evaluation.""" + + category: str + code: str + passed: bool + subject: str + expected: Any = None + observed: Any = None + + def to_dict(self) -> dict[str, Any]: + from graph_tool_call.learning import scrub_trace_payload + + return { + "category": self.category, + "code": self.code, + "passed": self.passed, + "subject": self.subject, + "expected": scrub_trace_payload(self.expected), + "observed": scrub_trace_payload(self.observed), + } + + +@dataclass(frozen=True) +class GoalEvaluation: + """Deterministic outcome and stage metrics for one scenario execution.""" + + scenario_id: str + goal_completed: bool + metrics: dict[str, float | int | None] + matched_milestones: dict[str, int] + checks: tuple[EvaluationCheck, ...] + failure_reason_codes: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "scenario_id": self.scenario_id, + "goal_completed": self.goal_completed, + "metrics": dict(self.metrics), + "matched_milestones": dict(self.matched_milestones), + "checks": [item.to_dict() for item in self.checks], + "failure_reason_codes": list(self.failure_reason_codes), + } diff --git a/tests/test_goal_completion_benchmark.py b/tests/test_goal_completion_benchmark.py new file mode 100644 index 0000000..662194f --- /dev/null +++ b/tests/test_goal_completion_benchmark.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from benchmarks.goal_completion.run import run_benchmark + + +def test_goal_completion_benchmark_reports_full_engine_baseline(): + report = run_benchmark() + summary = report["summary"] + + assert report["methodology"] == "natural_language_retrieve_plan_execute_goal_state" + assert report["model"] == "none" + assert report["scenario_count"] == 6 + assert summary["cases"] == 6 + assert summary["uncaught_error_count"] == 0 + assert summary["completed"] == 3 + assert summary["goal_completion_rate"] == 0.5 + assert 0.0 < summary["candidate_required_tool_recall"] < 1.0 + assert 0.0 < summary["plan_required_tool_recall"] < 1.0 + assert summary["binding_accuracy"] == 0.5 + assert summary["schema_valid_call_rate"] == 1.0 + assert 0.0 < summary["extraneous_call_rate"] < 0.2 + + rows = {row["id"]: row for row in report["cases"]} + assert rows["product_detail"]["evaluation"]["goal_completed"] is True + assert rows["coupon_checkout"]["evaluation"]["goal_completed"] is True + assert rows["create_review"]["evaluation"]["goal_completed"] is True + assert rows["inventory_lookup"]["selected_target"] == "searchProducts" + assert "missing_milestone" in rows["inventory_lookup"]["evaluation"]["failure_reason_codes"] + assert rows["add_cart_item"]["selected_target"] == "getCart" + assert rows["shipment_tracking"]["selected_target"] == "findOrders" + + +def test_goal_completion_benchmark_does_not_feed_gold_plan_to_engine(): + report = run_benchmark() + + for row in report["cases"]: + assert "expected_plan" not in row + assert row["planned_tools"] == row["executed_tools"] diff --git a/tests/test_goal_evaluation.py b/tests/test_goal_evaluation.py new file mode 100644 index 0000000..d20650a --- /dev/null +++ b/tests/test_goal_evaluation.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +from graph_tool_call import GoalExecutionRecord, ScenarioSpec, evaluate_goal_execution +from graph_tool_call.evaluation import ObservedToolCall +from graph_tool_call.plan import ExecutionTrace, Plan, PlanStep, StepTrace + + +def _scenario() -> ScenarioSpec: + return ScenarioSpec.from_dict( + { + "id": "change_shipping_address", + "query": "내 최근 주문을 찾아 배송지를 서울로 바꾸고 확인해줘", + "milestones": [ + {"id": "find_order", "tools": ["findOrders", "searchOrders"]}, + {"id": "change_address", "tools": ["updateShippingAddress"]}, + {"id": "verify_order", "tools": ["getOrderDetail"], "target": True}, + ], + "dependency_constraints": [ + {"before": "find_order", "after": "change_address"}, + {"before": "change_address", "after": "verify_order"}, + ], + "binding_constraints": [ + { + "source_milestone": "find_order", + "source_path": "orders[0].orderId", + "target_milestone": "change_address", + "target_arg": "orderId", + }, + { + "source_milestone": "find_order", + "source_path": "orders[0].orderId", + "target_milestone": "verify_order", + "target_arg": "orderId", + }, + ], + "final_state_assertions": [ + { + "scope": "final_state", + "path": "orders.O1.shippingAddress", + "operator": "eq", + "value": "서울", + } + ], + "forbidden_tools": ["cancelOrder"], + "max_calls": 4, + "max_replans": 1, + "timeout_sec": 5, + } + ) + + +def _valid_record() -> GoalExecutionRecord: + return GoalExecutionRecord( + calls=( + ObservedToolCall( + sequence=1, + tool="searchOrders", + args={"q": "recent"}, + output={"orders": [{"orderId": "O1"}]}, + schema_valid=True, + ), + ObservedToolCall( + sequence=2, + tool="updateShippingAddress", + args={"orderId": "O1", "shippingAddress": "서울"}, + output={"orderId": "O1", "updated": True}, + schema_valid=True, + ), + ObservedToolCall( + sequence=3, + tool="getOrderDetail", + args={"orderId": "O1"}, + output={"orderId": "O1", "shippingAddress": "서울"}, + schema_valid=True, + ), + ), + success=True, + retrieved_tools=("getOrderDetail", "updateShippingAddress", "searchOrders"), + candidate_tools=("getOrderDetail", "updateShippingAddress", "searchOrders"), + planned_tools=("searchOrders", "updateShippingAddress", "getOrderDetail"), + final_state={"orders": {"O1": {"shippingAddress": "서울"}}}, + latency_ms=40, + ) + + +def test_goal_evaluator_accepts_alternative_tools_and_valid_outcome(): + result = evaluate_goal_execution(_scenario(), _valid_record()) + + assert result.goal_completed is True + assert result.failure_reason_codes == () + assert result.metrics["candidate_required_tool_recall"] == 1.0 + assert result.metrics["plan_required_tool_recall"] == 1.0 + assert result.metrics["dependency_order_accuracy"] == 1.0 + assert result.metrics["binding_accuracy"] == 1.0 + assert result.metrics["final_state_accuracy"] == 1.0 + assert result.metrics["schema_valid_call_rate"] == 1.0 + assert result.metrics["goal_completion"] == 1.0 + + +def test_goal_evaluator_separates_order_binding_and_final_state_failures(): + valid = _valid_record() + broken = GoalExecutionRecord( + calls=( + ObservedToolCall( + sequence=1, + tool="updateShippingAddress", + args={"orderId": "O1", "shippingAddress": "서울"}, + output={"orderId": "O1", "updated": True}, + schema_valid=True, + ), + ObservedToolCall( + sequence=2, + tool="searchOrders", + args={"q": "recent"}, + output={"orders": [{"orderId": "O1"}]}, + schema_valid=True, + ), + ObservedToolCall( + sequence=3, + tool="getOrderDetail", + args={"orderId": "WRONG"}, + output={"orderId": "WRONG", "shippingAddress": "부산"}, + schema_valid=True, + ), + ), + success=True, + candidate_tools=valid.candidate_tools, + planned_tools=("updateShippingAddress", "searchOrders", "getOrderDetail"), + final_state={"orders": {"O1": {"shippingAddress": "부산"}}}, + ) + + result = evaluate_goal_execution(_scenario(), broken) + + assert result.goal_completed is False + assert "invalid_dependency_order" in result.failure_reason_codes + assert "binding_mismatch" in result.failure_reason_codes + assert "goal_state_mismatch" in result.failure_reason_codes + + +def test_goal_evaluator_reports_missing_milestone_policy_and_budget(): + record = GoalExecutionRecord( + calls=( + ObservedToolCall(sequence=1, tool="findOrders", output={"orders": []}), + ObservedToolCall(sequence=2, tool="cancelOrder"), + ObservedToolCall(sequence=3, tool="noop"), + ObservedToolCall(sequence=4, tool="noop"), + ObservedToolCall(sequence=5, tool="noop"), + ), + success=True, + final_state={"orders": {}}, + replans=2, + latency_ms=6000, + ) + + result = evaluate_goal_execution(_scenario(), record) + + assert result.goal_completed is False + assert "missing_milestone" in result.failure_reason_codes + assert "forbidden_tool_called" in result.failure_reason_codes + assert "max_calls_exceeded" in result.failure_reason_codes + assert "max_replans_exceeded" in result.failure_reason_codes + assert "timeout_exceeded" in result.failure_reason_codes + assert result.metrics["policy_violation_count"] == 1 + + +def test_goal_evaluation_serialization_scrubs_sensitive_values(): + scenario = ScenarioSpec.from_dict( + { + "id": "secret_safe", + "query": "인증 상태 확인", + "milestones": [{"id": "check", "tools": ["checkAuth"]}], + "final_state_assertions": [ + {"path": "email", "operator": "eq", "value": "person@example.com"} + ], + } + ) + record = GoalExecutionRecord( + calls=(ObservedToolCall(sequence=1, tool="checkAuth"),), + success=True, + final_state={"email": "wrong@example.com"}, + ) + + payload = evaluate_goal_execution(scenario, record).to_dict() + state_check = next(item for item in payload["checks"] if item["category"] == "state") + + assert "person@example.com" not in str(state_check) + assert "wrong@example.com" not in str(state_check) + + call_payload = ObservedToolCall( + sequence=1, + tool="checkAuth", + args={"authorization": "Bearer secret-token"}, + output={"email": "person@example.com"}, + ).to_dict() + assert "secret-token" not in str(call_payload) + assert "person@example.com" not in str(call_payload) + + +def test_goal_execution_record_adapts_plan_runner_trace(): + plan = Plan( + id="p1", + goal="lookup", + steps=[PlanStep(id="s1", tool="findOrders", args={"q": "recent"})], + ) + trace = ExecutionTrace( + plan_id="p1", + success=True, + steps=[ + StepTrace( + id="s1", + tool="findOrders", + args_resolved={"q": "recent"}, + output={"orders": [{"orderId": "O1"}]}, + duration_ms=3, + ) + ], + output={"orders": [{"orderId": "O1"}]}, + total_duration_ms=4, + ) + + record = GoalExecutionRecord.from_execution_trace( + trace, + plan=plan, + retrieved_tools=["findOrders"], + candidate_tools=["findOrders"], + schema_valid=True, + ) + + assert record.success is True + assert record.planned_tools == ("findOrders",) + assert record.calls[0].args == {"q": "recent"} + assert record.calls[0].schema_valid is True + + +def test_scenario_contract_round_trips_and_validates_references(): + scenario = _scenario() + + assert ScenarioSpec.from_dict(scenario.to_dict()) == scenario + + invalid = scenario.to_dict() + invalid["dependency_constraints"] = [{"before": "unknown", "after": "find_order"}] + try: + ScenarioSpec.from_dict(invalid) + except ValueError as exc: + assert "known milestones" in str(exc) + else: # pragma: no cover - contract guard + raise AssertionError("invalid milestone reference was accepted") + + +def test_milestone_matching_handles_overlapping_alternatives_and_arg_filters(): + scenario = ScenarioSpec.from_dict( + { + "id": "overlapping", + "query": "두 주문 상태를 확인", + "milestones": [ + {"id": "any_order", "tools": ["getOrder", "findOrder"]}, + { + "id": "specific_order", + "tools": ["getOrder"], + "match_args": {"orderId": "O2"}, + }, + ], + } + ) + record = GoalExecutionRecord( + calls=( + ObservedToolCall(sequence=1, tool="getOrder", args={"orderId": "O2"}), + ObservedToolCall(sequence=2, tool="findOrder", args={"query": "recent"}), + ), + success=True, + ) + + result = evaluate_goal_execution(scenario, record) + + assert result.goal_completed is True + assert result.matched_milestones == {"specific_order": 1, "any_order": 2} + assert result.metrics["extraneous_call_rate"] == 0.0 + + +def test_unsupported_partial_path_is_reported_as_state_mismatch(): + scenario = ScenarioSpec.from_dict( + { + "id": "invalid_path", + "query": "마지막 주문 확인", + "milestones": [{"id": "read", "tools": ["getOrders"]}], + "final_state_assertions": [ + {"path": "orders[-1].status", "operator": "eq", "value": "paid"} + ], + } + ) + record = GoalExecutionRecord( + calls=(ObservedToolCall(sequence=1, tool="getOrders"),), + success=True, + final_state={"orders": [{"status": "paid"}]}, + ) + + result = evaluate_goal_execution(scenario, record) + + assert result.goal_completed is False + assert result.failure_reason_codes == ("goal_state_mismatch",) From 4f0f173685e1d94671582a90cac945ab34c61334 Mon Sep 17 00:00:00 2001 From: SonAIengine Date: Sun, 16 Aug 2026 16:29:57 +0900 Subject: [PATCH 2/2] Add Arazzo long-horizon evaluation --- Makefile | 10 +- benchmarks/arazzo_long_horizon/__init__.py | 1 + benchmarks/arazzo_long_horizon/fixtures.py | 239 + benchmarks/arazzo_long_horizon/run.py | 407 ++ .../results/arazzo_long_horizon_0.42.json | 3904 +++++++++++++++++ docs/benchmarks.md | 20 + docs/research/long-horizon-goal-evaluation.md | 39 + tests/test_arazzo_long_horizon_benchmark.py | 37 + 8 files changed, 4656 insertions(+), 1 deletion(-) create mode 100644 benchmarks/arazzo_long_horizon/__init__.py create mode 100644 benchmarks/arazzo_long_horizon/fixtures.py create mode 100644 benchmarks/arazzo_long_horizon/run.py create mode 100644 benchmarks/results/arazzo_long_horizon_0.42.json create mode 100644 tests/test_arazzo_long_horizon_benchmark.py diff --git a/Makefile b/Makefile index c1c4b29..bcfe55f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: quick lint test verify research-check research-check-unit research-check-deterministic research-check-smoke paper-corpus-check paper-corpus-internal-review-check paper-corpus-claim-check paper-adapter-conformance paper-baseline-run paper-graph-ablation paper-producer-coverage paper-output-promotion paper-candidate-admission paper-contract-projection paper-model-loop paper-llm-catalog-baseline paper-model-loop-analysis paper-toolinkos-parity paper-openapi-closure paper-harness-check goal-completion-benchmark xgen-benchmark xgen-llm-benchmark xgen-scale-snapshot xgen-scale-snapshot-check xgen-scale-acceptance xgen-scale-sweep xgen-scale-gate-check xgen-scale-028-gate-check xgen-scale-contract-ablation bfcl-benchmark bfcl-llm-benchmark bfcl-sweep bfcl-027-gate bfcl-027-gate-check bfcl-028-gate bfcl-028-gate-check bfcl-failure-subset bfcl-inspect-failures bfcl-hard-cases release-check pypi-smoke public-smoke launch-evidence launch-evidence-check observability-evidence observability-evidence-check +.PHONY: quick lint test verify research-check research-check-unit research-check-deterministic research-check-smoke paper-corpus-check paper-corpus-internal-review-check paper-corpus-claim-check paper-adapter-conformance paper-baseline-run paper-graph-ablation paper-producer-coverage paper-output-promotion paper-candidate-admission paper-contract-projection paper-model-loop paper-llm-catalog-baseline paper-model-loop-analysis paper-toolinkos-parity paper-openapi-closure paper-harness-check goal-completion-benchmark arazzo-long-horizon-benchmark xgen-benchmark xgen-llm-benchmark xgen-scale-snapshot xgen-scale-snapshot-check xgen-scale-acceptance xgen-scale-sweep xgen-scale-gate-check xgen-scale-028-gate-check xgen-scale-contract-ablation bfcl-benchmark bfcl-llm-benchmark bfcl-sweep bfcl-027-gate bfcl-027-gate-check bfcl-028-gate bfcl-028-gate-check bfcl-failure-subset bfcl-inspect-failures bfcl-hard-cases release-check pypi-smoke public-smoke launch-evidence launch-evidence-check observability-evidence observability-evidence-check quick: scripts/quick-check.sh @@ -178,6 +178,14 @@ goal-completion-benchmark: --scenarios "$${SCENARIOS:-benchmarks/goal_completion/scenarios.json}" \ --output "$${OUT:-/tmp/graph-tool-call-goal-completion.json}" +arazzo-long-horizon-benchmark: + poetry run python -m benchmarks.arazzo_long_horizon.run \ + --catalog-size "$${CATALOG_SIZE:-1000}" \ + --lengths "$${LENGTHS:-3,10,30}" \ + --top-k "$${TOP_K:-8}" \ + --token-budget "$${TOKEN_BUDGET:-2048}" \ + --out "$${OUT:-/tmp/graph-tool-call-arazzo-long-horizon.json}" + xgen-benchmark: poetry run python -m benchmarks.xgen_tool_graph.run --suite all diff --git a/benchmarks/arazzo_long_horizon/__init__.py b/benchmarks/arazzo_long_horizon/__init__.py new file mode 100644 index 0000000..4ef189d --- /dev/null +++ b/benchmarks/arazzo_long_horizon/__init__.py @@ -0,0 +1 @@ +"""Deterministic Arazzo long-horizon evaluation.""" diff --git a/benchmarks/arazzo_long_horizon/fixtures.py b/benchmarks/arazzo_long_horizon/fixtures.py new file mode 100644 index 0000000..7d6b316 --- /dev/null +++ b/benchmarks/arazzo_long_horizon/fixtures.py @@ -0,0 +1,239 @@ +"""Generate domain-neutral long-horizon OpenAPI and Arazzo fixtures.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class WorkflowFamily: + """One unseen-family workflow configuration.""" + + id: str + title: str + query: str + first_operation: str + stage_operation_prefix: str + target_operation: str + certificate_field: str + + +FAMILIES: tuple[WorkflowFamily, ...] = ( + WorkflowFamily( + id="supplier_onboarding", + title="Supplier onboarding", + query="공급업체 등록 절차를 모두 완료하고 활성화 인증서를 발급해줘", + first_operation="resolveSupplierTenant", + stage_operation_prefix="completeSupplierOnboardingStage", + target_operation="issueSupplierActivationCertificate", + certificate_field="activationCertificateId", + ), + WorkflowFamily( + id="incident_recovery", + title="Incident recovery", + query="서비스 장애 복구 절차를 모두 수행하고 복구 검증 보고서를 발급해줘", + first_operation="openIncidentInvestigation", + stage_operation_prefix="completeIncidentRecoveryStage", + target_operation="issueIncidentRecoveryReport", + certificate_field="recoveryReportId", + ), + WorkflowFamily( + id="publication_approval", + title="Publication approval", + query="콘텐츠 검토 절차를 모두 완료하고 게시 승인 증명서를 발급해줘", + first_operation="openPublicationReview", + stage_operation_prefix="completePublicationReviewStage", + target_operation="issuePublicationApprovalCertificate", + certificate_field="publicationCertificateId", + ), +) + + +def build_fixture( + family: WorkflowFamily, + *, + workflow_length: int, + catalog_size: int, +) -> dict[str, Any]: + """Return an OpenAPI catalog, Arazzo workflow, and evaluator-only gold.""" + + if workflow_length < 2: + raise ValueError("workflow_length must be at least 2") + if catalog_size < workflow_length: + raise ValueError("catalog_size must be >= workflow_length") + + operations = _operation_names(family, workflow_length) + paths: dict[str, Any] = {} + for index, operation in enumerate(operations, start=1): + request_field = "workflowSeed" if index == 1 else f"handoffToken{index - 1:02d}" + response_properties = ( + { + family.certificate_field: {"type": "string"}, + "status": {"type": "string"}, + } + if index == workflow_length + else {f"stageEvidence{index:02d}": {"type": "string"}} + ) + summary = ( + f"{family.title}: {family.query} - 절차 시작" + if index == 1 + else ( + f"{family.title}: {family.query} - 최종 결과 발급" + if index == workflow_length + else f"{family.title}: {family.query} - 필수 단계 {index} 수행" + ) + ) + paths[f"/{family.id}/stages/{index:02d}"] = { + "post": { + "operationId": operation, + "summary": summary, + "description": f"Step {index} of {workflow_length} for {family.title}.", + "tags": [family.title], + "parameters": [ + { + "name": request_field, + "in": "query", + "required": True, + "schema": {"type": "string"}, + } + ], + "responses": { + "200": { + "description": "Successful stage result", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": response_properties, + "required": list(response_properties), + } + } + }, + } + }, + } + } + + distractor_count = catalog_size - workflow_length + for index in range(1, distractor_count + 1): + operation = f"lookupArchivedReference{index:04d}" + paths[f"/archive/references/{index:04d}"] = { + "get": { + "operationId": operation, + "summary": f"Read archived reference record {index}", + "description": "Unrelated read-only catalog distractor.", + "tags": ["Archive"], + "responses": { + "200": { + "description": "Archived record", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": {"recordId": {"type": "string"}}, + } + } + }, + } + }, + } + } + + openapi = { + "openapi": "3.1.0", + "info": {"title": f"{family.title} catalog", "version": "1.0.0"}, + "servers": [{"url": "https://example.invalid"}], + "paths": paths, + } + steps: list[dict[str, Any]] = [] + for index, operation in enumerate(operations, start=1): + step: dict[str, Any] = { + "stepId": f"step{index:02d}", + "operationId": operation, + } + if index < workflow_length: + step["outputs"] = {f"stage{index:02d}": f"$response.body#/stageEvidence{index:02d}"} + if index > 1: + step["parameters"] = [ + { + "name": f"handoffToken{index - 1:02d}", + "in": "query", + "value": f"$steps.step{index - 1:02d}.outputs.stage{index - 1:02d}", + } + ] + steps.append(step) + + arazzo = { + "arazzo": "1.1.0", + "info": {"title": f"{family.title} workflow", "version": "1.0.0"}, + "sourceDescriptions": [ + { + "name": "catalog", + "url": "https://example.invalid/openapi.json", + "type": "openapi", + } + ], + "workflows": [ + { + "workflowId": f"{family.id}Flow", + "summary": f"Complete {family.title}", + "steps": steps, + } + ], + } + milestones = [ + { + "id": f"stage_{index:02d}", + "tools": [operation], + "target": index == workflow_length, + } + for index, operation in enumerate(operations, start=1) + ] + scenario = { + "id": f"{family.id}_{workflow_length}", + "query": family.query, + "milestones": milestones, + "dependency_constraints": [ + {"before": f"stage_{index:02d}", "after": f"stage_{index + 1:02d}"} + for index in range(1, workflow_length) + ], + "binding_constraints": [ + { + "source_milestone": f"stage_{index:02d}", + "source_path": f"stageEvidence{index:02d}", + "target_milestone": f"stage_{index + 1:02d}", + "target_arg": f"handoffToken{index:02d}", + } + for index in range(1, workflow_length) + ], + "final_state_assertions": [ + { + "scope": "output", + "path": family.certificate_field, + "operator": "eq", + "value": f"CERT-{family.id}", + } + ], + "max_calls": workflow_length, + "max_replans": 0, + "timeout_sec": 30, + } + return { + "family": family, + "openapi": openapi, + "arazzo": arazzo, + "operations": operations, + "scenario": scenario, + "entities": {"workflowSeed": f"SEED-{family.id}"}, + } + + +def _operation_names(family: WorkflowFamily, workflow_length: int) -> list[str]: + if workflow_length == 2: + return [family.first_operation, family.target_operation] + return [ + family.first_operation, + *[f"{family.stage_operation_prefix}{index:02d}" for index in range(2, workflow_length)], + family.target_operation, + ] diff --git a/benchmarks/arazzo_long_horizon/run.py b/benchmarks/arazzo_long_horizon/run.py new file mode 100644 index 0000000..4b7e7f8 --- /dev/null +++ b/benchmarks/arazzo_long_horizon/run.py @@ -0,0 +1,407 @@ +"""Compare long-horizon goal completion with and without Arazzo evidence.""" + +from __future__ import annotations + +import argparse +import json +import math +import time +from pathlib import Path +from typing import Any + +from benchmarks.arazzo_long_horizon.fixtures import FAMILIES, WorkflowFamily, build_fixture +from graph_tool_call import __version__ +from graph_tool_call.evaluation import GoalExecutionRecord, ScenarioSpec, evaluate_goal_execution +from graph_tool_call.graphify import ( + apply_arazzo_workflows, + build_candidate_set, + ingest_openapi_graphify, + retrieve_graphify, + select_target_candidate, +) +from graph_tool_call.ingest.openapi import ingest_openapi +from graph_tool_call.plan import PathSynthesizer, PlanRunner + +DEFAULT_LENGTHS = (3, 10, 30) + + +class WorkflowSandbox: + """Execute generated workflow operations with strict value handoffs.""" + + def __init__(self, fixture: dict[str, Any]) -> None: + self._family: WorkflowFamily = fixture["family"] + self._operations = list(fixture["operations"]) + self._stages = {name: index for index, name in enumerate(self._operations, start=1)} + self.state = {"completed_stage": 0, "status": "pending"} + + def call_tool(self, tool: str, args: dict[str, Any]) -> dict[str, Any]: + stage = self._stages.get(tool) + if stage is None: + raise RuntimeError(f"unsupported benchmark tool: {tool}") + if stage == 1: + expected = f"SEED-{self._family.id}" + _require_value(args, "workflowSeed", expected) + else: + field = f"handoffToken{stage - 1:02d}" + _require_value(args, field, f"EVIDENCE-{self._family.id}-{stage - 1:02d}") + self.state["completed_stage"] = stage + if stage == len(self._operations): + self.state["status"] = "complete" + return { + self._family.certificate_field: f"CERT-{self._family.id}", + "status": "complete", + } + return {f"stageEvidence{stage:02d}": f"EVIDENCE-{self._family.id}-{stage:02d}"} + + def snapshot(self) -> dict[str, Any]: + return dict(self.state) + + +def run_benchmark( + *, + catalog_size: int = 1000, + workflow_lengths: tuple[int, ...] = DEFAULT_LENGTHS, + top_k: int = 8, + token_budget: int = 2048, +) -> dict[str, Any]: + """Run paired no-workflow/Arazzo conditions for each horizon tier.""" + + if len(workflow_lengths) > len(FAMILIES): + raise ValueError("workflow_lengths cannot exceed available unseen families") + rows: list[dict[str, Any]] = [] + for family, length in zip(FAMILIES, workflow_lengths): + fixture = build_fixture(family, workflow_length=length, catalog_size=catalog_size) + baseline = _run_condition( + fixture, + use_arazzo=False, + top_k=top_k, + token_budget=token_budget, + ) + enriched = _run_condition( + fixture, + use_arazzo=True, + top_k=top_k, + token_budget=token_budget, + ) + rows.append( + { + "id": fixture["scenario"]["id"], + "family": family.id, + "workflow_length": length, + "catalog_size": catalog_size, + "baseline": baseline, + "with_arazzo": enriched, + "lift": _condition_lift(baseline, enriched), + } + ) + + summary = _summarize(rows) + return { + "benchmark": "Arazzo Long-Horizon Paired Evaluation", + "methodology": "paired_deterministic_retrieve_plan_execute_goal_state", + "model": "none", + "graph_tool_call_version": __version__, + "catalog_size": catalog_size, + "workflow_lengths": list(workflow_lengths), + "top_k": top_k, + "token_budget": token_budget, + "summary": summary, + "cases": rows, + "limitations": [ + "The benchmark isolates engine behavior and does not test independent LLM reasoning.", + "Generated catalogs are contract-distinct synthetic fixtures, not production APIs.", + ( + "Arazzo evidence is supplied explicitly; workflow discovery without Arazzo " + "is the baseline." + ), + ], + } + + +def _run_condition( + fixture: dict[str, Any], + *, + use_arazzo: bool, + top_k: int, + token_budget: int, +) -> dict[str, Any]: + started = time.perf_counter() + tools, _ = ingest_openapi(fixture["openapi"]) + # Match the public collection-artifact defaults: request contracts are + # promoted for planning, while unrelated raw response leaves stay out of + # the search index. Arazzo then supplies only the explicit runtime aliases. + graph, edge_stats = ingest_openapi_graphify(tools, promote_contract_signals=True) + workflow_summary: dict[str, Any] = {} + if use_arazzo: + workflow_summary = apply_arazzo_workflows(graph, fixture["arazzo"]) + build_latency_ms = (time.perf_counter() - started) * 1000 + graph_payload = { + "graph": graph.graph.to_dict(), + "tools": {name: tool.to_dict() for name, tool in graph.tools.items()}, + } + + scenario = ScenarioSpec.from_dict(fixture["scenario"]) + started = time.perf_counter() + retrieval = retrieve_graphify( + graph, + scenario.query, + top_k=top_k, + depth=0, + token_budget=token_budget, + include_evidence=True, + ) + retrieval_latency_ms = (time.perf_counter() - started) * 1000 + retrieved = [str(row["name"]) for row in retrieval.get("results") or []] + selector = select_target_candidate( + scenario.query, + retrieved, + graph_payload["tools"], + retrieval_results=list(retrieval.get("results") or []), + ) + selected_target = str(selector.get("selected_target") or "") + candidates: list[str] = [] + plan = None + trace = None + failure: dict[str, str] = {} + sandbox = WorkflowSandbox(fixture) + + started = time.perf_counter() + if selected_target: + expanded = build_candidate_set( + retrieved, + graph_payload["tools"], + expansion_seed=[selected_target], + max_producers_per_field=1, + max_hops=len(fixture["operations"]) + 1, + ) + candidates = [str(name) for name in expanded.get("candidates") or []] + try: + plan = PathSynthesizer( + graph_payload, + max_depth=len(fixture["operations"]) + 1, + ).synthesize( + target=selected_target, + entities=dict(fixture["entities"]), + goal=scenario.query, + ) + trace = PlanRunner(sandbox.call_tool, binding_recovery=True).run( + plan, + input_context=dict(fixture["entities"]), + ) + except Exception as exc: # noqa: BLE001 - benchmark records stage failures + failure = {"reason": type(exc).__name__, "message": str(exc)} + plan_execute_latency_ms = (time.perf_counter() - started) * 1000 + + planned_tools = tuple(str(step.tool) for step in (getattr(plan, "steps", ()) or ())) + if trace is None: + record = GoalExecutionRecord( + calls=(), + success=False, + retrieved_tools=tuple(retrieved), + candidate_tools=tuple(candidates), + planned_tools=planned_tools, + final_state=sandbox.snapshot(), + ) + else: + record = GoalExecutionRecord.from_execution_trace( + trace, + plan=plan, + retrieved_tools=retrieved, + candidate_tools=candidates, + final_state=sandbox.snapshot(), + schema_valid=True, + ) + evaluation = evaluate_goal_execution(scenario, record) + expected = list(fixture["operations"]) + executed = [call.tool for call in record.calls] + target = expected[-1] + return { + "use_arazzo": use_arazzo, + "tool_count": len(graph.tools), + "edge_count": graph.graph.edge_count(), + "edge_stats": edge_stats, + "workflow_summary": workflow_summary, + "target_hit_at_k": float(target in retrieved), + "selected_target_exact": float(selected_target == target), + "selected_target": selected_target, + "retrieved_tools": retrieved, + "candidate_count": len(candidates), + "planned_call_count": len(planned_tools), + "executed_call_count": len(executed), + "plan_order_exact": float(list(planned_tools) == expected), + "execution_order_exact": float(executed == expected), + "planned_tools": list(planned_tools), + "executed_tools": executed, + "runner_success": bool(record.success), + "failure": failure, + "evaluation": evaluation.to_dict(), + "latency_ms": { + "build": round(build_latency_ms, 3), + "retrieve": round(retrieval_latency_ms, 3), + "plan_execute": round(plan_execute_latency_ms, 3), + }, + "token_budget_used": int( + retrieval.get("token_budget_used") + or (retrieval.get("stats") or {}).get("token_budget_used") + or 0 + ), + } + + +def _condition_lift(baseline: dict[str, Any], enriched: dict[str, Any]) -> dict[str, float]: + baseline_metrics = baseline["evaluation"]["metrics"] + enriched_metrics = enriched["evaluation"]["metrics"] + names = ( + "goal_completion", + "candidate_required_tool_recall", + "plan_required_tool_recall", + "execution_required_tool_recall", + "dependency_order_accuracy", + "binding_accuracy", + ) + return { + name: round( + float(enriched_metrics.get(name) or 0) - float(baseline_metrics.get(name) or 0), + 6, + ) + for name in names + } + + +def _summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: + baseline = [row["baseline"] for row in rows] + enriched = [row["with_arazzo"] for row in rows] + + def average(items: list[dict[str, Any]], path: tuple[str, ...]) -> float: + values: list[float] = [] + for item in items: + value: Any = item + for key in path: + value = value.get(key) if isinstance(value, dict) else None + values.append(float(value or 0)) + return round(sum(values) / len(values), 6) if values else 0.0 + + with_arazzo = { + "target_hit_at_k": average(enriched, ("target_hit_at_k",)), + "selected_target_exact": average(enriched, ("selected_target_exact",)), + "candidate_required_tool_recall": average( + enriched, + ("evaluation", "metrics", "candidate_required_tool_recall"), + ), + "plan_required_tool_recall": average( + enriched, + ("evaluation", "metrics", "plan_required_tool_recall"), + ), + "execution_required_tool_recall": average( + enriched, + ("evaluation", "metrics", "execution_required_tool_recall"), + ), + "goal_completion_rate": average(enriched, ("evaluation", "metrics", "goal_completion")), + "plan_order_exact": average(enriched, ("plan_order_exact",)), + "execution_order_exact": average(enriched, ("execution_order_exact",)), + "binding_accuracy": average(enriched, ("evaluation", "metrics", "binding_accuracy")), + "token_budget_used": { + "average": average(enriched, ("token_budget_used",)), + "max": max((int(item.get("token_budget_used") or 0) for item in enriched), default=0), + }, + "latency_ms": { + stage: _latency_percentiles(enriched, stage) + for stage in ("build", "retrieve", "plan_execute") + }, + } + without_arazzo = { + "goal_completion_rate": average(baseline, ("evaluation", "metrics", "goal_completion")), + "plan_order_exact": average(baseline, ("plan_order_exact",)), + "binding_accuracy": average(baseline, ("evaluation", "metrics", "binding_accuracy")), + } + gates = { + "target_hit_at_k": with_arazzo["target_hit_at_k"] == 1.0, + "selected_target_exact": with_arazzo["selected_target_exact"] == 1.0, + "candidate_required_tool_recall": (with_arazzo["candidate_required_tool_recall"] == 1.0), + "plan_required_tool_recall": with_arazzo["plan_required_tool_recall"] == 1.0, + "execution_required_tool_recall": (with_arazzo["execution_required_tool_recall"] == 1.0), + "goal_completion": with_arazzo["goal_completion_rate"] == 1.0, + "plan_order_exact": with_arazzo["plan_order_exact"] == 1.0, + "execution_order_exact": with_arazzo["execution_order_exact"] == 1.0, + "binding_accuracy": with_arazzo["binding_accuracy"] == 1.0, + "positive_goal_lift": ( + with_arazzo["goal_completion_rate"] > without_arazzo["goal_completion_rate"] + ), + } + return { + "status": "pass" if all(gates.values()) else "fail", + "case_count": len(rows), + "with_arazzo": with_arazzo, + "without_arazzo": without_arazzo, + "goal_completion_lift": round( + with_arazzo["goal_completion_rate"] - without_arazzo["goal_completion_rate"], 6 + ), + "gates": gates, + } + + +def _latency_percentiles(items: list[dict[str, Any]], stage: str) -> dict[str, float]: + values = sorted(float(item.get("latency_ms", {}).get(stage) or 0) for item in items) + return { + "p50": round(_percentile(values, 0.50), 3), + "p95": round(_percentile(values, 0.95), 3), + } + + +def _percentile(values: list[float], quantile: float) -> float: + if not values: + return 0.0 + position = (len(values) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return values[lower] + return values[lower] + (values[upper] - values[lower]) * (position - lower) + + +def _require_value(args: dict[str, Any], field: str, expected: str) -> None: + if args.get(field) != expected: + raise ValueError(f"invalid or missing {field}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--catalog-size", type=int, default=1000) + parser.add_argument("--lengths", default="3,10,30") + parser.add_argument("--top-k", type=int, default=8) + parser.add_argument("--token-budget", type=int, default=2048) + parser.add_argument("--out", type=Path) + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + lengths = tuple(int(value) for value in args.lengths.split(",") if value.strip()) + report = run_benchmark( + catalog_size=args.catalog_size, + workflow_lengths=lengths, + top_k=args.top_k, + token_budget=args.token_budget, + ) + rendered = json.dumps(report, ensure_ascii=False, indent=2) + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(rendered + "\n", encoding="utf-8") + if args.json: + print(rendered) + else: + summary = report["summary"] + print( + f"{report['benchmark']}: {summary['status']} " + f"goal {summary['without_arazzo']['goal_completion_rate']:.0%} -> " + f"{summary['with_arazzo']['goal_completion_rate']:.0%}" + ) + for row in report["cases"]: + print( + f"- {row['family']} {row['workflow_length']} steps / {row['catalog_size']} tools: " + f"plan={row['with_arazzo']['plan_order_exact']:.0f}, " + f"binding={row['with_arazzo']['evaluation']['metrics']['binding_accuracy']:.0f}, " + f"goal={'pass' if row['with_arazzo']['evaluation']['goal_completed'] else 'fail'}" + ) + return 0 if report["summary"]["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/results/arazzo_long_horizon_0.42.json b/benchmarks/results/arazzo_long_horizon_0.42.json new file mode 100644 index 0000000..e7ebbc4 --- /dev/null +++ b/benchmarks/results/arazzo_long_horizon_0.42.json @@ -0,0 +1,3904 @@ +{ + "benchmark": "Arazzo Long-Horizon Paired Evaluation", + "methodology": "paired_deterministic_retrieve_plan_execute_goal_state", + "model": "none", + "graph_tool_call_version": "0.41.0", + "catalog_size": 1000, + "workflow_lengths": [ + 3, + 10, + 30 + ], + "top_k": 8, + "token_budget": 2048, + "summary": { + "status": "pass", + "case_count": 3, + "with_arazzo": { + "target_hit_at_k": 1.0, + "selected_target_exact": 1.0, + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "goal_completion_rate": 1.0, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "binding_accuracy": 1.0, + "token_budget_used": { + "average": 260.666667, + "max": 301 + }, + "latency_ms": { + "build": { + "p50": 455.804, + "p95": 481.3 + }, + "retrieve": { + "p50": 97.714, + "p95": 100.802 + }, + "plan_execute": { + "p50": 6.079, + "p95": 7.683 + } + } + }, + "without_arazzo": { + "goal_completion_rate": 0.0, + "plan_order_exact": 0.0, + "binding_accuracy": 0.0 + }, + "goal_completion_lift": 1.0, + "gates": { + "target_hit_at_k": true, + "selected_target_exact": true, + "candidate_required_tool_recall": true, + "plan_required_tool_recall": true, + "execution_required_tool_recall": true, + "goal_completion": true, + "plan_order_exact": true, + "execution_order_exact": true, + "binding_accuracy": true, + "positive_goal_lift": true + } + }, + "cases": [ + { + "id": "supplier_onboarding_3", + "family": "supplier_onboarding", + "workflow_length": 3, + "catalog_size": 1000, + "baseline": { + "use_arazzo": false, + "tool_count": 1000, + "edge_count": 0, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 998, + "produces_consumer_aligned": 0, + "consumes_added": 3, + "produces_skipped": 3, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 3, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 2, + "create": 1, + "search": 997 + }, + "resource_counts": { + "archive": 997, + "supplier_onboarding": 3 + }, + "module_counts": { + "archive_references": 997, + "supplier_onboarding_stages": 3 + }, + "result_shape_counts": { + "list": 997, + "mutation": 3 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 997, + "rate": 0.997 + }, + { + "module": "supplier_onboarding_stages", + "count": 3, + "rate": 0.003 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": {}, + "target_hit_at_k": 1.0, + "selected_target_exact": 1.0, + "selected_target": "issueSupplierActivationCertificate", + "retrieved_tools": [ + "issueSupplierActivationCertificate", + "resolveSupplierTenant", + "completeSupplierOnboardingStage02" + ], + "candidate_count": 1, + "planned_call_count": 1, + "executed_call_count": 1, + "plan_order_exact": 0.0, + "execution_order_exact": 0.0, + "planned_tools": [ + "issueSupplierActivationCertificate" + ], + "executed_tools": [ + "issueSupplierActivationCertificate" + ], + "runner_success": false, + "failure": {}, + "evaluation": { + "scenario_id": "supplier_onboarding_3", + "goal_completed": false, + "metrics": { + "candidate_required_tool_recall": 0.333333, + "plan_required_tool_recall": 0.333333, + "execution_required_tool_recall": 0.0, + "required_tool_recall": 0.0, + "milestone_completion": 0.0, + "dependency_order_accuracy": 0.0, + "binding_accuracy": 0.0, + "final_state_accuracy": 0.0, + "schema_valid_call_rate": null, + "extraneous_call_rate": 1.0, + "policy_violation_count": 0, + "call_count": 1, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 1, + "recovery_success": 0, + "goal_completion": 0.0 + }, + "matched_milestones": {}, + "checks": [ + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_01", + "expected": [ + "resolveSupplierTenant" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_02", + "expected": [ + "completeSupplierOnboardingStage02" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_03", + "expected": [ + "issueSupplierActivationCertificate" + ], + "observed": "" + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "", + "observed": "" + }, + { + "category": "state", + "code": "goal_state_mismatch", + "passed": false, + "subject": "output.activationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-supplier_onboarding" + }, + "observed": "" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 3, + "observed": 1 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "execution", + "code": "execution_failed", + "passed": false, + "subject": "runner", + "expected": "completed", + "observed": "failed" + } + ], + "failure_reason_codes": [ + "missing_milestone", + "invalid_dependency_order", + "binding_mismatch", + "goal_state_mismatch", + "execution_failed" + ] + }, + "latency_ms": { + "build": 465.411, + "retrieve": 102.285, + "plan_execute": 5.312 + }, + "token_budget_used": 73 + }, + "with_arazzo": { + "use_arazzo": true, + "tool_count": 1000, + "edge_count": 2, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 998, + "produces_consumer_aligned": 0, + "consumes_added": 3, + "produces_skipped": 3, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 3, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 2, + "create": 1, + "search": 997 + }, + "resource_counts": { + "archive": 997, + "supplier_onboarding": 3 + }, + "module_counts": { + "archive_references": 997, + "supplier_onboarding_stages": 3 + }, + "result_shape_counts": { + "list": 997, + "mutation": 3 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 997, + "rate": 0.997 + }, + { + "module": "supplier_onboarding_stages", + "count": 3, + "rate": 0.003 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 3, + "relation_count": 2, + "by_dependency_kind": { + "runtime_reference": 2 + }, + "edge_stats": { + "added": 2, + "merged": 0, + "binding_aliases_added": 2 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "08929f2c7d36ff7b0036617930b6636dcfda90ca514a2f1207564c01acfd317a", + "bytes": 793, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "selected_target_exact": 1.0, + "selected_target": "issueSupplierActivationCertificate", + "retrieved_tools": [ + "issueSupplierActivationCertificate", + "completeSupplierOnboardingStage02", + "resolveSupplierTenant" + ], + "candidate_count": 3, + "planned_call_count": 3, + "executed_call_count": 3, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "resolveSupplierTenant", + "completeSupplierOnboardingStage02", + "issueSupplierActivationCertificate" + ], + "executed_tools": [ + "resolveSupplierTenant", + "completeSupplierOnboardingStage02", + "issueSupplierActivationCertificate" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "supplier_onboarding_3", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 3, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "resolveSupplierTenant" + ], + "observed": "resolveSupplierTenant" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completeSupplierOnboardingStage02" + ], + "observed": "completeSupplierOnboardingStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "issueSupplierActivationCertificate" + ], + "observed": "issueSupplierActivationCertificate" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-supplier_onboarding-01", + "observed": "EVIDENCE-supplier_onboarding-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-supplier_onboarding-02", + "observed": "EVIDENCE-supplier_onboarding-02" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.activationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-supplier_onboarding" + }, + "observed": "CERT-supplier_onboarding" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 3, + "observed": 3 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 3, + "observed": 3 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "build": 455.804, + "retrieve": 97.714, + "plan_execute": 6.033 + }, + "token_budget_used": 190 + }, + "lift": { + "goal_completion": 1.0, + "candidate_required_tool_recall": 0.666667, + "plan_required_tool_recall": 0.666667, + "execution_required_tool_recall": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0 + } + }, + { + "id": "incident_recovery_10", + "family": "incident_recovery", + "workflow_length": 10, + "catalog_size": 1000, + "baseline": { + "use_arazzo": false, + "tool_count": 1000, + "edge_count": 0, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 991, + "produces_consumer_aligned": 0, + "consumes_added": 10, + "produces_skipped": 10, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 10, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 10, + "search": 990 + }, + "resource_counts": { + "archive": 990, + "incident_recovery": 10 + }, + "module_counts": { + "archive_references": 990, + "incident_recovery_stages": 10 + }, + "result_shape_counts": { + "list": 990, + "mutation": 10 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 990, + "rate": 0.99 + }, + { + "module": "incident_recovery_stages", + "count": 10, + "rate": 0.01 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": {}, + "target_hit_at_k": 1.0, + "selected_target_exact": 0.0, + "selected_target": "openIncidentInvestigation", + "retrieved_tools": [ + "openIncidentInvestigation", + "issueIncidentRecoveryReport", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04" + ], + "candidate_count": 1, + "planned_call_count": 1, + "executed_call_count": 1, + "plan_order_exact": 0.0, + "execution_order_exact": 0.0, + "planned_tools": [ + "openIncidentInvestigation" + ], + "executed_tools": [ + "openIncidentInvestigation" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "incident_recovery_10", + "goal_completed": false, + "metrics": { + "candidate_required_tool_recall": 0.1, + "plan_required_tool_recall": 0.1, + "execution_required_tool_recall": 0.1, + "required_tool_recall": 0.1, + "milestone_completion": 0.1, + "dependency_order_accuracy": 0.0, + "binding_accuracy": 0.0, + "final_state_accuracy": 0.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 1, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 0.0 + }, + "matched_milestones": { + "stage_01": 1 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openIncidentInvestigation" + ], + "observed": "openIncidentInvestigation" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_02", + "expected": [ + "completeIncidentRecoveryStage02" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_03", + "expected": [ + "completeIncidentRecoveryStage03" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_04", + "expected": [ + "completeIncidentRecoveryStage04" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_05", + "expected": [ + "completeIncidentRecoveryStage05" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_06", + "expected": [ + "completeIncidentRecoveryStage06" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_07", + "expected": [ + "completeIncidentRecoveryStage07" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_08", + "expected": [ + "completeIncidentRecoveryStage08" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_09", + "expected": [ + "completeIncidentRecoveryStage09" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_10", + "expected": [ + "issueIncidentRecoveryReport" + ], + "observed": "" + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-incident_recovery-01", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "", + "observed": "" + }, + { + "category": "state", + "code": "goal_state_mismatch", + "passed": false, + "subject": "output.recoveryReportId", + "expected": { + "operator": "eq", + "value": "CERT-incident_recovery" + }, + "observed": "" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 10, + "observed": 1 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 1, + "observed": 1 + } + ], + "failure_reason_codes": [ + "missing_milestone", + "invalid_dependency_order", + "binding_mismatch", + "goal_state_mismatch" + ] + }, + "latency_ms": { + "build": 475.985, + "retrieve": 97.232, + "plan_execute": 5.541 + }, + "token_budget_used": 118 + }, + "with_arazzo": { + "use_arazzo": true, + "tool_count": 1000, + "edge_count": 9, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 991, + "produces_consumer_aligned": 0, + "consumes_added": 10, + "produces_skipped": 10, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 10, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 10, + "search": 990 + }, + "resource_counts": { + "archive": 990, + "incident_recovery": 10 + }, + "module_counts": { + "archive_references": 990, + "incident_recovery_stages": 10 + }, + "result_shape_counts": { + "list": 990, + "mutation": 10 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 990, + "rate": 0.99 + }, + { + "module": "incident_recovery_stages", + "count": 10, + "rate": 0.01 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 10, + "relation_count": 9, + "by_dependency_kind": { + "runtime_reference": 9 + }, + "edge_stats": { + "added": 9, + "merged": 0, + "binding_aliases_added": 9 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "42cbf08002ff711f27fda523ebfe3974c14eaa8e1ab48f8a8857e3d0ca8b31fc", + "bytes": 2308, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "selected_target_exact": 1.0, + "selected_target": "issueIncidentRecoveryReport", + "retrieved_tools": [ + "issueIncidentRecoveryReport", + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04" + ], + "candidate_count": 10, + "planned_call_count": 10, + "executed_call_count": 10, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04", + "completeIncidentRecoveryStage05", + "completeIncidentRecoveryStage06", + "completeIncidentRecoveryStage07", + "completeIncidentRecoveryStage08", + "completeIncidentRecoveryStage09", + "issueIncidentRecoveryReport" + ], + "executed_tools": [ + "openIncidentInvestigation", + "completeIncidentRecoveryStage02", + "completeIncidentRecoveryStage03", + "completeIncidentRecoveryStage04", + "completeIncidentRecoveryStage05", + "completeIncidentRecoveryStage06", + "completeIncidentRecoveryStage07", + "completeIncidentRecoveryStage08", + "completeIncidentRecoveryStage09", + "issueIncidentRecoveryReport" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "incident_recovery_10", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 10, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3, + "stage_04": 4, + "stage_05": 5, + "stage_06": 6, + "stage_07": 7, + "stage_08": 8, + "stage_09": 9, + "stage_10": 10 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openIncidentInvestigation" + ], + "observed": "openIncidentInvestigation" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completeIncidentRecoveryStage02" + ], + "observed": "completeIncidentRecoveryStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "completeIncidentRecoveryStage03" + ], + "observed": "completeIncidentRecoveryStage03" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_04", + "expected": [ + "completeIncidentRecoveryStage04" + ], + "observed": "completeIncidentRecoveryStage04" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_05", + "expected": [ + "completeIncidentRecoveryStage05" + ], + "observed": "completeIncidentRecoveryStage05" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_06", + "expected": [ + "completeIncidentRecoveryStage06" + ], + "observed": "completeIncidentRecoveryStage06" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_07", + "expected": [ + "completeIncidentRecoveryStage07" + ], + "observed": "completeIncidentRecoveryStage07" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_08", + "expected": [ + "completeIncidentRecoveryStage08" + ], + "observed": "completeIncidentRecoveryStage08" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_09", + "expected": [ + "completeIncidentRecoveryStage09" + ], + "observed": "completeIncidentRecoveryStage09" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_10", + "expected": [ + "issueIncidentRecoveryReport" + ], + "observed": "issueIncidentRecoveryReport" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": 3, + "after_sequence": 4 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": 4, + "after_sequence": 5 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": 5, + "after_sequence": 6 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": 6, + "after_sequence": 7 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": 7, + "after_sequence": 8 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": 8, + "after_sequence": 9 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": 9, + "after_sequence": 10 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-incident_recovery-01", + "observed": "EVIDENCE-incident_recovery-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-incident_recovery-02", + "observed": "EVIDENCE-incident_recovery-02" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "EVIDENCE-incident_recovery-03", + "observed": "EVIDENCE-incident_recovery-03" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "EVIDENCE-incident_recovery-04", + "observed": "EVIDENCE-incident_recovery-04" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "EVIDENCE-incident_recovery-05", + "observed": "EVIDENCE-incident_recovery-05" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "EVIDENCE-incident_recovery-06", + "observed": "EVIDENCE-incident_recovery-06" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "EVIDENCE-incident_recovery-07", + "observed": "EVIDENCE-incident_recovery-07" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "EVIDENCE-incident_recovery-08", + "observed": "EVIDENCE-incident_recovery-08" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "EVIDENCE-incident_recovery-09", + "observed": "EVIDENCE-incident_recovery-09" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.recoveryReportId", + "expected": { + "operator": "eq", + "value": "CERT-incident_recovery" + }, + "observed": "CERT-incident_recovery" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 10, + "observed": 10 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 10, + "observed": 10 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "build": 452.269, + "retrieve": 95.358, + "plan_execute": 6.079 + }, + "token_budget_used": 291 + }, + "lift": { + "goal_completion": 1.0, + "candidate_required_tool_recall": 0.9, + "plan_required_tool_recall": 0.9, + "execution_required_tool_recall": 0.9, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0 + } + }, + { + "id": "publication_approval_30", + "family": "publication_approval", + "workflow_length": 30, + "catalog_size": 1000, + "baseline": { + "use_arazzo": false, + "tool_count": 1000, + "edge_count": 0, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 971, + "produces_consumer_aligned": 0, + "consumes_added": 30, + "produces_skipped": 30, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 30, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 30, + "search": 970 + }, + "resource_counts": { + "archive": 970, + "publication_approval": 30 + }, + "module_counts": { + "archive_references": 970, + "publication_approval_stages": 30 + }, + "result_shape_counts": { + "list": 970, + "mutation": 30 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 970, + "rate": 0.97 + }, + { + "module": "publication_approval_stages", + "count": 30, + "rate": 0.03 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": {}, + "target_hit_at_k": 1.0, + "selected_target_exact": 0.0, + "selected_target": "openPublicationReview", + "retrieved_tools": [ + "openPublicationReview", + "issuePublicationApprovalCertificate", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04" + ], + "candidate_count": 1, + "planned_call_count": 1, + "executed_call_count": 1, + "plan_order_exact": 0.0, + "execution_order_exact": 0.0, + "planned_tools": [ + "openPublicationReview" + ], + "executed_tools": [ + "openPublicationReview" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "publication_approval_30", + "goal_completed": false, + "metrics": { + "candidate_required_tool_recall": 0.033333, + "plan_required_tool_recall": 0.033333, + "execution_required_tool_recall": 0.033333, + "required_tool_recall": 0.033333, + "milestone_completion": 0.033333, + "dependency_order_accuracy": 0.0, + "binding_accuracy": 0.0, + "final_state_accuracy": 0.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 1, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 0.0 + }, + "matched_milestones": { + "stage_01": 1 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openPublicationReview" + ], + "observed": "openPublicationReview" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_02", + "expected": [ + "completePublicationReviewStage02" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_03", + "expected": [ + "completePublicationReviewStage03" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_04", + "expected": [ + "completePublicationReviewStage04" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_05", + "expected": [ + "completePublicationReviewStage05" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_06", + "expected": [ + "completePublicationReviewStage06" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_07", + "expected": [ + "completePublicationReviewStage07" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_08", + "expected": [ + "completePublicationReviewStage08" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_09", + "expected": [ + "completePublicationReviewStage09" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_10", + "expected": [ + "completePublicationReviewStage10" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_11", + "expected": [ + "completePublicationReviewStage11" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_12", + "expected": [ + "completePublicationReviewStage12" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_13", + "expected": [ + "completePublicationReviewStage13" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_14", + "expected": [ + "completePublicationReviewStage14" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_15", + "expected": [ + "completePublicationReviewStage15" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_16", + "expected": [ + "completePublicationReviewStage16" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_17", + "expected": [ + "completePublicationReviewStage17" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_18", + "expected": [ + "completePublicationReviewStage18" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_19", + "expected": [ + "completePublicationReviewStage19" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_20", + "expected": [ + "completePublicationReviewStage20" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_21", + "expected": [ + "completePublicationReviewStage21" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_22", + "expected": [ + "completePublicationReviewStage22" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_23", + "expected": [ + "completePublicationReviewStage23" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_24", + "expected": [ + "completePublicationReviewStage24" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_25", + "expected": [ + "completePublicationReviewStage25" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_26", + "expected": [ + "completePublicationReviewStage26" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_27", + "expected": [ + "completePublicationReviewStage27" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_28", + "expected": [ + "completePublicationReviewStage28" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_29", + "expected": [ + "completePublicationReviewStage29" + ], + "observed": "" + }, + { + "category": "milestone", + "code": "missing_milestone", + "passed": false, + "subject": "stage_30", + "expected": [ + "issuePublicationApprovalCertificate" + ], + "observed": "" + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_10->stage_11", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_11->stage_12", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_12->stage_13", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_13->stage_14", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_14->stage_15", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_15->stage_16", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_16->stage_17", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_17->stage_18", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_18->stage_19", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_19->stage_20", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_20->stage_21", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_21->stage_22", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_22->stage_23", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_23->stage_24", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_24->stage_25", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_25->stage_26", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_26->stage_27", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_27->stage_28", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_28->stage_29", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "dependency", + "code": "invalid_dependency_order", + "passed": false, + "subject": "stage_29->stage_30", + "expected": "before", + "observed": { + "before_sequence": null, + "after_sequence": null + } + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-publication_approval-01", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_10.stageEvidence10->stage_11.handoffToken10", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_11.stageEvidence11->stage_12.handoffToken11", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_12.stageEvidence12->stage_13.handoffToken12", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_13.stageEvidence13->stage_14.handoffToken13", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_14.stageEvidence14->stage_15.handoffToken14", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_15.stageEvidence15->stage_16.handoffToken15", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_16.stageEvidence16->stage_17.handoffToken16", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_17.stageEvidence17->stage_18.handoffToken17", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_18.stageEvidence18->stage_19.handoffToken18", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_19.stageEvidence19->stage_20.handoffToken19", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_20.stageEvidence20->stage_21.handoffToken20", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_21.stageEvidence21->stage_22.handoffToken21", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_22.stageEvidence22->stage_23.handoffToken22", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_23.stageEvidence23->stage_24.handoffToken23", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_24.stageEvidence24->stage_25.handoffToken24", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_25.stageEvidence25->stage_26.handoffToken25", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_26.stageEvidence26->stage_27.handoffToken26", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_27.stageEvidence27->stage_28.handoffToken27", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_28.stageEvidence28->stage_29.handoffToken28", + "expected": "", + "observed": "" + }, + { + "category": "binding", + "code": "binding_mismatch", + "passed": false, + "subject": "stage_29.stageEvidence29->stage_30.handoffToken29", + "expected": "", + "observed": "" + }, + { + "category": "state", + "code": "goal_state_mismatch", + "passed": false, + "subject": "output.publicationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-publication_approval" + }, + "observed": "" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 30, + "observed": 1 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 1, + "observed": 1 + } + ], + "failure_reason_codes": [ + "missing_milestone", + "invalid_dependency_order", + "binding_mismatch", + "goal_state_mismatch" + ] + }, + "latency_ms": { + "build": 458.637, + "retrieve": 98.179, + "plan_execute": 6.067 + }, + "token_budget_used": 125 + }, + "with_arazzo": { + "use_arazzo": true, + "tool_count": 1000, + "edge_count": 29, + "edge_stats": { + "EXTRACTED": 0, + "INFERRED": 0, + "AMBIGUOUS": 0, + "dropped": 0, + "by_relation": { + "requires": 0 + }, + "cross_source": 0, + "tool_count": 1000, + "edge_count": 0, + "refs_preserved": 0, + "contract_signals": { + "tools_promoted": 1000, + "produces_added": 971, + "produces_consumer_aligned": 0, + "consumes_added": 30, + "produces_skipped": 30, + "produces_skipped_path_cap": 0, + "consumes_skipped": 0, + "required_consumer_demand_keys": 0 + }, + "contract_edges": { + "added": 0, + "merged": 0, + "skipped_self": 0, + "skipped_no_producer": 30, + "skipped_echo": 0, + "skipped_consumer_scope": 0 + }, + "openapi_link_signals": { + "links_seen": 0, + "produces_added": 0, + "skipped_no_response_source": 0 + }, + "openapi_link_edges": { + "added": 0, + "merged": 0, + "skipped_unresolved": 0, + "skipped_self": 0, + "by_relation": {} + }, + "semantic_metadata": { + "tool_count": 1000, + "canonical_action_known_count": 1000, + "canonical_action_known_rate": 1.0, + "primary_resource_assigned_count": 1000, + "primary_resource_assigned_rate": 1.0, + "path_module_assigned_count": 1000, + "path_module_assigned_rate": 1.0, + "action_counts": { + "action": 30, + "search": 970 + }, + "resource_counts": { + "archive": 970, + "publication_approval": 30 + }, + "module_counts": { + "archive_references": 970, + "publication_approval_stages": 30 + }, + "result_shape_counts": { + "list": 970, + "mutation": 30 + }, + "top_modules": [ + { + "module": "archive_references", + "count": 970, + "rate": 0.97 + }, + { + "module": "publication_approval_stages", + "count": 30, + "rate": 0.03 + } + ], + "semantic_confidence_counts": { + "high": 1000 + }, + "unknown_samples": [] + }, + "relation_budget": 100000, + "relation_budget_reached": false, + "pair_edges": { + "manual": 0, + "auto": 0, + "skipped_target_missing": 0, + "skipped_self": 0, + "skipped_existing_structural": 0 + } + }, + "workflow_summary": { + "source_count": 1, + "workflow_count": 1, + "step_count": 30, + "relation_count": 29, + "by_dependency_kind": { + "runtime_reference": 29 + }, + "edge_stats": { + "added": 29, + "merged": 0, + "binding_aliases_added": 29 + }, + "source_snapshot_manifest": { + "spec_count": 1, + "specs": [ + { + "index": 1, + "source": "inline:1", + "sha256": "21a3ad4765fcfda8aea532b05c84a753829039e9d45653270a918c2deda939e1", + "bytes": 6709, + "arazzo_version": "1.1.0", + "workflow_count": 1 + } + ] + } + }, + "target_hit_at_k": 1.0, + "selected_target_exact": 1.0, + "selected_target": "issuePublicationApprovalCertificate", + "retrieved_tools": [ + "issuePublicationApprovalCertificate", + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04" + ], + "candidate_count": 30, + "planned_call_count": 30, + "executed_call_count": 30, + "plan_order_exact": 1.0, + "execution_order_exact": 1.0, + "planned_tools": [ + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04", + "completePublicationReviewStage05", + "completePublicationReviewStage06", + "completePublicationReviewStage07", + "completePublicationReviewStage08", + "completePublicationReviewStage09", + "completePublicationReviewStage10", + "completePublicationReviewStage11", + "completePublicationReviewStage12", + "completePublicationReviewStage13", + "completePublicationReviewStage14", + "completePublicationReviewStage15", + "completePublicationReviewStage16", + "completePublicationReviewStage17", + "completePublicationReviewStage18", + "completePublicationReviewStage19", + "completePublicationReviewStage20", + "completePublicationReviewStage21", + "completePublicationReviewStage22", + "completePublicationReviewStage23", + "completePublicationReviewStage24", + "completePublicationReviewStage25", + "completePublicationReviewStage26", + "completePublicationReviewStage27", + "completePublicationReviewStage28", + "completePublicationReviewStage29", + "issuePublicationApprovalCertificate" + ], + "executed_tools": [ + "openPublicationReview", + "completePublicationReviewStage02", + "completePublicationReviewStage03", + "completePublicationReviewStage04", + "completePublicationReviewStage05", + "completePublicationReviewStage06", + "completePublicationReviewStage07", + "completePublicationReviewStage08", + "completePublicationReviewStage09", + "completePublicationReviewStage10", + "completePublicationReviewStage11", + "completePublicationReviewStage12", + "completePublicationReviewStage13", + "completePublicationReviewStage14", + "completePublicationReviewStage15", + "completePublicationReviewStage16", + "completePublicationReviewStage17", + "completePublicationReviewStage18", + "completePublicationReviewStage19", + "completePublicationReviewStage20", + "completePublicationReviewStage21", + "completePublicationReviewStage22", + "completePublicationReviewStage23", + "completePublicationReviewStage24", + "completePublicationReviewStage25", + "completePublicationReviewStage26", + "completePublicationReviewStage27", + "completePublicationReviewStage28", + "completePublicationReviewStage29", + "issuePublicationApprovalCertificate" + ], + "runner_success": true, + "failure": {}, + "evaluation": { + "scenario_id": "publication_approval_30", + "goal_completed": true, + "metrics": { + "candidate_required_tool_recall": 1.0, + "plan_required_tool_recall": 1.0, + "execution_required_tool_recall": 1.0, + "required_tool_recall": 1.0, + "milestone_completion": 1.0, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0, + "final_state_accuracy": 1.0, + "schema_valid_call_rate": 1.0, + "extraneous_call_rate": 0.0, + "policy_violation_count": 0, + "call_count": 30, + "replan_count": 0, + "latency_ms": 0, + "recovery_attempted": 0, + "recovery_success": null, + "goal_completion": 1.0 + }, + "matched_milestones": { + "stage_01": 1, + "stage_02": 2, + "stage_03": 3, + "stage_04": 4, + "stage_05": 5, + "stage_06": 6, + "stage_07": 7, + "stage_08": 8, + "stage_09": 9, + "stage_10": 10, + "stage_11": 11, + "stage_12": 12, + "stage_13": 13, + "stage_14": 14, + "stage_15": 15, + "stage_16": 16, + "stage_17": 17, + "stage_18": 18, + "stage_19": 19, + "stage_20": 20, + "stage_21": 21, + "stage_22": 22, + "stage_23": 23, + "stage_24": 24, + "stage_25": 25, + "stage_26": 26, + "stage_27": 27, + "stage_28": 28, + "stage_29": 29, + "stage_30": 30 + }, + "checks": [ + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_01", + "expected": [ + "openPublicationReview" + ], + "observed": "openPublicationReview" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_02", + "expected": [ + "completePublicationReviewStage02" + ], + "observed": "completePublicationReviewStage02" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_03", + "expected": [ + "completePublicationReviewStage03" + ], + "observed": "completePublicationReviewStage03" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_04", + "expected": [ + "completePublicationReviewStage04" + ], + "observed": "completePublicationReviewStage04" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_05", + "expected": [ + "completePublicationReviewStage05" + ], + "observed": "completePublicationReviewStage05" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_06", + "expected": [ + "completePublicationReviewStage06" + ], + "observed": "completePublicationReviewStage06" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_07", + "expected": [ + "completePublicationReviewStage07" + ], + "observed": "completePublicationReviewStage07" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_08", + "expected": [ + "completePublicationReviewStage08" + ], + "observed": "completePublicationReviewStage08" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_09", + "expected": [ + "completePublicationReviewStage09" + ], + "observed": "completePublicationReviewStage09" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_10", + "expected": [ + "completePublicationReviewStage10" + ], + "observed": "completePublicationReviewStage10" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_11", + "expected": [ + "completePublicationReviewStage11" + ], + "observed": "completePublicationReviewStage11" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_12", + "expected": [ + "completePublicationReviewStage12" + ], + "observed": "completePublicationReviewStage12" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_13", + "expected": [ + "completePublicationReviewStage13" + ], + "observed": "completePublicationReviewStage13" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_14", + "expected": [ + "completePublicationReviewStage14" + ], + "observed": "completePublicationReviewStage14" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_15", + "expected": [ + "completePublicationReviewStage15" + ], + "observed": "completePublicationReviewStage15" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_16", + "expected": [ + "completePublicationReviewStage16" + ], + "observed": "completePublicationReviewStage16" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_17", + "expected": [ + "completePublicationReviewStage17" + ], + "observed": "completePublicationReviewStage17" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_18", + "expected": [ + "completePublicationReviewStage18" + ], + "observed": "completePublicationReviewStage18" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_19", + "expected": [ + "completePublicationReviewStage19" + ], + "observed": "completePublicationReviewStage19" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_20", + "expected": [ + "completePublicationReviewStage20" + ], + "observed": "completePublicationReviewStage20" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_21", + "expected": [ + "completePublicationReviewStage21" + ], + "observed": "completePublicationReviewStage21" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_22", + "expected": [ + "completePublicationReviewStage22" + ], + "observed": "completePublicationReviewStage22" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_23", + "expected": [ + "completePublicationReviewStage23" + ], + "observed": "completePublicationReviewStage23" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_24", + "expected": [ + "completePublicationReviewStage24" + ], + "observed": "completePublicationReviewStage24" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_25", + "expected": [ + "completePublicationReviewStage25" + ], + "observed": "completePublicationReviewStage25" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_26", + "expected": [ + "completePublicationReviewStage26" + ], + "observed": "completePublicationReviewStage26" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_27", + "expected": [ + "completePublicationReviewStage27" + ], + "observed": "completePublicationReviewStage27" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_28", + "expected": [ + "completePublicationReviewStage28" + ], + "observed": "completePublicationReviewStage28" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_29", + "expected": [ + "completePublicationReviewStage29" + ], + "observed": "completePublicationReviewStage29" + }, + { + "category": "milestone", + "code": "milestone_completed", + "passed": true, + "subject": "stage_30", + "expected": [ + "issuePublicationApprovalCertificate" + ], + "observed": "issuePublicationApprovalCertificate" + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_01->stage_02", + "expected": "before", + "observed": { + "before_sequence": 1, + "after_sequence": 2 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_02->stage_03", + "expected": "before", + "observed": { + "before_sequence": 2, + "after_sequence": 3 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_03->stage_04", + "expected": "before", + "observed": { + "before_sequence": 3, + "after_sequence": 4 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_04->stage_05", + "expected": "before", + "observed": { + "before_sequence": 4, + "after_sequence": 5 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_05->stage_06", + "expected": "before", + "observed": { + "before_sequence": 5, + "after_sequence": 6 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_06->stage_07", + "expected": "before", + "observed": { + "before_sequence": 6, + "after_sequence": 7 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_07->stage_08", + "expected": "before", + "observed": { + "before_sequence": 7, + "after_sequence": 8 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_08->stage_09", + "expected": "before", + "observed": { + "before_sequence": 8, + "after_sequence": 9 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_09->stage_10", + "expected": "before", + "observed": { + "before_sequence": 9, + "after_sequence": 10 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_10->stage_11", + "expected": "before", + "observed": { + "before_sequence": 10, + "after_sequence": 11 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_11->stage_12", + "expected": "before", + "observed": { + "before_sequence": 11, + "after_sequence": 12 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_12->stage_13", + "expected": "before", + "observed": { + "before_sequence": 12, + "after_sequence": 13 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_13->stage_14", + "expected": "before", + "observed": { + "before_sequence": 13, + "after_sequence": 14 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_14->stage_15", + "expected": "before", + "observed": { + "before_sequence": 14, + "after_sequence": 15 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_15->stage_16", + "expected": "before", + "observed": { + "before_sequence": 15, + "after_sequence": 16 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_16->stage_17", + "expected": "before", + "observed": { + "before_sequence": 16, + "after_sequence": 17 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_17->stage_18", + "expected": "before", + "observed": { + "before_sequence": 17, + "after_sequence": 18 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_18->stage_19", + "expected": "before", + "observed": { + "before_sequence": 18, + "after_sequence": 19 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_19->stage_20", + "expected": "before", + "observed": { + "before_sequence": 19, + "after_sequence": 20 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_20->stage_21", + "expected": "before", + "observed": { + "before_sequence": 20, + "after_sequence": 21 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_21->stage_22", + "expected": "before", + "observed": { + "before_sequence": 21, + "after_sequence": 22 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_22->stage_23", + "expected": "before", + "observed": { + "before_sequence": 22, + "after_sequence": 23 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_23->stage_24", + "expected": "before", + "observed": { + "before_sequence": 23, + "after_sequence": 24 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_24->stage_25", + "expected": "before", + "observed": { + "before_sequence": 24, + "after_sequence": 25 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_25->stage_26", + "expected": "before", + "observed": { + "before_sequence": 25, + "after_sequence": 26 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_26->stage_27", + "expected": "before", + "observed": { + "before_sequence": 26, + "after_sequence": 27 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_27->stage_28", + "expected": "before", + "observed": { + "before_sequence": 27, + "after_sequence": 28 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_28->stage_29", + "expected": "before", + "observed": { + "before_sequence": 28, + "after_sequence": 29 + } + }, + { + "category": "dependency", + "code": "dependency_order_valid", + "passed": true, + "subject": "stage_29->stage_30", + "expected": "before", + "observed": { + "before_sequence": 29, + "after_sequence": 30 + } + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_01.stageEvidence01->stage_02.handoffToken01", + "expected": "EVIDENCE-publication_approval-01", + "observed": "EVIDENCE-publication_approval-01" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_02.stageEvidence02->stage_03.handoffToken02", + "expected": "EVIDENCE-publication_approval-02", + "observed": "EVIDENCE-publication_approval-02" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_03.stageEvidence03->stage_04.handoffToken03", + "expected": "EVIDENCE-publication_approval-03", + "observed": "EVIDENCE-publication_approval-03" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_04.stageEvidence04->stage_05.handoffToken04", + "expected": "EVIDENCE-publication_approval-04", + "observed": "EVIDENCE-publication_approval-04" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_05.stageEvidence05->stage_06.handoffToken05", + "expected": "EVIDENCE-publication_approval-05", + "observed": "EVIDENCE-publication_approval-05" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_06.stageEvidence06->stage_07.handoffToken06", + "expected": "EVIDENCE-publication_approval-06", + "observed": "EVIDENCE-publication_approval-06" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_07.stageEvidence07->stage_08.handoffToken07", + "expected": "EVIDENCE-publication_approval-07", + "observed": "EVIDENCE-publication_approval-07" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_08.stageEvidence08->stage_09.handoffToken08", + "expected": "EVIDENCE-publication_approval-08", + "observed": "EVIDENCE-publication_approval-08" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_09.stageEvidence09->stage_10.handoffToken09", + "expected": "EVIDENCE-publication_approval-09", + "observed": "EVIDENCE-publication_approval-09" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_10.stageEvidence10->stage_11.handoffToken10", + "expected": "EVIDENCE-publication_approval-10", + "observed": "EVIDENCE-publication_approval-10" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_11.stageEvidence11->stage_12.handoffToken11", + "expected": "EVIDENCE-publication_approval-11", + "observed": "EVIDENCE-publication_approval-11" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_12.stageEvidence12->stage_13.handoffToken12", + "expected": "EVIDENCE-publication_approval-12", + "observed": "EVIDENCE-publication_approval-12" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_13.stageEvidence13->stage_14.handoffToken13", + "expected": "EVIDENCE-publication_approval-13", + "observed": "EVIDENCE-publication_approval-13" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_14.stageEvidence14->stage_15.handoffToken14", + "expected": "EVIDENCE-publication_approval-14", + "observed": "EVIDENCE-publication_approval-14" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_15.stageEvidence15->stage_16.handoffToken15", + "expected": "EVIDENCE-publication_approval-15", + "observed": "EVIDENCE-publication_approval-15" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_16.stageEvidence16->stage_17.handoffToken16", + "expected": "EVIDENCE-publication_approval-16", + "observed": "EVIDENCE-publication_approval-16" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_17.stageEvidence17->stage_18.handoffToken17", + "expected": "EVIDENCE-publication_approval-17", + "observed": "EVIDENCE-publication_approval-17" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_18.stageEvidence18->stage_19.handoffToken18", + "expected": "EVIDENCE-publication_approval-18", + "observed": "EVIDENCE-publication_approval-18" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_19.stageEvidence19->stage_20.handoffToken19", + "expected": "EVIDENCE-publication_approval-19", + "observed": "EVIDENCE-publication_approval-19" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_20.stageEvidence20->stage_21.handoffToken20", + "expected": "EVIDENCE-publication_approval-20", + "observed": "EVIDENCE-publication_approval-20" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_21.stageEvidence21->stage_22.handoffToken21", + "expected": "EVIDENCE-publication_approval-21", + "observed": "EVIDENCE-publication_approval-21" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_22.stageEvidence22->stage_23.handoffToken22", + "expected": "EVIDENCE-publication_approval-22", + "observed": "EVIDENCE-publication_approval-22" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_23.stageEvidence23->stage_24.handoffToken23", + "expected": "EVIDENCE-publication_approval-23", + "observed": "EVIDENCE-publication_approval-23" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_24.stageEvidence24->stage_25.handoffToken24", + "expected": "EVIDENCE-publication_approval-24", + "observed": "EVIDENCE-publication_approval-24" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_25.stageEvidence25->stage_26.handoffToken25", + "expected": "EVIDENCE-publication_approval-25", + "observed": "EVIDENCE-publication_approval-25" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_26.stageEvidence26->stage_27.handoffToken26", + "expected": "EVIDENCE-publication_approval-26", + "observed": "EVIDENCE-publication_approval-26" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_27.stageEvidence27->stage_28.handoffToken27", + "expected": "EVIDENCE-publication_approval-27", + "observed": "EVIDENCE-publication_approval-27" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_28.stageEvidence28->stage_29.handoffToken28", + "expected": "EVIDENCE-publication_approval-28", + "observed": "EVIDENCE-publication_approval-28" + }, + { + "category": "binding", + "code": "binding_valid", + "passed": true, + "subject": "stage_29.stageEvidence29->stage_30.handoffToken29", + "expected": "EVIDENCE-publication_approval-29", + "observed": "EVIDENCE-publication_approval-29" + }, + { + "category": "state", + "code": "state_assertion_valid", + "passed": true, + "subject": "output.publicationCertificateId", + "expected": { + "operator": "eq", + "value": "CERT-publication_approval" + }, + "observed": "CERT-publication_approval" + }, + { + "category": "budget", + "code": "call_budget_valid", + "passed": true, + "subject": "max_calls", + "expected": 30, + "observed": 30 + }, + { + "category": "budget", + "code": "replan_budget_valid", + "passed": true, + "subject": "max_replans", + "expected": 0, + "observed": 0 + }, + { + "category": "budget", + "code": "latency_budget_valid", + "passed": true, + "subject": "timeout_sec", + "expected": 30.0, + "observed": 0.0 + }, + { + "category": "schema", + "code": "schema_valid", + "passed": true, + "subject": "tool_calls", + "expected": 30, + "observed": 30 + } + ], + "failure_reason_codes": [] + }, + "latency_ms": { + "build": 484.133, + "retrieve": 101.145, + "plan_execute": 7.861 + }, + "token_budget_used": 301 + }, + "lift": { + "goal_completion": 1.0, + "candidate_required_tool_recall": 0.966667, + "plan_required_tool_recall": 0.966667, + "execution_required_tool_recall": 0.966667, + "dependency_order_accuracy": 1.0, + "binding_accuracy": 1.0 + } + } + ], + "limitations": [ + "The benchmark isolates engine behavior and does not test independent LLM reasoning.", + "Generated catalogs are contract-distinct synthetic fixtures, not production APIs.", + "Arazzo evidence is supplied explicitly; workflow discovery without Arazzo is the baseline." + ] +} diff --git a/docs/benchmarks.md b/docs/benchmarks.md index f9e6ccb..9fe8cd4 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1156,3 +1156,23 @@ This first fixture validates the harness, not long-horizon or model quality. The next gates add 5-8, 9-15, and 16-30 step scenarios, repeated model runs, failure recovery, and XGEN dev API assertions. See [`docs/research/long-horizon-goal-evaluation.md`](research/long-horizon-goal-evaluation.md). + +### Arazzo 3/10/30-step paired evaluation + +The deterministic Arazzo gate compares the same 1,000-tool catalog before and +after explicit workflow evidence is applied. It uses three distinct workflow +families and requires exact response-to-request value handoffs across 3, 10, +and 30 calls. + +```bash +make arazzo-long-horizon-benchmark +``` + +On the frozen artifact, OpenAPI-only planning reaches `0.0` goal completion +because the opaque handoff fields cannot be justified. Adding Arazzo reaches +`1.0` target hit@8, target selection, exact plan order, exact execution order, +binding accuracy, and final goal completion. The result is recorded in +[`benchmarks/results/arazzo_long_horizon_0.42.json`](../benchmarks/results/arazzo_long_horizon_0.42.json). + +This benchmark measures graph-tool-call's deterministic middleware. It does +not claim that a model independently discovered the 30-step workflow. diff --git a/docs/research/long-horizon-goal-evaluation.md b/docs/research/long-horizon-goal-evaluation.md index 11bc609..10b58f0 100644 --- a/docs/research/long-horizon-goal-evaluation.md +++ b/docs/research/long-horizon-goal-evaluation.md @@ -81,3 +81,42 @@ After the L1 selector gate passes, work proceeds in this order: README or paper claims may use only versioned scenario files and saved reports that can be reproduced by the documented command. + +## Arazzo paired long-horizon gate + +The Arazzo gate isolates one causal question: does an explicit workflow +description improve execution when OpenAPI request and response field names do +not reveal the runtime handoff by themselves? + +Three generated, contract-distinct domains use separate Korean requests and +English operation IDs. Each catalog contains 1,000 tools. The required horizons +are 3, 10, and 30 calls; all other tools are read-only distractors. Gold +milestones, order constraints, bindings, and final-state assertions are read +only after execution. + +```bash +make arazzo-long-horizon-benchmark +``` + +The paired 0.41 engine result is: + +| Metric | OpenAPI only | OpenAPI + Arazzo | +|---|---:|---:| +| Target hit@8 | `1.0` | `1.0` | +| Final target exact | `1.0` | `1.0` | +| Goal completion | `0.0` | `1.0` | +| Exact plan order | `0.0` | `1.0` | +| Exact execution order | `0.0` | `1.0` | +| Runtime binding accuracy | `0.0` | `1.0` | + +The Arazzo condition executes exactly 3, 10, and 30 calls. It extracts 2, 9, +and 29 runtime-reference relations respectively, then promotes only the +declared response-path to request-field aliases. The saved case-level artifact +is +[`arazzo_long_horizon_0.42.json`](../../benchmarks/results/arazzo_long_horizon_0.42.json). + +This is deterministic engine evidence, not an LLM reasoning score and not a +claim about arbitrary production workflows. The next release-candidate gate +runs model target selection above the same sealed catalogs, followed by a +read-only XGEN dev replay. Production API names and payloads remain outside the +public artifact. diff --git a/tests/test_arazzo_long_horizon_benchmark.py b/tests/test_arazzo_long_horizon_benchmark.py new file mode 100644 index 0000000..952f2e9 --- /dev/null +++ b/tests/test_arazzo_long_horizon_benchmark.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from benchmarks.arazzo_long_horizon.run import run_benchmark + + +def test_arazzo_completes_three_step_goal_that_plain_openapi_cannot_bind(): + report = run_benchmark(catalog_size=40, workflow_lengths=(3,)) + row = report["cases"][0] + + assert row["baseline"]["evaluation"]["goal_completed"] is False + assert row["with_arazzo"]["evaluation"]["goal_completed"] is True + assert row["with_arazzo"]["plan_order_exact"] == 1.0 + assert row["with_arazzo"]["execution_order_exact"] == 1.0 + assert row["with_arazzo"]["evaluation"]["metrics"]["binding_accuracy"] == 1.0 + assert row["with_arazzo"]["workflow_summary"]["relation_count"] == 2 + + +def test_arazzo_horizon_gate_covers_3_10_30_steps_in_1000_tool_catalogs(): + report = run_benchmark(catalog_size=1000) + summary = report["summary"] + + assert report["workflow_lengths"] == [3, 10, 30] + assert summary["status"] == "pass" + assert summary["with_arazzo"]["target_hit_at_k"] == 1.0 + assert summary["with_arazzo"]["selected_target_exact"] == 1.0 + assert summary["with_arazzo"]["candidate_required_tool_recall"] == 1.0 + assert summary["with_arazzo"]["plan_required_tool_recall"] == 1.0 + assert summary["with_arazzo"]["execution_required_tool_recall"] == 1.0 + assert summary["with_arazzo"]["goal_completion_rate"] == 1.0 + assert summary["with_arazzo"]["plan_order_exact"] == 1.0 + assert summary["with_arazzo"]["execution_order_exact"] == 1.0 + assert summary["with_arazzo"]["binding_accuracy"] == 1.0 + assert summary["goal_completion_lift"] > 0 + assert summary["with_arazzo"]["latency_ms"]["retrieve"]["p95"] > 0 + assert summary["with_arazzo"]["token_budget_used"]["average"] > 0 + assert summary["with_arazzo"]["token_budget_used"]["max"] <= report["token_budget"] + assert [row["with_arazzo"]["executed_call_count"] for row in report["cases"]] == [3, 10, 30]