From 2bdececb542347a9247ca6740d345871c0659767 Mon Sep 17 00:00:00 2001 From: SonAIengine Date: Sun, 16 Aug 2026 20:34:19 +0900 Subject: [PATCH] Prepare graph-tool-call 0.42.0 release --- CHANGELOG.md | 23 +- README.md | 4 +- .../results/arazzo_long_horizon_0.42.json | 50 +-- benchmarks/results/releases/v0.42.0/README.md | 29 ++ .../v0.42.0/dependency-chain-evidence.json | 328 ++++++++++++++++++ .../v0.42.0/observability-evidence.json | 51 +++ docs/benchmarks.md | 6 +- graph_tool_call/__init__.py | 2 +- pyproject.toml | 2 +- 9 files changed, 462 insertions(+), 33 deletions(-) create mode 100644 benchmarks/results/releases/v0.42.0/README.md create mode 100644 benchmarks/results/releases/v0.42.0/dependency-chain-evidence.json create mode 100644 benchmarks/results/releases/v0.42.0/observability-evidence.json diff --git a/CHANGELOG.md b/CHANGELOG.md index cfee3d4..a1020aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.42.0] - 2026-08-16 + +### Added +- Public goal-completion evaluation contracts for multi-tool executions, + including milestone, dependency-order, binding, final-state, policy, and + recovery checks. +- Reproducible Arazzo long-horizon evaluation across 1,000-tool catalogs and + 3-, 10-, and 30-call workflows, with paired OpenAPI-only and + OpenAPI-plus-Arazzo evidence. + +### Changed +- OpenAPI ingestion now resolves component references per path with shared + reference and schema-field caches, avoiding whole-document expansion for + large specifications. + +### Fixed +- OpenAPI ingestion preserves literal `$ref` properties, decodes escaped JSON + Pointer segments, and emits serializable stubs for circular references. + ## [0.41.0] - 2026-08-16 ### Added @@ -682,7 +701,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Tests**: 32 tests passing across all modules - **Example**: `quickstart.py` demonstrating full workflow -[Unreleased]: https://github.com/SonAIengine/graph-tool-call/compare/v0.40.0...HEAD +[Unreleased]: https://github.com/SonAIengine/graph-tool-call/compare/v0.42.0...HEAD [0.39.0]: https://github.com/SonAIengine/graph-tool-call/compare/v0.38.0...v0.39.0 [0.38.0]: https://github.com/SonAIengine/graph-tool-call/compare/v0.37.0...v0.38.0 [0.37.0]: https://github.com/SonAIengine/graph-tool-call/compare/v0.36.0...v0.37.0 @@ -721,3 +740,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [0.8.0]: https://github.com/SonAIengine/graph-tool-call/compare/v0.5.0...v0.8.0 [0.28.0]: https://github.com/SonAIengine/graph-tool-call/compare/v0.27.0...v0.28.0 [0.40.0]: https://github.com/SonAIengine/graph-tool-call/compare/v0.39.0...v0.40.0 +[0.41.0]: https://github.com/SonAIengine/graph-tool-call/compare/v0.40.0...v0.41.0 +[0.42.0]: https://github.com/SonAIengine/graph-tool-call/compare/v0.41.0...v0.42.0 diff --git a/README.md b/README.md index 66b8935..66cf9e6 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ selected target produced: | Candidate binding support | 14.3% | **100%** | | Target Recall@5 | - | **100%** | -The [case-level v0.41.0 artifact](benchmarks/results/releases/v0.41.0/dependency-chain-evidence.json) +The [case-level v0.42.0 artifact](benchmarks/results/releases/v0.42.0/dependency-chain-evidence.json) records fixture hashes, every expected target and producer, and replay commands: ```bash @@ -239,7 +239,7 @@ make launch-evidence-check ``` The separate -[observability artifact](benchmarks/results/releases/v0.41.0/observability-evidence.json) +[observability artifact](benchmarks/results/releases/v0.42.0/observability-evidence.json) checks that tracing leaves engine inputs unchanged, replays deterministically, scrubs secrets, explains every decision, and stays below the documented `5ms/span` p95 capture-cost gate: diff --git a/benchmarks/results/arazzo_long_horizon_0.42.json b/benchmarks/results/arazzo_long_horizon_0.42.json index e7ebbc4..2c38c91 100644 --- a/benchmarks/results/arazzo_long_horizon_0.42.json +++ b/benchmarks/results/arazzo_long_horizon_0.42.json @@ -2,7 +2,7 @@ "benchmark": "Arazzo Long-Horizon Paired Evaluation", "methodology": "paired_deterministic_retrieve_plan_execute_goal_state", "model": "none", - "graph_tool_call_version": "0.41.0", + "graph_tool_call_version": "0.42.0", "catalog_size": 1000, "workflow_lengths": [ 3, @@ -30,16 +30,16 @@ }, "latency_ms": { "build": { - "p50": 455.804, - "p95": 481.3 + "p50": 394.394, + "p95": 404.765 }, "retrieve": { - "p50": 97.714, - "p95": 100.802 + "p50": 93.644, + "p95": 97.273 }, "plan_execute": { - "p50": 6.079, - "p95": 7.683 + "p50": 6.277, + "p95": 7.275 } } }, @@ -333,9 +333,9 @@ ] }, "latency_ms": { - "build": 465.411, - "retrieve": 102.285, - "plan_execute": 5.312 + "build": 409.959, + "retrieve": 99.292, + "plan_execute": 5.352 }, "token_budget_used": 73 }, @@ -632,9 +632,9 @@ "failure_reason_codes": [] }, "latency_ms": { - "build": 455.804, - "retrieve": 97.714, - "plan_execute": 6.033 + "build": 388.415, + "retrieve": 91.597, + "plan_execute": 6.277 }, "token_budget_used": 190 }, @@ -1122,9 +1122,9 @@ ] }, "latency_ms": { - "build": 475.985, - "retrieve": 97.232, - "plan_execute": 5.541 + "build": 393.437, + "retrieve": 94.572, + "plan_execute": 5.458 }, "token_budget_used": 118 }, @@ -1646,9 +1646,9 @@ "failure_reason_codes": [] }, "latency_ms": { - "build": 452.269, - "retrieve": 95.358, - "plan_execute": 6.079 + "build": 394.394, + "retrieve": 93.644, + "plan_execute": 5.975 }, "token_budget_used": 291 }, @@ -2716,9 +2716,9 @@ ] }, "latency_ms": { - "build": 458.637, - "retrieve": 98.179, - "plan_execute": 6.067 + "build": 393.102, + "retrieve": 99.029, + "plan_execute": 5.531 }, "token_budget_used": 125 }, @@ -3880,9 +3880,9 @@ "failure_reason_codes": [] }, "latency_ms": { - "build": 484.133, - "retrieve": 101.145, - "plan_execute": 7.861 + "build": 405.917, + "retrieve": 97.676, + "plan_execute": 7.386 }, "token_budget_used": 301 }, diff --git a/benchmarks/results/releases/v0.42.0/README.md b/benchmarks/results/releases/v0.42.0/README.md new file mode 100644 index 0000000..6f3505e --- /dev/null +++ b/benchmarks/results/releases/v0.42.0/README.md @@ -0,0 +1,29 @@ +# v0.42.0 release evidence + +This directory contains the model-free release artifacts for graph-tool-call +0.42.0. + +- `dependency-chain-evidence.json` records the deterministic target and + prerequisite-producer regression used by the README. +- `observability-evidence.json` records result invariance, deterministic replay, + secret scrubbing, reason coverage, serialized trace size, and measured trace + capture overhead. +- `../../arazzo_long_horizon_0.42.json` records paired OpenAPI-only and + OpenAPI-plus-Arazzo evaluation for 3-, 10-, and 30-call workflows in + 1,000-tool catalogs. + +Regenerate and validate the release artifacts with: + +```bash +make launch-evidence +make launch-evidence-check +make observability-evidence +make observability-evidence-check +make arazzo-long-horizon-benchmark \ + OUT=benchmarks/results/arazzo_long_horizon_0.42.json +``` + +These deterministic benchmarks do not use an LLM. The observability latency +value is a local Python microbenchmark and does not measure exporter backend or +service network latency. The Arazzo result measures engine workflow evidence, +planning, binding, and execution-order behavior rather than model reasoning. diff --git a/benchmarks/results/releases/v0.42.0/dependency-chain-evidence.json b/benchmarks/results/releases/v0.42.0/dependency-chain-evidence.json new file mode 100644 index 0000000..bcd3357 --- /dev/null +++ b/benchmarks/results/releases/v0.42.0/dependency-chain-evidence.json @@ -0,0 +1,328 @@ +{ + "benchmark": "XGEN Commerce Tool Graph Search", + "cases": [ + { + "case_id": "product_search_direct_ko", + "expanded_candidates": [ + "searchProducts" + ], + "expected_producers": [], + "expected_target": "searchProducts", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 0, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "상품 검색", + "retrieved": [ + "searchProducts", + "findOrders", + "addCartItem", + "getOrderDetail", + "createProductReview" + ], + "selected_target": "searchProducts", + "target_only": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 0, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "searchProducts" + ] + }, + { + "case_id": "product_detail_ko", + "expanded_candidates": [ + "getProductDetail", + "searchProducts" + ], + "expected_producers": [ + "searchProducts" + ], + "expected_target": "getProductDetail", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 1, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "상품명으로 검색해서 상품 상세를 보여줘", + "retrieved": [ + "searchProducts", + "getProductDetail", + "addCartItem", + "getCart", + "createProductReview" + ], + "selected_target": "getProductDetail", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.5, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "getProductDetail" + ] + }, + { + "case_id": "inventory_chain_ko", + "expanded_candidates": [ + "getInventory", + "searchProducts", + "getProductDetail", + "getCart" + ], + "expected_producers": [ + "getProductDetail" + ], + "expected_target": "getInventory", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 3, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "상품 sku 재고 수량 확인해줘", + "retrieved": [ + "getInventory", + "addCartItem", + "getProductDetail", + "searchProducts", + "getCart" + ], + "selected_target": "getInventory", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.3333333333333333, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "getInventory" + ] + }, + { + "case_id": "add_cart_chain_ko", + "expanded_candidates": [ + "addCartItem", + "searchProducts", + "getProductDetail", + "getCart" + ], + "expected_producers": [ + "getProductDetail" + ], + "expected_target": "addCartItem", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 3, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "장바구니에 상품 하나 담아줘", + "retrieved": [ + "addCartItem", + "getCart", + "searchProducts", + "checkoutCart", + "getProductDetail" + ], + "selected_target": "addCartItem", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.3333333333333333, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "addCartItem" + ] + }, + { + "case_id": "coupon_checkout_ko", + "expanded_candidates": [ + "checkoutCart", + "getCart", + "validateCoupon" + ], + "expected_producers": [ + "getCart", + "validateCoupon" + ], + "expected_target": "checkoutCart", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 2, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "쿠폰 적용해서 장바구니 결제 진행", + "retrieved": [ + "checkoutCart", + "validateCoupon", + "getCart", + "addCartItem", + "getProductDetail" + ], + "selected_target": "checkoutCart", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.3333333333333333, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "checkoutCart" + ] + }, + { + "case_id": "shipment_tracking_ko", + "expanded_candidates": [ + "getShipmentTracking", + "getOrderDetail", + "findOrders", + "checkoutCart" + ], + "expected_producers": [ + "getOrderDetail" + ], + "expected_target": "getShipmentTracking", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 3, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "주문한 상품 배송 추적 상태 알려줘", + "retrieved": [ + "getShipmentTracking", + "getOrderDetail", + "searchProducts", + "findOrders", + "addCartItem" + ], + "selected_target": "getShipmentTracking", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.3333333333333333, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "getShipmentTracking" + ] + }, + { + "case_id": "review_user_slots_ko", + "expanded_candidates": [ + "createProductReview", + "searchProducts" + ], + "expected_producers": [ + "searchProducts" + ], + "expected_target": "createProductReview", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 1, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "구매한 상품 리뷰 작성할래", + "retrieved": [ + "createProductReview", + "getProductDetail", + "searchProducts", + "addCartItem", + "getInventory" + ], + "selected_target": "createProductReview", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.5, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "createProductReview" + ] + } + ], + "claims": { + "candidate_binding_support": { + "after": 1.0, + "before": 0.142857, + "delta": 0.857143 + }, + "candidate_plan_coverage": { + "after": 1.0, + "before": 0.47619, + "delta": 0.52381 + }, + "expanded_pipeline_status": "pass", + "producer_needed_cases": 6, + "producer_recall": { + "after": 1.0, + "before": 0.142857, + "delta": 0.857143 + }, + "target_recall_at_5": 1.0, + "target_selector_exact": 1.0, + "unneeded_expansion_cases": 0 + }, + "dataset": { + "case_count": 7, + "cases_path": "benchmarks/xgen_tool_graph/cases.json", + "cases_sha256": "7db14a433a4c4e7637d4df566e0345cefcc193ecb6fcaefe553a432cba32a6af", + "edge_count": 13, + "spec_path": "benchmarks/xgen_tool_graph/commerce_openapi.json", + "spec_sha256": "06de6b2ff75f11000d5b415103348eab0bae6be9ec5584bf637491a96312651e", + "tool_count": 11 + }, + "graph_tool_call_version": "0.42.0", + "limitations": [ + "Deterministic engine regression; no LLM is used.", + "Seven curated commerce cases; not a population-level accuracy estimate.", + "Measures retrieval, target selection, dependency expansion, and plan coverage." + ], + "methodology": "deterministic_engine_contract", + "model": "none", + "release_ref": "v0.42.0", + "replay": { + "check_command": "make launch-evidence-check", + "command": "make launch-evidence" + }, + "schema_version": "launch-evidence-v1" +} diff --git a/benchmarks/results/releases/v0.42.0/observability-evidence.json b/benchmarks/results/releases/v0.42.0/observability-evidence.json new file mode 100644 index 0000000..bea6646 --- /dev/null +++ b/benchmarks/results/releases/v0.42.0/observability-evidence.json @@ -0,0 +1,51 @@ +{ + "captured_at_utc": "2026-08-16T11:31:05.893909+00:00", + "checks": { + "overhead_within_threshold": true, + "reason_coverage_complete": true, + "replay_deterministic": true, + "result_invariant": true, + "secret_scan_passed": true, + "serialized_size_bounded": true, + "stage_order_complete": true + }, + "environment": { + "implementation": "CPython", + "platform": "linux", + "python": "3.12.3" + }, + "graph_tool_call_version": "0.42.0", + "limitations": [ + "Local deterministic adapter scenario; no external API or LLM is used.", + "Latency is a Python microbenchmark and not an end-to-end service SLO.", + "The gate measures trace capture cost, not backend exporter latency." + ], + "overhead": { + "iterations": 200, + "per_span_p50_ms": 0.087649, + "per_span_p95_ms": 0.095467, + "spans_per_trace": 6, + "threshold_p95_ms": 5.0 + }, + "release_ref": "v0.42.0", + "replay": { + "check_command": "make observability-evidence-check", + "generate_command": "make observability-evidence" + }, + "scenario": { + "decision_count": 11, + "max_serialized_trace_bytes": 65536, + "reason_coverage": 1.0, + "serialized_trace_bytes": 5763, + "stage_order": [ + "retrieval", + "target_selection", + "dependency_closure", + "schema_admission", + "plan", + "runner" + ] + }, + "schema_version": "observability-release-evidence-v1", + "status": "pass" +} diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 9fe8cd4..41d1ebf 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -16,8 +16,8 @@ expensive full model benchmarks. - **Pipelines compared**: `baseline` (all tools), `retrieve-k3 / k5 / k10`, plus `+ embedding`, `+ ontology` - **Reproduce**: see [Reproduce](#reproduce) at the bottom -> **Release-claim policy:** the `v0.41.0` README headline uses the checked-in, -> model-free [dependency-chain release artifact](../benchmarks/results/releases/v0.41.0/dependency-chain-evidence.json). +> **Release-claim policy:** the `v0.42.0` README headline uses the checked-in, +> model-free [dependency-chain release artifact](../benchmarks/results/releases/v0.42.0/dependency-chain-evidence.json). > The older `qwen3:4b` tables below document historical self-hosted runs. Their > original case-level output was not preserved in the current repository, so > they are not used as a current release or leaderboard claim. @@ -48,7 +48,7 @@ population-level estimate of LLM tool-calling accuracy. The 0.38 release introduced a measured, replayable trace gate for retrieval, target selection, dependency closure, schema admission, planning, and runner events. The current -[observability artifact](../benchmarks/results/releases/v0.41.0/observability-evidence.json) +[observability artifact](../benchmarks/results/releases/v0.42.0/observability-evidence.json) records the environment and case-level checks. CI also runs a fresh local microbenchmark instead of trusting only the stored number. diff --git a/graph_tool_call/__init__.py b/graph_tool_call/__init__.py index e8dbe84..9e756dd 100644 --- a/graph_tool_call/__init__.py +++ b/graph_tool_call/__init__.py @@ -57,7 +57,7 @@ "unregister_ingest_adapter", ] -__version__ = "0.41.0" +__version__ = "0.42.0" # Lazy imports for analyze/assist symbols — avoid loading heavy submodules at import time _LAZY_IMPORTS: dict[str, tuple[str, str]] = { diff --git a/pyproject.toml b/pyproject.toml index f7e19df..559ecb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "graph-tool-call" -version = "0.41.0" +version = "0.42.0" description = "Graph-structured tool retrieval for LLM agents — zero-dependency, ontology-aware hybrid search" authors = ["SonAIengine"] license = "MIT"