diff --git a/CHANGELOG.md b/CHANGELOG.md index 2190388..cfee3d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.41.0] - 2026-08-16 + +### Added +- Arazzo 1.0/1.1 workflow evidence ingestion for explicit call ordering, + cross-workflow dependencies, and response-to-request runtime bindings. +- Public workflow evidence helpers and optional OpenAPI collection artifact + sources with deterministic provenance, source hashing, and summary metrics. +- CLI and English/Korean documentation for applying workflow specifications + without changing existing OpenAPI-only collection behavior. + +### Changed +- Execution-flow diagnostics and graph quality summaries now classify Arazzo + relations as strong workflow evidence. +- Planning interprets explicit `precedes` edges in producer-to-consumer order. + ## [0.40.0] - 2026-08-15 ### Added diff --git a/README.md b/README.md index a2a41c7..66b8935 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ selected target produced: | Candidate binding support | 14.3% | **100%** | | Target Recall@5 | - | **100%** | -The [case-level v0.40.0 artifact](benchmarks/results/releases/v0.40.0/dependency-chain-evidence.json) +The [case-level v0.41.0 artifact](benchmarks/results/releases/v0.41.0/dependency-chain-evidence.json) records fixture hashes, every expected target and producer, and replay commands: ```bash @@ -239,7 +239,7 @@ make launch-evidence-check ``` The separate -[observability artifact](benchmarks/results/releases/v0.40.0/observability-evidence.json) +[observability artifact](benchmarks/results/releases/v0.41.0/observability-evidence.json) checks that tracing leaves engine inputs unchanged, replays deterministically, scrubs secrets, explains every decision, and stays below the documented `5ms/span` p95 capture-cost gate: diff --git a/benchmarks/results/releases/v0.41.0/README.md b/benchmarks/results/releases/v0.41.0/README.md new file mode 100644 index 0000000..aab3bfe --- /dev/null +++ b/benchmarks/results/releases/v0.41.0/README.md @@ -0,0 +1,25 @@ +# v0.41.0 release evidence + +This directory contains the model-free release artifacts for graph-tool-call +0.41.0. + +- `dependency-chain-evidence.json` records the deterministic target and + prerequisite-producer regression used by the README. +- `observability-evidence.json` records result invariance, deterministic replay, + secret scrubbing, reason coverage, serialized trace size, and measured trace + capture overhead. +- Arazzo workflow ordering, runtime binding promotion, artifact persistence, and + source provenance are covered by the public workflow evidence tests. + +Regenerate and validate the artifacts with: + +```bash +make launch-evidence +make launch-evidence-check +make observability-evidence +make observability-evidence-check +``` + +The deterministic release benchmark does not use an LLM. The observability +latency value is a local Python microbenchmark and does not measure exporter +backend or service network latency. diff --git a/benchmarks/results/releases/v0.41.0/dependency-chain-evidence.json b/benchmarks/results/releases/v0.41.0/dependency-chain-evidence.json new file mode 100644 index 0000000..917c743 --- /dev/null +++ b/benchmarks/results/releases/v0.41.0/dependency-chain-evidence.json @@ -0,0 +1,328 @@ +{ + "benchmark": "XGEN Commerce Tool Graph Search", + "cases": [ + { + "case_id": "product_search_direct_ko", + "expanded_candidates": [ + "searchProducts" + ], + "expected_producers": [], + "expected_target": "searchProducts", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 0, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "상품 검색", + "retrieved": [ + "searchProducts", + "findOrders", + "addCartItem", + "getOrderDetail", + "createProductReview" + ], + "selected_target": "searchProducts", + "target_only": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 0, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "searchProducts" + ] + }, + { + "case_id": "product_detail_ko", + "expanded_candidates": [ + "getProductDetail", + "searchProducts" + ], + "expected_producers": [ + "searchProducts" + ], + "expected_target": "getProductDetail", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 1, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "상품명으로 검색해서 상품 상세를 보여줘", + "retrieved": [ + "searchProducts", + "getProductDetail", + "addCartItem", + "getCart", + "createProductReview" + ], + "selected_target": "getProductDetail", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.5, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "getProductDetail" + ] + }, + { + "case_id": "inventory_chain_ko", + "expanded_candidates": [ + "getInventory", + "searchProducts", + "getProductDetail", + "getCart" + ], + "expected_producers": [ + "getProductDetail" + ], + "expected_target": "getInventory", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 3, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "상품 sku 재고 수량 확인해줘", + "retrieved": [ + "getInventory", + "addCartItem", + "getProductDetail", + "searchProducts", + "getCart" + ], + "selected_target": "getInventory", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.3333333333333333, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "getInventory" + ] + }, + { + "case_id": "add_cart_chain_ko", + "expanded_candidates": [ + "addCartItem", + "searchProducts", + "getProductDetail", + "getCart" + ], + "expected_producers": [ + "getProductDetail" + ], + "expected_target": "addCartItem", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 3, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "장바구니에 상품 하나 담아줘", + "retrieved": [ + "addCartItem", + "getCart", + "searchProducts", + "checkoutCart", + "getProductDetail" + ], + "selected_target": "addCartItem", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.3333333333333333, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "addCartItem" + ] + }, + { + "case_id": "coupon_checkout_ko", + "expanded_candidates": [ + "checkoutCart", + "getCart", + "validateCoupon" + ], + "expected_producers": [ + "getCart", + "validateCoupon" + ], + "expected_target": "checkoutCart", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 2, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "쿠폰 적용해서 장바구니 결제 진행", + "retrieved": [ + "checkoutCart", + "validateCoupon", + "getCart", + "addCartItem", + "getProductDetail" + ], + "selected_target": "checkoutCart", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.3333333333333333, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "checkoutCart" + ] + }, + { + "case_id": "shipment_tracking_ko", + "expanded_candidates": [ + "getShipmentTracking", + "getOrderDetail", + "findOrders", + "checkoutCart" + ], + "expected_producers": [ + "getOrderDetail" + ], + "expected_target": "getShipmentTracking", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 3, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "주문한 상품 배송 추적 상태 알려줘", + "retrieved": [ + "getShipmentTracking", + "getOrderDetail", + "searchProducts", + "findOrders", + "addCartItem" + ], + "selected_target": "getShipmentTracking", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.3333333333333333, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "getShipmentTracking" + ] + }, + { + "case_id": "review_user_slots_ko", + "expanded_candidates": [ + "createProductReview", + "searchProducts" + ], + "expected_producers": [ + "searchProducts" + ], + "expected_target": "createProductReview", + "graph_with_producers": { + "candidate_binding_support": 1.0, + "candidate_plan_coverage": 1.0, + "producer_added_count": 1, + "producer_recall": 1.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "query": "구매한 상품 리뷰 작성할래", + "retrieved": [ + "createProductReview", + "getProductDetail", + "searchProducts", + "addCartItem", + "getInventory" + ], + "selected_target": "createProductReview", + "target_only": { + "candidate_binding_support": 0.0, + "candidate_plan_coverage": 0.5, + "producer_added_count": 0, + "producer_recall": 0.0, + "target_recall_at_k": 1.0, + "target_selector_exact": 1.0 + }, + "target_only_candidates": [ + "createProductReview" + ] + } + ], + "claims": { + "candidate_binding_support": { + "after": 1.0, + "before": 0.142857, + "delta": 0.857143 + }, + "candidate_plan_coverage": { + "after": 1.0, + "before": 0.47619, + "delta": 0.52381 + }, + "expanded_pipeline_status": "pass", + "producer_needed_cases": 6, + "producer_recall": { + "after": 1.0, + "before": 0.142857, + "delta": 0.857143 + }, + "target_recall_at_5": 1.0, + "target_selector_exact": 1.0, + "unneeded_expansion_cases": 0 + }, + "dataset": { + "case_count": 7, + "cases_path": "benchmarks/xgen_tool_graph/cases.json", + "cases_sha256": "7db14a433a4c4e7637d4df566e0345cefcc193ecb6fcaefe553a432cba32a6af", + "edge_count": 13, + "spec_path": "benchmarks/xgen_tool_graph/commerce_openapi.json", + "spec_sha256": "06de6b2ff75f11000d5b415103348eab0bae6be9ec5584bf637491a96312651e", + "tool_count": 11 + }, + "graph_tool_call_version": "0.41.0", + "limitations": [ + "Deterministic engine regression; no LLM is used.", + "Seven curated commerce cases; not a population-level accuracy estimate.", + "Measures retrieval, target selection, dependency expansion, and plan coverage." + ], + "methodology": "deterministic_engine_contract", + "model": "none", + "release_ref": "v0.41.0", + "replay": { + "check_command": "make launch-evidence-check", + "command": "make launch-evidence" + }, + "schema_version": "launch-evidence-v1" +} diff --git a/benchmarks/results/releases/v0.41.0/observability-evidence.json b/benchmarks/results/releases/v0.41.0/observability-evidence.json new file mode 100644 index 0000000..339f5f7 --- /dev/null +++ b/benchmarks/results/releases/v0.41.0/observability-evidence.json @@ -0,0 +1,51 @@ +{ + "captured_at_utc": "2026-08-16T05:59:52.045704+00:00", + "checks": { + "overhead_within_threshold": true, + "reason_coverage_complete": true, + "replay_deterministic": true, + "result_invariant": true, + "secret_scan_passed": true, + "serialized_size_bounded": true, + "stage_order_complete": true + }, + "environment": { + "implementation": "CPython", + "platform": "linux", + "python": "3.12.3" + }, + "graph_tool_call_version": "0.41.0", + "limitations": [ + "Local deterministic adapter scenario; no external API or LLM is used.", + "Latency is a Python microbenchmark and not an end-to-end service SLO.", + "The gate measures trace capture cost, not backend exporter latency." + ], + "overhead": { + "iterations": 200, + "per_span_p50_ms": 0.086966, + "per_span_p95_ms": 0.172001, + "spans_per_trace": 6, + "threshold_p95_ms": 5.0 + }, + "release_ref": "v0.41.0", + "replay": { + "check_command": "make observability-evidence-check", + "generate_command": "make observability-evidence" + }, + "scenario": { + "decision_count": 11, + "max_serialized_trace_bytes": 65536, + "reason_coverage": 1.0, + "serialized_trace_bytes": 5763, + "stage_order": [ + "retrieval", + "target_selection", + "dependency_closure", + "schema_admission", + "plan", + "runner" + ] + }, + "schema_version": "observability-release-evidence-v1", + "status": "pass" +} diff --git a/docs/api-reference.md b/docs/api-reference.md index 5a5e3bd..0b21ec6 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -32,7 +32,7 @@ tools = tg.retrieve("create a pet", top_k=5) | `ingest_mcp_tools(tools)` | Ingest from MCP tool list | | `ingest_mcp_server(url)` | Fetch and ingest from an MCP HTTP server | | `ingest_functions(fns)` | Ingest from Python callables (uses type hints + docstrings) | -| `ingest_arazzo(source)` | Ingest Arazzo 1.0.0 workflow spec | +| `ingest_arazzo(source)` | Ingest Arazzo 1.0/1.1 workflow evidence | | `add_relation(src, tgt, type)` | Add a manual relation between two tools | ### Source-agnostic ingest adapters @@ -531,6 +531,7 @@ from graph_tool_call.graphify import build_openapi_collection_artifact artifact = build_openapi_collection_artifact( "https://api.example.com/swagger-ui/index.html", + workflow_sources=["./arazzo.yaml"], allow_private_hosts=True, context_field_names={"siteNo", "tenantId"}, paging_field_names={"pageNo", "pageSize"}, @@ -550,11 +551,22 @@ The artifact includes: - `edge_stats`: graphify edge and contract-promotion statistics - `semantic_summary`: action/resource/module coverage and unknown samples - `edge_quality_summary`: deterministic/manual/run/name-based edge provenance +- `workflow_summary`: Arazzo source/workflow/step/relation counts, dependency + kinds, edge merge counts, and source snapshot hashes + +When `workflow_sources` is supplied, Arazzo `dependsOn`, ordered steps, and +runtime output references are merged as `arazzo` evidence. A runtime binding +such as `$steps.lookup.outputs.itemId` also creates a producer alias with the +concrete response path, allowing `PathSynthesizer` to bind the consumer input +without guessing from similarly named response leaves. Explicit Arazzo and +OpenAPI Link bindings outrank generic schema-derived paths. Existing manual, +trace, and OpenAPI contract metadata remains additive. The CLI equivalent is: ```bash graph-tool-call build-openapi-collection openapi.json -o collection.json \ + --workflow arazzo.yaml \ --resource-alias goods=product \ --module-alias goodsCommonApi=goods_common ``` diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 462644c..f4b1aa9 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -16,8 +16,8 @@ expensive full model benchmarks. - **Pipelines compared**: `baseline` (all tools), `retrieve-k3 / k5 / k10`, plus `+ embedding`, `+ ontology` - **Reproduce**: see [Reproduce](#reproduce) at the bottom -> **Release-claim policy:** the `v0.40.0` README headline uses the checked-in, -> model-free [dependency-chain release artifact](../benchmarks/results/releases/v0.40.0/dependency-chain-evidence.json). +> **Release-claim policy:** the `v0.41.0` README headline uses the checked-in, +> model-free [dependency-chain release artifact](../benchmarks/results/releases/v0.41.0/dependency-chain-evidence.json). > The older `qwen3:4b` tables below document historical self-hosted runs. Their > original case-level output was not preserved in the current repository, so > they are not used as a current release or leaderboard claim. @@ -48,7 +48,7 @@ population-level estimate of LLM tool-calling accuracy. The 0.38 release introduced a measured, replayable trace gate for retrieval, target selection, dependency closure, schema admission, planning, and runner events. The current -[observability artifact](../benchmarks/results/releases/v0.40.0/observability-evidence.json) +[observability artifact](../benchmarks/results/releases/v0.41.0/observability-evidence.json) records the environment and case-level checks. CI also runs a fresh local microbenchmark instead of trusting only the stored number. diff --git a/docs/design/call-ordering.md b/docs/design/call-ordering.md index 066697e..06cc967 100644 --- a/docs/design/call-ordering.md +++ b/docs/design/call-ordering.md @@ -116,8 +116,11 @@ workflows: dependsOn: find-pet ``` -Arazzo spec이 있으면 완벽한 순서 관계를 추출할 수 있다. -없으면 Layer 1+2의 휴리스틱으로 추론. +Arazzo spec이 있으면 명시된 `dependsOn`, step 순서, runtime output 참조를 +`arazzo` 근거로 추출한다. `$steps..outputs.` 참조는 호출 순서뿐 +아니라 producer 응답 경로와 consumer 입력 필드의 binding에도 사용한다. +명세에 기록되지 않은 분기나 비즈니스 규칙은 보장하지 않으며, 해당 부분은 +OpenAPI Link, 실행 trace, 수동 근거와 함께 보완한다. ## 커머스 도메인 특화 패턴 diff --git a/docs/design/openapi-guide.md b/docs/design/openapi-guide.md index acbc848..7b946a3 100644 --- a/docs/design/openapi-guide.md +++ b/docs/design/openapi-guide.md @@ -204,23 +204,32 @@ API 간 호출 순서가 중요하면 Arazzo spec 추가: ```yaml # arazzo.yaml -arazzo: 1.0.0 +arazzo: 1.1.0 info: title: Order Workflow + version: 1.0.0 +sourceDescriptions: [] workflows: - workflowId: order-cancel steps: - stepId: list operationId: listOrders + outputs: + orderId: $response.body#/items/0/orderId - stepId: get operationId: getOrder dependsOn: list + parameters: + - name: orderId + in: path + value: $steps.list.outputs.orderId - stepId: cancel operationId: cancelOrder dependsOn: get ``` -**효과**: 완벽한 호출 순서 관계 추출 (confidence 1.0) +**효과**: 명시된 호출 순서와 response-to-request binding을 deterministic하게 +추출한다. 명세 밖의 경로는 별도 계약/trace 근거가 필요하다. ## 체크리스트 diff --git a/graph_tool_call/__init__.py b/graph_tool_call/__init__.py index 91e1bdb..4bfd5fa 100644 --- a/graph_tool_call/__init__.py +++ b/graph_tool_call/__init__.py @@ -53,7 +53,7 @@ "unregister_ingest_adapter", ] -__version__ = "0.40.0" +__version__ = "0.41.0" # Lazy imports for analyze/assist symbols — avoid loading heavy submodules at import time _LAZY_IMPORTS: dict[str, tuple[str, str]] = { diff --git a/graph_tool_call/__main__.py b/graph_tool_call/__main__.py index 66a4f51..f3114fc 100644 --- a/graph_tool_call/__main__.py +++ b/graph_tool_call/__main__.py @@ -121,6 +121,13 @@ def _build_parser() -> argparse.ArgumentParser: default="collection.json", help="Output artifact path, or '-' for stdout", ) + p_build_collection.add_argument( + "--workflow", + action="append", + dest="workflow_sources", + default=[], + help="Arazzo workflow URL or file path (repeatable)", + ) p_build_collection.add_argument("--required-only", action="store_true") p_build_collection.add_argument("--include-deprecated", action="store_true") p_build_collection.add_argument( @@ -619,6 +626,7 @@ def cmd_build_openapi_collection(args: argparse.Namespace) -> None: artifact = build_openapi_collection_artifact( args.source, + workflow_sources=args.workflow_sources or None, required_only=args.required_only, skip_deprecated=not args.include_deprecated, allow_private_hosts=args.allow_private_hosts, diff --git a/graph_tool_call/graphify/__init__.py b/graph_tool_call/graphify/__init__.py index 57db146..1ac0d43 100644 --- a/graph_tool_call/graphify/__init__.py +++ b/graph_tool_call/graphify/__init__.py @@ -32,6 +32,7 @@ ) from graph_tool_call.graphify.edges import ( EVIDENCE_API_CONTRACT, + EVIDENCE_ARAZZO, EVIDENCE_LLM_CURATED, EVIDENCE_MANUAL, EVIDENCE_NAME_BASED, @@ -77,6 +78,10 @@ summarize_edge_quality, summarize_openapi_semantics, ) +from graph_tool_call.graphify.workflow_evidence import ( + apply_arazzo_relations, + apply_arazzo_workflows, +) __all__ = [ "COLLECTION_GRAPH_VERSION", @@ -85,6 +90,7 @@ "DEFAULT_CONF_AMBIGUOUS", "DEFAULT_CONF_EXTRACTED", "DEFAULT_CONF_INFERRED", + "EVIDENCE_ARAZZO", "EVIDENCE_API_CONTRACT", "EVIDENCE_LLM_CURATED", "EVIDENCE_MANUAL", @@ -100,6 +106,8 @@ "_apply_pair_hints", "annotate_graphify_metadata", "annotate_openapi_tool_semantics", + "apply_arazzo_relations", + "apply_arazzo_workflows", "assemble_tool_bundle", "bucket_confidence", "build_candidate_set", diff --git a/graph_tool_call/graphify/collection_artifact.py b/graph_tool_call/graphify/collection_artifact.py index 80fcafb..274e45a 100644 --- a/graph_tool_call/graphify/collection_artifact.py +++ b/graph_tool_call/graphify/collection_artifact.py @@ -27,6 +27,7 @@ summarize_edge_quality, summarize_openapi_semantics, ) +from graph_tool_call.graphify.workflow_evidence import apply_arazzo_workflows from graph_tool_call.ingest.openapi import _load_spec, ingest_openapi from graph_tool_call.tool_graph import _discover_spec_urls @@ -44,6 +45,7 @@ class _LoadedOpenAPISpec: def build_openapi_collection_artifact( source: dict[str, Any] | str | Sequence[dict[str, Any] | str], *, + workflow_sources: dict[str, Any] | str | Sequence[dict[str, Any] | str] | None = None, required_only: bool = False, skip_deprecated: bool = True, allow_private_hosts: bool = False, @@ -104,6 +106,16 @@ def build_openapi_collection_artifact( paging_field_names=paging_field_names, search_filter_field_names=search_filter_field_names, ) + workflow_summary = _empty_workflow_summary() + if workflow_sources is not None: + workflow_summary = apply_arazzo_workflows( + tg, + workflow_sources, + allow_private_hosts=allow_private_hosts, + max_response_bytes=max_response_bytes, + ) + edge_stats["edge_count"] = tg.graph.edge_count() + edge_stats["arazzo_workflows"] = workflow_summary["edge_stats"] semantic_summary = summarize_openapi_semantics(unique_tools, options=semantic_options) edge_quality_summary = summarize_edge_quality(tg.graph) readiness_report = analyze_openapi_tools( @@ -129,6 +141,7 @@ def build_openapi_collection_artifact( "search_filter_field_names": _sorted_names(search_filter_field_names), "auth_field_names": _sorted_names(auth_field_names), "user_input_field_names": _sorted_names(user_input_field_names), + "workflow_source_count": int(workflow_summary["source_count"]), } if contract_signal_options: build_options["contract_signal_options"] = dict(contract_signal_options) @@ -147,6 +160,7 @@ def build_openapi_collection_artifact( "edge_stats": edge_stats, "semantic_summary": semantic_summary, "edge_quality_summary": edge_quality_summary, + "workflow_summary": workflow_summary, "build_options": build_options, } if metadata: @@ -166,6 +180,7 @@ def build_openapi_collection_artifact( "edge_stats": edge_stats, "semantic_summary": semantic_summary, "edge_quality_summary": edge_quality_summary, + "workflow_summary": workflow_summary, } return annotate_graphify_metadata( graph_payload, @@ -174,6 +189,18 @@ def build_openapi_collection_artifact( ) +def _empty_workflow_summary() -> dict[str, Any]: + return { + "source_count": 0, + "workflow_count": 0, + "step_count": 0, + "relation_count": 0, + "by_dependency_kind": {}, + "edge_stats": {"added": 0, "merged": 0, "binding_aliases_added": 0}, + "source_snapshot_manifest": {"spec_count": 0, "specs": []}, + } + + def _load_openapi_collection_sources( source: dict[str, Any] | str | Sequence[dict[str, Any] | str], *, diff --git a/graph_tool_call/graphify/edges.py b/graph_tool_call/graphify/edges.py index f4d1c93..7c98d9e 100644 --- a/graph_tool_call/graphify/edges.py +++ b/graph_tool_call/graphify/edges.py @@ -20,6 +20,7 @@ EVIDENCE_MANUAL = "manual" EVIDENCE_API_CONTRACT = "api_contract" EVIDENCE_OPENAPI_LINK = "openapi_link" +EVIDENCE_ARAZZO = "arazzo" _DATA_FLOW_RELATIONS = frozenset({"requires", "precedes", "produces_for"}) _BINDING_RE = re.compile(r"^\$\{(\w+)\.(.+)\}$") @@ -251,6 +252,7 @@ def _infer_kind(relation: Any) -> str: __all__ = [ + "EVIDENCE_ARAZZO", "EVIDENCE_API_CONTRACT", "EVIDENCE_LLM_CURATED", "EVIDENCE_MANUAL", diff --git a/graph_tool_call/graphify/execution_flow.py b/graph_tool_call/graphify/execution_flow.py index ceb71e4..bdd6f37 100644 --- a/graph_tool_call/graphify/execution_flow.py +++ b/graph_tool_call/graphify/execution_flow.py @@ -58,14 +58,18 @@ "from_path", "input_field", "link_name", + "dependency_kind", "observed_count", "output_field", + "source_step_id", "source_field", "source_field_path", "target_field", "target_field_path", "to_field", + "target_step_id", "value_type", + "workflow_id", } ) @@ -227,6 +231,8 @@ def _evidence_type(sources: list[str]) -> str: return "observed" if values.intersection({"api_contract", "openapi_link"}): return "contract" + if "arazzo" in values: + return "workflow" if values.intersection({"manual", "llm_curated", "llm_validated"}): return "curated" return "inferred" diff --git a/graph_tool_call/graphify/semantics.py b/graph_tool_call/graphify/semantics.py index 6cf1fab..6cb5585 100644 --- a/graph_tool_call/graphify/semantics.py +++ b/graph_tool_call/graphify/semantics.py @@ -16,6 +16,7 @@ from graph_tool_call.core.tool import ToolSchema from graph_tool_call.graphify.edges import ( EVIDENCE_API_CONTRACT, + EVIDENCE_ARAZZO, EVIDENCE_MANUAL, EVIDENCE_OPENAPI_LINK, EVIDENCE_PROVEN, @@ -439,6 +440,8 @@ def summarize_edge_quality(graph: GraphEngine | None) -> dict[str, Any]: counters["name_based"] += 1 if EVIDENCE_MANUAL in evidence_sources or attrs.get("is_manual"): counters["manual"] += 1 + if EVIDENCE_ARAZZO in evidence_sources: + counters["workflow"] += 1 if EVIDENCE_RUN in evidence_sources or EVIDENCE_PROVEN in evidence_sources: counters["trace"] += 1 if _has_strong_deterministic_evidence(evidence_sources, confidence, conf_score): @@ -902,6 +905,7 @@ def _edge_quality_payload( "structural": counters.get("structural", 0), "name_based": counters.get("name_based", 0), "manual": counters.get("manual", 0), + "workflow": counters.get("workflow", 0), "trace": counters.get("trace", 0), "strong_deterministic_evidence": counters.get("strong_deterministic_evidence", 0), "visual_edge_candidate_count": visual_candidate_count, @@ -925,7 +929,12 @@ def _has_strong_deterministic_evidence( confidence: str, conf_score: float, ) -> bool: - if evidence_sources & {EVIDENCE_API_CONTRACT, EVIDENCE_OPENAPI_LINK, EVIDENCE_STRUCTURAL}: + if evidence_sources & { + EVIDENCE_API_CONTRACT, + EVIDENCE_ARAZZO, + EVIDENCE_OPENAPI_LINK, + EVIDENCE_STRUCTURAL, + }: return True return confidence == "EXTRACTED" and conf_score >= 0.85 @@ -937,7 +946,12 @@ def _is_visual_edge_candidate( conf_score: float, attrs: Mapping[str, Any], ) -> bool: - if evidence_sources & {EVIDENCE_API_CONTRACT, EVIDENCE_OPENAPI_LINK, EVIDENCE_MANUAL}: + if evidence_sources & { + EVIDENCE_API_CONTRACT, + EVIDENCE_ARAZZO, + EVIDENCE_OPENAPI_LINK, + EVIDENCE_MANUAL, + }: return True if evidence_sources & {EVIDENCE_RUN, EVIDENCE_PROVEN}: return True diff --git a/graph_tool_call/graphify/workflow_evidence.py b/graph_tool_call/graphify/workflow_evidence.py new file mode 100644 index 0000000..99a9f23 --- /dev/null +++ b/graph_tool_call/graphify/workflow_evidence.py @@ -0,0 +1,293 @@ +"""Merge explicit Arazzo workflow evidence into a tool graph. + +Only operation names, step identifiers, dependency kinds, and response-path +bindings are persisted. Runtime parameter values and API payloads are never +copied into the graph artifact. +""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from collections.abc import Sequence +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +from graph_tool_call.graphify.edges import ( + EVIDENCE_ARAZZO, + merge_graph_edges, + normalize_graph_edge, +) +from graph_tool_call.ingest.arazzo import ArazzoRelation, _load_spec, ingest_arazzo +from graph_tool_call.ontology.schema import Confidence, RelationType +from graph_tool_call.tool_graph import ToolGraph + + +def apply_arazzo_workflows( + graph: ToolGraph, + sources: dict[str, Any] | str | Sequence[dict[str, Any] | str], + *, + allow_private_hosts: bool = False, + max_response_bytes: int = 5_000_000, +) -> dict[str, Any]: + """Apply one or more Arazzo descriptions as extracted workflow evidence.""" + + requested = _requested_sources(sources) + resolver = _operation_resolver(graph) + relations: list[ArazzoRelation] = [] + manifests: list[dict[str, Any]] = [] + workflow_count = 0 + step_count = 0 + + for index, source in enumerate(requested, start=1): + spec = ( + source + if isinstance(source, dict) + else _load_spec( + source, + allow_private_hosts=allow_private_hosts, + max_response_bytes=max_response_bytes, + ) + ) + canonical = json.dumps(spec, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + workflows = [row for row in spec.get("workflows") or [] if isinstance(row, dict)] + workflow_count += len(workflows) + step_count += sum( + len([step for step in workflow.get("steps") or [] if isinstance(step, dict)]) + for workflow in workflows + ) + manifests.append( + { + "index": index, + "source": ( + f"inline:{index}" if isinstance(source, dict) else _safe_source_label(source) + ), + "sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), + "bytes": len(canonical.encode("utf-8")), + "arazzo_version": str(spec.get("arazzo") or ""), + "workflow_count": len(workflows), + } + ) + relations.extend( + ingest_arazzo( + spec, + registered_tools=set(graph.tools), + operation_resolver=resolver, + ) + ) + + edge_stats = apply_arazzo_relations(graph, relations) + return { + "source_count": len(requested), + "workflow_count": workflow_count, + "step_count": step_count, + "relation_count": len(relations), + "by_dependency_kind": dict( + sorted(Counter(relation.dependency_kind for relation in relations).items()) + ), + "edge_stats": edge_stats, + "source_snapshot_manifest": { + "spec_count": len(manifests), + "specs": manifests, + }, + } + + +def apply_arazzo_relations( + graph: ToolGraph, + relations: Sequence[ArazzoRelation], +) -> dict[str, int]: + """Apply parsed relations while preserving graphify evidence metadata.""" + + stats = {"added": 0, "merged": 0, "binding_aliases_added": 0} + for relation in relations: + bindings = [dict(binding) for binding in relation.bindings] + incoming = normalize_graph_edge( + { + "source": relation.source, + "target": relation.target, + "relation": RelationType.PRECEDES, + "confidence": Confidence.EXTRACTED, + "conf_score": 1.0 if relation.dependency_kind != "sequential" else 0.95, + "layer": 1, + "evidence": ( + f"arazzo workflow {relation.workflow}: " + f"{relation.source_step} -> {relation.target_step} " + f"({relation.dependency_kind})" + ), + "kind": "data" if bindings else "workflow", + "evidence_sources": [EVIDENCE_ARAZZO], + "execution_direction": "source_to_target", + "data_flow": { + "workflow_id": relation.workflow, + "source_step_id": relation.source_step, + "target_step_id": relation.target_step, + "dependency_kind": relation.dependency_kind, + "from_operation": relation.source, + "to_operation": relation.target, + "parameters": bindings, + **_primary_binding(bindings), + }, + }, + default_source=EVIDENCE_ARAZZO, + ) + if graph.graph.has_edge(relation.source, relation.target): + existing = graph.graph.get_edge_attrs(relation.source, relation.target) + merged = merge_graph_edges( + {"source": relation.source, "target": relation.target, **existing}, + incoming, + ) + merged["execution_direction"] = "source_to_target" + _put_edge(graph, relation.source, relation.target, merged) + stats["merged"] += 1 + else: + _put_edge(graph, relation.source, relation.target, incoming) + stats["added"] += 1 + for binding in bindings: + stats["binding_aliases_added"] += _promote_binding_alias( + graph, + producer=relation.source, + binding=binding, + relation=relation, + ) + return stats + + +def _promote_binding_alias( + graph: ToolGraph, + *, + producer: str, + binding: dict[str, Any], + relation: ArazzoRelation, +) -> int: + field_name = str(binding.get("target_field") or "").strip() + json_path = str(binding.get("source_path") or "").strip() + if not field_name or not json_path or producer not in graph.tools: + return 0 + tool = graph.tools[producer] + metadata = tool.metadata if isinstance(tool.metadata, dict) else {} + tool.metadata = metadata + produces = metadata.setdefault("produces", []) + if not isinstance(produces, list): + produces = [] + metadata["produces"] = produces + if any( + isinstance(row, dict) + and str(row.get("field_name") or "") == field_name + and str(row.get("json_path") or "") == json_path + for row in produces + ): + return 0 + consumer_contract = _consumer_contract_row(graph, relation.target, field_name) + # Explicit Arazzo runtime bindings outrank generic OpenAPI leaf paths for + # the same consumer field. PathSynthesizer preserves produces order when + # resolving a field, so keep the workflow alias first. + produces.insert( + 0, + { + "field_name": field_name, + "json_path": json_path, + "field_type": str(consumer_contract.get("field_type") or "string"), + "required": False, + "kind": "data", + "search_signal": False, + "contract_source": EVIDENCE_ARAZZO, + "arazzo_workflow_id": relation.workflow, + "arazzo_source_step_id": relation.source_step, + "arazzo_source_output": binding.get("source_output"), + **( + {"semantic_tag": consumer_contract["semantic_tag"]} + if consumer_contract.get("semantic_tag") + else {} + ), + }, + ) + return 1 + + +def _consumer_contract_row(graph: ToolGraph, consumer: str, field_name: str) -> dict[str, Any]: + tool = graph.tools.get(consumer) + if tool is None: + return {} + metadata = tool.metadata if isinstance(tool.metadata, dict) else {} + for row in metadata.get("consumes") or []: + if isinstance(row, dict) and str(row.get("field_name") or "") == field_name: + return row + return {} + + +def _operation_resolver(graph: ToolGraph): + operation_ids: dict[str, str] = {} + operation_refs: dict[str, str] = {} + for name, tool in graph.tools.items(): + metadata = tool.metadata if isinstance(tool.metadata, dict) else {} + openapi = metadata.get("openapi") if isinstance(metadata.get("openapi"), dict) else {} + operation_id = str(openapi.get("operation_id") or name).strip() + if operation_id and operation_id not in operation_ids: + operation_ids[operation_id] = name + path = str(metadata.get("path") or openapi.get("path") or "").strip() + method = str(metadata.get("method") or openapi.get("method") or "").strip().lower() + if path and method: + escaped = path.replace("~", "~0").replace("/", "~1") + operation_refs[f"#/paths/{escaped}/{method}"] = name + + def resolve(step: dict[str, Any]) -> str | None: + operation_id = str(step.get("operationId") or "").strip() + if operation_id.startswith("$sourceDescriptions."): + operation_id = operation_id.rsplit(".", 1)[-1] + if operation_id: + return operation_ids.get(operation_id, operation_id) + operation_path = str(step.get("operationPath") or "").strip() + if "#" in operation_path: + operation_path = "#" + operation_path.split("#", 1)[1] + return operation_refs.get(operation_path) + + return resolve + + +def _primary_binding(bindings: list[dict[str, Any]]) -> dict[str, Any]: + if not bindings: + return {} + binding = bindings[0] + return { + "from_path": binding.get("source_path") or "", + "from_field": binding.get("source_output") or "", + "to_field": binding.get("target_field") or "", + } + + +def _put_edge(graph: ToolGraph, source: str, target: str, edge: dict[str, Any]) -> None: + graph.graph.add_edge( + source, + target, + **{key: value for key, value in edge.items() if key not in {"source", "target"}}, + ) + + +def _requested_sources( + source: dict[str, Any] | str | Sequence[dict[str, Any] | str], +) -> list[dict[str, Any] | str]: + if isinstance(source, (dict, str)): + return [source] + values = list(source) + if not values: + raise ValueError("workflow sources must not be empty") + if not all(isinstance(item, (dict, str)) for item in values): + raise TypeError("workflow sources must contain only Arazzo dicts or string sources") + return values + + +def _safe_source_label(source: str) -> str: + value = str(source or "") + parsed = urlsplit(value) + if parsed.scheme in {"http", "https"} and parsed.hostname: + host = parsed.hostname + if parsed.port: + host = f"{host}:{parsed.port}" + return urlunsplit((parsed.scheme, host, parsed.path, "", "")) + return Path(value).name + + +__all__ = ["apply_arazzo_relations", "apply_arazzo_workflows"] diff --git a/graph_tool_call/ingest/arazzo.py b/graph_tool_call/ingest/arazzo.py index adcb1ff..d80bee6 100644 --- a/graph_tool_call/ingest/arazzo.py +++ b/graph_tool_call/ingest/arazzo.py @@ -1,8 +1,10 @@ -"""Ingest Arazzo 1.0.0 workflow specifications into tool graph relations.""" +"""Ingest Arazzo workflow descriptions into tool graph relations.""" from __future__ import annotations import json +import re +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from typing import Any @@ -30,6 +32,17 @@ class ArazzoRelation: target: str # operationId that depends on source workflow: str # workflow name relation_type: RelationType = RelationType.PRECEDES + source_step: str = "" + target_step: str = "" + dependency_kind: str = "sequential" + bindings: tuple[dict[str, Any], ...] = () + + +_STEP_OUTPUT_RE = re.compile(r"\$steps\.([A-Za-z0-9_-]+)\.outputs\.([A-Za-z0-9._-]+)(#[^\s\"'}]+)?") +_WORKFLOW_STEP_OUTPUT_RE = re.compile( + r"\$workflows\.([A-Za-z0-9_-]+)\.steps\.([A-Za-z0-9_-]+)\.outputs\." + r"([A-Za-z0-9._-]+)(#[^\s\"'}]+)?" +) # --------------------------------------------------------------------------- @@ -57,20 +70,31 @@ def _load_spec( allow_private_hosts=allow_private_hosts, max_response_bytes=max_response_bytes, ) - return json.loads(text) + return _parse_spec_text(text) path = Path(source) text = path.read_text(encoding="utf-8") - if path.suffix in (".yaml", ".yml"): - if not _HAS_YAML: - raise ImportError( - "PyYAML is required for YAML files. " - "Install with: pip install graph-tool-call[openapi]" - ) - return yaml.safe_load(text) - - return json.loads(text) + return _parse_spec_text(text, prefer_yaml=path.suffix in (".yaml", ".yml")) + + +def _parse_spec_text(text: str, *, prefer_yaml: bool = False) -> dict[str, Any]: + if not prefer_yaml: + try: + value = json.loads(text) + except json.JSONDecodeError: + value = None + if isinstance(value, dict): + return value + if not _HAS_YAML: + raise ImportError( + "PyYAML is required for YAML Arazzo descriptions. " + "Install with: pip install graph-tool-call[openapi]" + ) + value = yaml.safe_load(text) + if not isinstance(value, dict): + raise ValueError("Arazzo description must be a JSON/YAML object") + return value # --------------------------------------------------------------------------- @@ -78,11 +102,19 @@ def _load_spec( # --------------------------------------------------------------------------- -def _extract_operation_id(step: dict[str, Any]) -> str | None: +def _extract_operation_id( + step: dict[str, Any], + operation_resolver: Callable[[dict[str, Any]], str | None] | None = None, +) -> str | None: """Extract operationId from an Arazzo step.""" + if operation_resolver is not None and (resolved := operation_resolver(step)): + return resolved # Direct operationId reference if "operationId" in step: - return step["operationId"] + operation_id = str(step["operationId"] or "").strip() + if operation_id.startswith("$sourceDescriptions."): + return operation_id.rsplit(".", 1)[-1] + return operation_id or None # operationPath format: "{sourceDescription}#{jsonPointer}" or just operationId if "operationPath" in step: op_path = step["operationPath"] @@ -99,8 +131,13 @@ def ingest_arazzo( registered_tools: set[str] | None = None, allow_private_hosts: bool = False, max_response_bytes: int = 5_000_000, + operation_resolver: Callable[[dict[str, Any]], str | None] | None = None, ) -> list[ArazzoRelation]: - """Parse an Arazzo 1.0.0 spec and extract workflow step dependencies as PRECEDES relations. + """Parse an Arazzo spec and extract workflow dependencies as PRECEDES relations. + + Arazzo 1.0 and 1.1 ordering constructs are accepted. Explicit + ``dependsOn`` and runtime output references outrank implicit sequential + step order when they describe the same operation pair. Parameters ---------- @@ -122,68 +159,304 @@ def ingest_arazzo( ) relations: list[ArazzoRelation] = [] - workflows = spec.get("workflows", []) + workflows = [row for row in spec.get("workflows", []) if isinstance(row, dict)] + workflow_steps: dict[str, dict[str, dict[str, Any]]] = {} + workflow_order: dict[str, list[str]] = {} for workflow in workflows: - wf_name = workflow.get("workflowId", "unknown") - steps = workflow.get("steps", []) - - # Build step_id → operationId mapping - step_ops: dict[str, str] = {} - for step in steps: - step_id = step.get("stepId", "") - op_id = _extract_operation_id(step) - if step_id and op_id: - step_ops[step_id] = op_id - - # Extract dependsOn → PRECEDES relations - for step in steps: - step_id = step.get("stepId", "") - target_op = step_ops.get(step_id) - if not target_op: + workflow_id = str(workflow.get("workflowId") or "unknown") + step_rows: dict[str, dict[str, Any]] = {} + ordered_ids: list[str] = [] + for step in workflow.get("steps") or []: + if not isinstance(step, dict): continue + step_id = str(step.get("stepId") or "").strip() + operation_id = _extract_operation_id(step, operation_resolver) + if not step_id or not operation_id: + continue + step_rows[step_id] = { + "step": step, + "operation_id": operation_id, + "outputs": step.get("outputs") if isinstance(step.get("outputs"), dict) else {}, + } + ordered_ids.append(step_id) + workflow_steps[workflow_id] = step_rows + workflow_order[workflow_id] = ordered_ids - depends_on = step.get("dependsOn", []) - for dep_step_id in depends_on: - source_op = step_ops.get(dep_step_id) - if not source_op: - continue - if source_op == target_op: - continue - - # Filter by registered tools if provided - if registered_tools is not None: - if source_op not in registered_tools or target_op not in registered_tools: - continue - - relations.append( - ArazzoRelation( - source=source_op, - target=target_op, - workflow=wf_name, + for workflow in workflows: + workflow_id = str(workflow.get("workflowId") or "unknown") + step_rows = workflow_steps.get(workflow_id, {}) + ordered_ids = workflow_order.get(workflow_id, []) + pending: dict[tuple[str, str], dict[str, Any]] = {} + + for target_step_id in ordered_ids: + target_row = step_rows[target_step_id] + target_step = target_row["step"] + for dependency in target_step.get("dependsOn") or []: + source_ref = _resolve_step_reference( + str(dependency), + current_workflow=workflow_id, + workflow_steps=workflow_steps, + ) + if source_ref: + _record_dependency( + pending, + source_ref=source_ref, + target_ref=(workflow_id, target_step_id), + dependency_kind="depends_on", ) + + for binding in _runtime_step_bindings( + target_step, + current_workflow=workflow_id, + workflow_steps=workflow_steps, + ): + source_ref = (binding.pop("source_workflow"), binding["source_step_id"]) + _record_dependency( + pending, + source_ref=source_ref, + target_ref=(workflow_id, target_step_id), + dependency_kind="runtime_reference", + binding=binding, ) - # Sequential step ordering (implicit): each step PRECEDES the next - ordered_ops: list[str] = [] - for step in steps: - step_id = step.get("stepId", "") - op_id = step_ops.get(step_id) - if op_id: - ordered_ops.append(op_id) - - for i in range(len(ordered_ops) - 1): - src, tgt = ordered_ops[i], ordered_ops[i + 1] - if src == tgt: - continue - if registered_tools is not None: - if src not in registered_tools or tgt not in registered_tools: - continue + for source_step_id, target_step_id in zip(ordered_ids, ordered_ids[1:]): + _record_dependency( + pending, + source_ref=(workflow_id, source_step_id), + target_ref=(workflow_id, target_step_id), + dependency_kind="sequential", + ) - # Only add if not already captured by dependsOn - already = any( - r.source == src and r.target == tgt and r.workflow == wf_name for r in relations + for row in pending.values(): + source_workflow, source_step_id = row["source_ref"] + target_workflow, target_step_id = row["target_ref"] + source_row = workflow_steps.get(source_workflow, {}).get(source_step_id) + target_row = workflow_steps.get(target_workflow, {}).get(target_step_id) + if not source_row or not target_row: + continue + source_op = str(source_row["operation_id"]) + target_op = str(target_row["operation_id"]) + if source_op == target_op: + continue + if registered_tools is not None and ( + source_op not in registered_tools or target_op not in registered_tools + ): + continue + relations.append( + ArazzoRelation( + source=source_op, + target=target_op, + workflow=target_workflow, + source_step=source_step_id, + target_step=target_step_id, + dependency_kind=str(row["dependency_kind"]), + bindings=tuple(row["bindings"]), + ) ) - if not already: - relations.append(ArazzoRelation(source=src, target=tgt, workflow=wf_name)) return relations + + +def _record_dependency( + pending: dict[tuple[str, str], dict[str, Any]], + *, + source_ref: tuple[str, str], + target_ref: tuple[str, str], + dependency_kind: str, + binding: dict[str, Any] | None = None, +) -> None: + if source_ref == target_ref: + return + key = ("::".join(source_ref), "::".join(target_ref)) + row = pending.setdefault( + key, + { + "source_ref": source_ref, + "target_ref": target_ref, + "dependency_kind": dependency_kind, + "bindings": [], + }, + ) + rank = {"sequential": 1, "depends_on": 2, "runtime_reference": 3} + if rank.get(dependency_kind, 0) > rank.get(str(row["dependency_kind"]), 0): + row["dependency_kind"] = dependency_kind + if binding and binding not in row["bindings"]: + row["bindings"].append(binding) + + +def _resolve_step_reference( + value: str, + *, + current_workflow: str, + workflow_steps: dict[str, dict[str, dict[str, Any]]], +) -> tuple[str, str] | None: + text = value.strip() + if text in workflow_steps.get(current_workflow, {}): + return current_workflow, text + match = re.search(r"\$workflows\.([A-Za-z0-9_-]+)\.steps\.([A-Za-z0-9_-]+)", text) + if match and match.group(2) in workflow_steps.get(match.group(1), {}): + return match.group(1), match.group(2) + return None + + +def _runtime_step_bindings( + step: dict[str, Any], + *, + current_workflow: str, + workflow_steps: dict[str, dict[str, dict[str, Any]]], +) -> list[dict[str, Any]]: + bindings: list[dict[str, Any]] = [] + parameter_values = step.get("parameters") or [] + if isinstance(parameter_values, list): + for parameter in parameter_values: + if not isinstance(parameter, dict): + continue + _collect_runtime_bindings( + parameter.get("value"), + target_field=str(parameter.get("name") or ""), + target_location=str(parameter.get("in") or "parameter"), + current_workflow=current_workflow, + workflow_steps=workflow_steps, + bindings=bindings, + ) + request_body = step.get("requestBody") + if isinstance(request_body, dict): + _walk_request_body_bindings( + request_body, + path="requestBody", + current_workflow=current_workflow, + workflow_steps=workflow_steps, + bindings=bindings, + ) + return bindings + + +def _walk_request_body_bindings( + value: Any, + *, + path: str, + current_workflow: str, + workflow_steps: dict[str, dict[str, dict[str, Any]]], + bindings: list[dict[str, Any]], +) -> None: + if isinstance(value, dict): + for key, item in value.items(): + _walk_request_body_bindings( + item, + path=f"{path}.{key}", + current_workflow=current_workflow, + workflow_steps=workflow_steps, + bindings=bindings, + ) + return + if isinstance(value, list): + for index, item in enumerate(value): + _walk_request_body_bindings( + item, + path=f"{path}[{index}]", + current_workflow=current_workflow, + workflow_steps=workflow_steps, + bindings=bindings, + ) + return + _collect_runtime_bindings( + value, + target_field=_request_body_target_field(path), + target_location="request_body", + target_path=path, + current_workflow=current_workflow, + workflow_steps=workflow_steps, + bindings=bindings, + ) + + +def _collect_runtime_bindings( + value: Any, + *, + target_field: str, + target_location: str, + target_path: str = "", + current_workflow: str, + workflow_steps: dict[str, dict[str, dict[str, Any]]], + bindings: list[dict[str, Any]], +) -> None: + if not isinstance(value, str): + return + matches: list[tuple[str, str, str, str, str]] = [] + matches.extend( + ( + current_workflow, + match.group(1), + match.group(2), + match.group(3) or "", + match.group(0), + ) + for match in _STEP_OUTPUT_RE.finditer(value) + ) + matches.extend( + ( + match.group(1), + match.group(2), + match.group(3), + match.group(4) or "", + match.group(0), + ) + for match in _WORKFLOW_STEP_OUTPUT_RE.finditer(value) + ) + for source_workflow, source_step_id, source_output, suffix, expression in matches: + source_row = workflow_steps.get(source_workflow, {}).get(source_step_id) + if not source_row: + continue + output_expression = source_row["outputs"].get(source_output) + source_path = _response_expression_path(output_expression, suffix=suffix) + binding = { + "source_workflow": source_workflow, + "source_step_id": source_step_id, + "source_output": source_output, + "source_path": source_path, + "target_field": target_field, + "target_location": target_location, + "expression": expression, + **({"target_path": target_path} if target_path else {}), + } + if binding not in bindings: + bindings.append(binding) + + +def _request_body_target_field(path: str) -> str: + segments = [segment for segment in re.split(r"\.|\[\d+\]", path) if segment] + for segment in reversed(segments): + if segment not in {"requestBody", "payload", "contentType", "replacements"}: + return segment + return "" + + +def _response_expression_path(value: Any, *, suffix: str = "") -> str: + expression = str(value or "").strip() + if expression.startswith("$response.body"): + base = _json_pointer_to_json_path(expression.removeprefix("$response.body")) + elif expression.startswith("$response.header."): + base = f"$.headers.{expression.removeprefix('$response.header.')}" + elif expression.startswith("$message.payload"): + payload_path = _json_pointer_to_json_path(expression.removeprefix("$message.payload")) + base = "$.payload" + (payload_path[1:] if payload_path != "$" else "") + else: + base = "" + if suffix and base: + suffix_path = _json_pointer_to_json_path(suffix) + if suffix_path != "$": + base += suffix_path[1:] + return base + + +def _json_pointer_to_json_path(pointer: str) -> str: + value = str(pointer or "") + if value.startswith("#"): + value = value[1:] + if not value: + return "$" + path = "$" + for segment in value.lstrip("/").split("/"): + decoded = segment.replace("~1", "/").replace("~0", "~") + path += f"[{decoded}]" if decoded.isdigit() else f".{decoded}" + return path diff --git a/graph_tool_call/plan/synthesizer.py b/graph_tool_call/plan/synthesizer.py index bf4f149..95f0a89 100644 --- a/graph_tool_call/plan/synthesizer.py +++ b/graph_tool_call/plan/synthesizer.py @@ -713,9 +713,13 @@ def _index_workflow_edges(self, graph: dict[str, Any]) -> None: ).lower() if not src or not tgt or rel_str not in self._WORKFLOW_RELATIONS: continue - self._workflow_edges_out.setdefault(src, []).append( + # REQUIRES is stored consumer -> producer. PRECEDES follows + # execution order (producer -> consumer), so invert it for the + # planner's consumer -> producer dependency index. + planning_source, planning_target = (tgt, src) if rel_str == "precedes" else (src, tgt) + self._workflow_edges_out.setdefault(planning_source, []).append( { - "target": tgt, + "target": planning_target, "relation": rel_str, "confidence": e.get("confidence"), "conf_score": float(e.get("conf_score") or 0.0), diff --git a/graph_tool_call/tool_graph.py b/graph_tool_call/tool_graph.py index 780cdb6..879df83 100644 --- a/graph_tool_call/tool_graph.py +++ b/graph_tool_call/tool_graph.py @@ -501,7 +501,7 @@ def ingest_arazzo( allow_private_hosts: bool = False, max_response_bytes: int = 5_000_000, ) -> list: - """Ingest an Arazzo 1.0.0 workflow spec, adding PRECEDES relations. + """Ingest an Arazzo workflow spec, adding PRECEDES relations. Only adds relations between tools already registered in the graph. @@ -518,8 +518,9 @@ def ingest_arazzo( allow_private_hosts=allow_private_hosts, max_response_bytes=max_response_bytes, ) - for rel in relations: - self._builder.add_relation(rel.source, rel.target, rel.relation_type) + from graph_tool_call.graphify.workflow_evidence import apply_arazzo_relations + + apply_arazzo_relations(self, relations) self._invalidate_retrieval() return relations diff --git a/pyproject.toml b/pyproject.toml index 7f6eaae..f7e19df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "graph-tool-call" -version = "0.40.0" +version = "0.41.0" description = "Graph-structured tool retrieval for LLM agents — zero-dependency, ontology-aware hybrid search" authors = ["SonAIengine"] license = "MIT" diff --git a/tests/test_graphify_collection_artifact.py b/tests/test_graphify_collection_artifact.py index 5079ef3..55c6047 100644 --- a/tests/test_graphify_collection_artifact.py +++ b/tests/test_graphify_collection_artifact.py @@ -10,6 +10,7 @@ COLLECTION_GRAPH_VERSION, build_openapi_collection_artifact, ) +from graph_tool_call.plan import PathSynthesizer def _collection_spec() -> dict: @@ -116,6 +117,11 @@ def test_build_openapi_collection_artifact_is_loadable_and_preserves_build_evide assert len(artifact["source_snapshot_manifest"]["specs"][0]["sha256"]) == 64 assert artifact["ingest_summary"]["registered_tool_count"] == 2 assert artifact["edge_stats"]["tool_count"] == 2 + assert artifact["edge_stats"]["arazzo_workflows"] == { + "added": 0, + "merged": 0, + "binding_aliases_added": 0, + } assert artifact["semantic_summary"]["canonical_action_known_rate"] == 1.0 assert artifact["semantic_summary"]["primary_resource_assigned_rate"] == 1.0 assert artifact["edge_quality_summary"]["total"] == len(artifact["graph"]["edges"]) @@ -157,10 +163,142 @@ def test_build_openapi_collection_artifact_dedupes_multiple_sources() -> None: assert artifact["ingest_summary"]["duplicate_tool_count"] == 2 +def test_build_openapi_collection_artifact_applies_arazzo_order_and_binding() -> None: + workflow = { + "arazzo": "1.1.0", + "info": {"title": "Product workflow", "version": "1.0.0"}, + "sourceDescriptions": [], + "workflows": [ + { + "workflowId": "createProductFlow", + "steps": [ + { + "stepId": "brands", + "operationId": "listBrands", + "outputs": {"brandId": "$response.body#/data/items/0/brandNo"}, + }, + { + "stepId": "create", + "operationId": "createProduct", + "parameters": [ + { + "name": "brandNo", + "in": "body", + "value": "$steps.brands.outputs.brandId", + } + ], + }, + ], + } + ], + } + + artifact = build_openapi_collection_artifact( + _collection_spec(), + workflow_sources=workflow, + context_field_names={"siteNo"}, + ) + + summary = artifact["workflow_summary"] + assert summary["source_count"] == 1 + assert summary["workflow_count"] == 1 + assert summary["relation_count"] == 1 + assert summary["by_dependency_kind"] == {"runtime_reference": 1} + assert summary["edge_stats"] == { + "added": 1, + "merged": 0, + "binding_aliases_added": 1, + } + assert artifact["metadata"]["workflow_summary"] == summary + assert artifact["metadata"]["build_options"]["workflow_source_count"] == 1 + assert artifact["edge_quality_summary"]["workflow"] == 1 + edge = next( + edge + for edge in artifact["graph"]["edges"] + if edge["source"] == "listBrands" and edge["target"] == "createProduct" + ) + assert edge["relation"] == "precedes" + assert edge["evidence_sources"] == ["arazzo"] + assert edge["data_flow"]["from_path"] == "$.data.items[0].brandNo" + assert edge["data_flow"]["to_field"] == "brandNo" + aliases = artifact["tools"]["listBrands"]["metadata"]["produces"] + assert any( + row.get("field_name") == "brandNo" + and row.get("json_path") == "$.data.items[0].brandNo" + and row.get("contract_source") == "arazzo" + for row in aliases + ) + + plan = PathSynthesizer(artifact).synthesize( + target="createProduct", + goal="상품 등록", + entities={"productName": "Example", "siteNo": "1"}, + ) + + assert [step.tool for step in plan.steps] == ["listBrands", "createProduct"] + assert plan.steps[-1].args["brandNo"] == "${s1.data.items[0].brandNo}" + + +def test_build_openapi_collection_artifact_resolves_arazzo_operation_path() -> None: + workflow = { + "arazzo": "1.1.0", + "info": {"title": "Path workflow", "version": "1.0.0"}, + "sourceDescriptions": [], + "workflows": [ + { + "workflowId": "pathFlow", + "steps": [ + { + "stepId": "brands", + "operationPath": ("{$sourceDescriptions.api.url}#/paths/~1brands/get"), + }, + { + "stepId": "create", + "operationPath": ("{$sourceDescriptions.api.url}#/paths/~1products/post"), + }, + ], + } + ], + } + + artifact = build_openapi_collection_artifact( + _collection_spec(), + workflow_sources=workflow, + ) + + assert artifact["workflow_summary"]["relation_count"] == 1 + assert any( + edge["source"] == "listBrands" + and edge["target"] == "createProduct" + and edge["evidence_sources"] == ["arazzo"] + for edge in artifact["graph"]["edges"] + ) + + def test_build_openapi_collection_cli_writes_artifact(tmp_path: Path) -> None: spec_path = tmp_path / "openapi.json" + workflow_path = tmp_path / "arazzo.json" artifact_path = tmp_path / "collection.json" spec_path.write_text(json.dumps(_collection_spec()), encoding="utf-8") + workflow_path.write_text( + json.dumps( + { + "arazzo": "1.1.0", + "info": {"title": "CLI workflow", "version": "1.0.0"}, + "sourceDescriptions": [], + "workflows": [ + { + "workflowId": "create-product", + "steps": [ + {"stepId": "brands", "operationId": "listBrands"}, + {"stepId": "create", "operationId": "createProduct"}, + ], + } + ], + } + ), + encoding="utf-8", + ) result = subprocess.run( [ @@ -171,6 +309,8 @@ def test_build_openapi_collection_cli_writes_artifact(tmp_path: Path) -> None: str(spec_path), "-o", str(artifact_path), + "--workflow", + str(workflow_path), "--context-field", "siteNo", ], @@ -185,3 +325,4 @@ def test_build_openapi_collection_cli_writes_artifact(tmp_path: Path) -> None: assert payload["metadata"]["build_options"]["context_field_names"] == ["siteNo"] assert payload["metadata"]["build_options"]["derive_semantic_metadata"] is True assert payload["semantic_summary"]["canonical_action_known_rate"] == 1.0 + assert payload["workflow_summary"]["relation_count"] == 1 diff --git a/tests/test_graphify_workflow_evidence.py b/tests/test_graphify_workflow_evidence.py new file mode 100644 index 0000000..de990b2 --- /dev/null +++ b/tests/test_graphify_workflow_evidence.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from unittest.mock import patch + +from graph_tool_call import ToolGraph +from graph_tool_call.core.tool import ToolSchema +from graph_tool_call.graphify import apply_arazzo_workflows + + +def _workflow_spec() -> dict: + return { + "arazzo": "1.1.0", + "info": {"title": "Order flow", "version": "1.0.0"}, + "sourceDescriptions": [], + "workflows": [ + { + "workflowId": "read-order", + "steps": [ + {"stepId": "list", "operationId": "listOrders"}, + {"stepId": "read", "operationId": "getOrder"}, + ], + } + ], + } + + +def test_remote_source_manifest_drops_credentials_query_and_fragment() -> None: + graph = ToolGraph() + graph.add_tool(ToolSchema(name="listOrders")) + graph.add_tool(ToolSchema(name="getOrder")) + source = "https://user:password@example.com/flow.yaml?token=secret#fragment" + + with patch( + "graph_tool_call.graphify.workflow_evidence._load_spec", + return_value=_workflow_spec(), + ): + summary = apply_arazzo_workflows(graph, source) + + manifest = summary["source_snapshot_manifest"]["specs"][0] + assert manifest["source"] == "https://example.com/flow.yaml" + assert "password" not in str(summary) + assert "secret" not in str(summary) + assert graph.graph.has_edge("listOrders", "getOrder") diff --git a/tests/test_ingest_arazzo.py b/tests/test_ingest_arazzo.py index fcc1436..67c5130 100644 --- a/tests/test_ingest_arazzo.py +++ b/tests/test_ingest_arazzo.py @@ -125,6 +125,153 @@ def test_sequential_ordering(self): assert any(r.source == "op1" and r.target == "op2" for r in relations) assert any(r.source == "op2" and r.target == "op3" for r in relations) + def test_runtime_output_reference_is_implicit_dependency_with_binding(self): + spec = { + "arazzo": "1.1.0", + "info": {"title": "Runtime binding", "version": "1.0.0"}, + "sourceDescriptions": [], + "workflows": [ + { + "workflowId": "profileFlow", + "steps": [ + { + "stepId": "createStep", + "operationId": "createProfile", + "outputs": {"profileId": "$response.body#/id"}, + }, + { + "stepId": "readStep", + "operationId": "getProfile", + "parameters": [ + { + "name": "userId", + "in": "path", + "value": "$steps.createStep.outputs.profileId", + } + ], + }, + ], + } + ], + } + + relations = ingest_arazzo(spec) + + assert len(relations) == 1 + relation = relations[0] + assert relation.source == "createProfile" + assert relation.target == "getProfile" + assert relation.dependency_kind == "runtime_reference" + assert relation.bindings == ( + { + "source_step_id": "createStep", + "source_output": "profileId", + "source_path": "$.id", + "target_field": "userId", + "target_location": "path", + "expression": "$steps.createStep.outputs.profileId", + }, + ) + + def test_qualified_operation_id_uses_registered_operation_tail(self): + spec = { + "arazzo": "1.1.0", + "info": {"title": "Qualified operations", "version": "1.0.0"}, + "sourceDescriptions": [], + "workflows": [ + { + "workflowId": "flow", + "steps": [ + { + "stepId": "first", + "operationId": "$sourceDescriptions.store.createItem", + }, + { + "stepId": "second", + "operationId": "$sourceDescriptions.store.getItem", + }, + ], + } + ], + } + + relations = ingest_arazzo(spec, registered_tools={"createItem", "getItem"}) + + assert [(row.source, row.target) for row in relations] == [("createItem", "getItem")] + + def test_nested_request_body_binding_preserves_leaf_and_target_path(self): + spec = { + "arazzo": "1.1.0", + "info": {"title": "Body binding", "version": "1.0.0"}, + "sourceDescriptions": [], + "workflows": [ + { + "workflowId": "bodyFlow", + "steps": [ + { + "stepId": "lookup", + "operationId": "lookupOrder", + "outputs": {"orderId": "$response.body#/items/0/id"}, + }, + { + "stepId": "submit", + "operationId": "submitOrder", + "requestBody": { + "contentType": "application/json", + "payload": { + "order": { + "orderId": "$steps.lookup.outputs.orderId", + } + }, + }, + }, + ], + } + ], + } + + relations = ingest_arazzo(spec) + + assert relations[0].bindings[0]["target_field"] == "orderId" + assert relations[0].bindings[0]["target_location"] == "request_body" + assert relations[0].bindings[0]["target_path"] == "requestBody.payload.order.orderId" + + def test_runtime_binding_does_not_persist_surrounding_literal_text(self): + spec = { + "arazzo": "1.1.0", + "info": {"title": "Safe binding", "version": "1.0.0"}, + "sourceDescriptions": [], + "workflows": [ + { + "workflowId": "safeFlow", + "steps": [ + { + "stepId": "lookup", + "operationId": "lookupOrder", + "outputs": {"orderId": "$response.body#/id"}, + }, + { + "stepId": "read", + "operationId": "readOrder", + "parameters": [ + { + "name": "orderId", + "in": "path", + "value": "Bearer $steps.lookup.outputs.orderId trailing-secret", + } + ], + }, + ], + } + ], + } + + relation = ingest_arazzo(spec)[0] + + assert relation.bindings[0]["expression"] == "$steps.lookup.outputs.orderId" + assert "Bearer" not in str(relation.bindings) + assert "trailing-secret" not in str(relation.bindings) + class TestIngestArazzoFile: def test_yaml_file(self): @@ -156,6 +303,32 @@ def test_private_host_allowed_with_opt_in(self): ) assert len(relations) >= 2 + def test_remote_yaml_is_supported(self): + pytest.importorskip("yaml") + yaml_text = """ +arazzo: 1.1.0 +info: {title: Remote, version: 1.0.0} +sourceDescriptions: [] +workflows: + - workflowId: remote + steps: + - {stepId: first, operationId: firstOperation} + - {stepId: second, operationId: secondOperation} +""" + mock_resp = MagicMock() + mock_resp.read.return_value = yaml_text.encode() + mock_resp.headers = {"Content-Type": "application/yaml"} + mock_resp.geturl.return_value = "https://example.com/arazzo.yaml" + mock_resp.__enter__ = lambda s: s + mock_resp.__exit__ = MagicMock(return_value=False) + + with patch("graph_tool_call.net._open_url", return_value=mock_resp): + relations = ingest_arazzo("https://example.com/arazzo.yaml") + + assert [(row.source, row.target) for row in relations] == [ + ("firstOperation", "secondOperation") + ] + class TestToolGraphIntegration: def test_ingest_arazzo(self): diff --git a/website/docs/guides/openapi-collections.md b/website/docs/guides/openapi-collections.md index 4d00059..8c37e02 100644 --- a/website/docs/guides/openapi-collections.md +++ b/website/docs/guides/openapi-collections.md @@ -87,6 +87,7 @@ from graph_tool_call.graphify import ( artifact = build_openapi_collection_artifact( "openapi.json", + workflow_sources=["arazzo.yaml"], derive_semantic_metadata=True, promote_contract_signals=True, context_field_names={"tenantId", "siteNo"}, @@ -129,6 +130,7 @@ dictionaries to the library. | `metadata` | build options, source identity, version, summary copies | | `semantic_summary` | action/resource/module/result-shape coverage | | `edge_quality_summary` | evidence distribution for graph edges | +| `workflow_summary` | Arazzo workflow, step, relation, and binding counts | | `readiness_report` | deterministic OpenAPI readiness diagnostics | | `source_snapshot_manifest` | source labels, URLs, hashes, operation counts | | `ingest_summary` | duplicate handling and operation counts | @@ -138,6 +140,27 @@ dictionaries to the library. Store the whole artifact. If you persist only `graph`, the product loses the diagnostics needed to explain readiness and rebuild behavior. +## Add Explicit Call Order + +OpenAPI describes operations and contracts but often does not describe the +business call order. Pass an Arazzo document when that evidence exists. + +```bash +graph-tool-call build-openapi-collection openapi.json \ + --workflow arazzo.yaml \ + -o collection.json +``` + +The engine merges `dependsOn`, sequential steps, and runtime output references +as `arazzo` evidence. A reference such as +`$steps.lookup.outputs.orderId` records both the execution order and the +response path used for the next request field. The artifact stores operation, +step, and binding metadata, not literal request values or credentials. + +Without Arazzo, the existing structural and contract inference remains +unchanged. Treat inferred edges as hypotheses and explicit Arazzo edges as +strong deterministic evidence. + ## Contract Index Use `extract_openapi_contract_index()` when an adapter needs operation-level diff --git a/website/i18n/ko/docusaurus-plugin-content-docs/current/guides/openapi-collections.md b/website/i18n/ko/docusaurus-plugin-content-docs/current/guides/openapi-collections.md index e9a9ffe..9f89316 100644 --- a/website/i18n/ko/docusaurus-plugin-content-docs/current/guides/openapi-collections.md +++ b/website/i18n/ko/docusaurus-plugin-content-docs/current/guides/openapi-collections.md @@ -85,6 +85,7 @@ from graph_tool_call.graphify import ( artifact = build_openapi_collection_artifact( "openapi.json", + workflow_sources=["arazzo.yaml"], derive_semantic_metadata=True, promote_contract_signals=True, context_field_names={"tenantId", "siteNo"}, @@ -127,6 +128,7 @@ library에 넣지 않습니다. | `metadata` | build option, source identity, version, summary copy | | `semantic_summary` | action/resource/module/result-shape coverage | | `edge_quality_summary` | graph edge의 evidence 분포 | +| `workflow_summary` | Arazzo workflow, step, relation, binding 수 | | `readiness_report` | deterministic OpenAPI readiness diagnostics | | `source_snapshot_manifest` | source label, URL, hash, operation count | | `ingest_summary` | duplicate 처리와 operation count | @@ -136,6 +138,25 @@ library에 넣지 않습니다. artifact 전체를 저장하는 것을 권장합니다. `graph`만 저장하면 readiness와 rebuild behavior를 설명하는 diagnostics를 잃게 됩니다. +## 명시적 호출 순서 추가 + +OpenAPI는 operation과 contract를 설명하지만 비즈니스 호출 순서는 생략하는 경우가 많습니다. +해당 근거가 있다면 Arazzo 문서를 함께 전달합니다. + +```bash +graph-tool-call build-openapi-collection openapi.json \ + --workflow arazzo.yaml \ + -o collection.json +``` + +엔진은 `dependsOn`, 순차 step, runtime output reference를 `arazzo` 근거로 병합합니다. +`$steps.lookup.outputs.orderId` 같은 참조는 실행 순서뿐 아니라 다음 요청 필드가 사용할 +response path도 기록합니다. artifact에는 operation, step, binding metadata만 저장하고 +literal request 값이나 credential은 저장하지 않습니다. + +Arazzo가 없으면 기존 structural/contract 추론은 그대로 동작합니다. 추론 edge는 가설로, +명시된 Arazzo edge는 강한 deterministic evidence로 취급합니다. + ## Contract Index Adapter가 full graph build 전이나 외부에서 operation-level fact를 확인해야 한다면