diff --git a/.api-sync/sync.py b/.api-sync/sync.py index df7d351..0ad3b61 100644 --- a/.api-sync/sync.py +++ b/.api-sync/sync.py @@ -42,10 +42,12 @@ import argparse import ast import json +import re +import subprocess import sys from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Optional +from typing import Any, Optional, Union ROOT = Path(__file__).resolve().parent.parent API_SYNC = Path(__file__).resolve().parent @@ -57,6 +59,7 @@ UNMODELED_PATH = API_SYNC / "unmodeled.json" SCALAR_TYPE_MAP = {"string": "str", "integer": "int", "number": "float", "boolean": "bool"} +HTTP_VERBS = ("get", "post", "put", "patch", "delete") # --------------------------------------------------------------------------- # @@ -89,10 +92,12 @@ def load_unmodeled() -> list[dict]: required = {"schema", "property", "reason", "owner"} elif kind == "nested_object": required = {"schema", "path", "reason", "owner"} + elif kind == "operation": + required = {"method", "path", "reason", "owner"} else: raise SystemExit( f"{UNMODELED_PATH}[{i}]: unknown or missing 'kind' " - f"(expected 'property', 'enum', 'enum_coverage', or 'nested_object')" + f"(expected 'property', 'enum', 'enum_coverage', 'nested_object', or 'operation')" ) missing = required - e.keys() if missing: @@ -783,11 +788,11 @@ def diff_removals_and_changes(old_spec: dict, new_spec: dict, map_data: dict, in ) ) - # new operations (paths present in new, absent from old) - old_paths = set(old_spec.get("paths", {}).keys()) - new_paths = set(new_spec.get("paths", {}).keys()) - for p in sorted(new_paths - old_paths): - problems.append(NeedsHuman("new_operation", f"new path: {p}")) + # New/uncovered operations are handled by reconcile_operations (state-based: + # it checks whether the CURRENT spec's operations have a matching SDK call + # site, not whether the path key is new relative to the old snapshot -- an + # operation can be "new" to the SDK without its path being new to the spec, + # e.g. right after a human deletes a method by hand). # new reachable schemas not present before mapped_or_ignored: set[str] = set() @@ -817,6 +822,503 @@ def diff_removals_and_changes(old_spec: dict, new_spec: dict, map_data: dict, in return problems +# --------------------------------------------------------------------------- # +# Operation coverage (state-based): does every spec operation have a matching +# SDK call site? Coverage is derived, never hand-maintained: an operation is +# "covered" iff some resource module already issues that exact (HTTP verb, +# path template) request. Uncovered operations are then classified as either +# STANDARD (deterministically generatable: plain JSON in/out, maps cleanly to +# an existing resource module by path prefix) or NON-STANDARD (needs-human, +# with a precise reason). +# --------------------------------------------------------------------------- # + + +def normalize_spec_path(path: str) -> str: + """Strips the "/v1" prefix (the SDK's base_url already carries it, so no + call-site f-string ever repeats it) and collapses every `{param}` segment + to a bare `{}`, matching the shape produced by `_fstring_template`.""" + if path.startswith("/v1/"): + path = path[3:] + elif path == "/v1": + path = "" + return re.sub(r"\{[^}]*\}", "{}", path) + + +def path_param_names(path: str) -> list[str]: + return re.findall(r"\{([^}]+)\}", path) + + +def _fstring_template(node: ast.expr) -> Optional[str]: + """Reduces a call-site path argument to a template with every interpolation + collapsed to `{}`, or None if it isn't a plain string/f-string literal (in + which case it cannot be reconciled against the spec and is simply not + counted as covering anything -- never mistaken for a match).""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.JoinedStr): + parts: list[str] = [] + for v in node.values: + if isinstance(v, ast.Constant) and isinstance(v.value, str): + parts.append(v.value) + elif isinstance(v, ast.FormattedValue): + parts.append("{}") + else: + return None + return "".join(parts) + return None + + +def _client_call_verb(call: ast.Call) -> Optional[str]: + f = call.func + if not isinstance(f, ast.Attribute) or f.attr not in HTTP_VERBS: + return None + obj = f.value + if isinstance(obj, ast.Attribute) and obj.attr == "_client" and isinstance(obj.value, ast.Name): + return f.attr if obj.value.id == "self" else None + return None + + +def _singularize(name: str) -> str: + return name[:-1] if name.endswith("s") and not name.endswith("ss") else name + + +@dataclass +class ResourceMethod: + name: str + verb: str + template: str # no "/v1" prefix, params collapsed to "{}" + + +@dataclass +class ResourceModule: + file: Path + async_class_name: str + sync_class_name: Optional[str] + factory_name: Optional[str] + factory_sync_name: Optional[str] + uses_instance_id: bool + methods: list[ResourceMethod] + prefix: str # singularized PascalCase, e.g. "PartnerFeesResource" -> "PartnerFee" + + +def discover_resource_modules(src_root: Path = SRC_ROOT) -> list[ResourceModule]: + """Derives, from the SDK source itself, which (verb, path template) each + resource module already implements. Deliberately not a hand-maintained + map (unlike spec-map.json's schema/enum entries, which need curation + because names diverge): call-site f-strings are a direct, mechanical + record of what a resource module actually does.""" + modules: list[ResourceModule] = [] + resources_dir = src_root / "resources" + if not resources_dir.exists(): + return modules + for f in sorted(resources_dir.rglob("*.py")): + if f.name == "__init__.py": + continue + tree = ast.parse(f.read_text(), filename=str(f)) + classes = {n.name: n for n in ast.walk(tree) if isinstance(n, ast.ClassDef)} + for name, async_cls in sorted(classes.items()): + if not name.endswith("Resource") or name.endswith("ResourceSync"): + continue + uses_instance_id = False + for stmt in async_cls.body: + if isinstance(stmt, ast.FunctionDef) and stmt.name == "__init__": + uses_instance_id = any(a.arg == "instance_id" for a in stmt.args.args) + methods: list[ResourceMethod] = [] + for stmt in async_cls.body: + if not isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)) or stmt.name == "__init__": + continue + for node in ast.walk(stmt): + if not isinstance(node, ast.Call): + continue + verb = _client_call_verb(node) + if verb is None or not node.args: + continue + tmpl = _fstring_template(node.args[0]) + if tmpl is None: + continue + tmpl = tmpl.split("?")[0] # literal query strings (e.g. ?rail=...) aren't part of the route + tmpl = re.sub(r"\{[^}]*\}", "{}", tmpl) + # A `{}` glued directly onto the previous character (no "/" before it) is an + # interpolated query-string suffix built at runtime (e.g. f"...{query_string}"), + # never a path segment -- drop it so it isn't mistaken for a trailing path param. + tmpl = re.sub(r"(? set[tuple[str, str]]: + return {(m.verb, m.template) for mod in modules for m in mod.methods} + + +def _collection_prefix(template: str) -> str: + """Strips trailing `/{}` path-param segments, e.g. "/x/{}/y/{}" -> "/x/{}/y".""" + while template.endswith("/{}"): + template = template[: -len("/{}")] + return template + + +def _item_prefix(template: str) -> Optional[str]: + """Everything up to and including the LAST path param, e.g. + "/x/{}/y/{}/secret" -> "/x/{}/y/{}" -- this is what lets a GET whose path + is "item param, then a literal sub-action" (".../{id}/secret") match a + sibling method on the same item (".../{id}" DELETE). Only meaningful with + 2+ params: with exactly one (just the resource's own instance-id), this + would collapse to a generic "/instances/{}" shared by nearly every + resource and defeat the whole point of prefix matching.""" + if template.count("{}") < 2: + return None + idx = template.rfind("{}") + return template[: idx + 2] + + +def find_matching_module(template: str, modules: list[ResourceModule]) -> Union[ResourceModule, str, None]: + """The resource module whose own call templates already share this + operation's collection prefix (or, failing that, its item prefix -- + see `_item_prefix`), `"ambiguous"` if more than one module qualifies, + or None if none do.""" + target_collection = _collection_prefix(template) + target_item = _item_prefix(template) + matches = [] + for mod in modules: + collections = {_collection_prefix(m.template) for m in mod.methods} + items = {_item_prefix(m.template) for m in mod.methods} - {None} + if target_collection in collections or (target_item is not None and target_item in items): + matches.append(mod) + if not matches: + return None + if len(matches) > 1: + return "ambiguous" + return matches[0] + + +class SynthesisError(Exception): + def __init__(self, reason: str): + self.reason = reason + super().__init__(reason) + + +def _lookup_type_map(map_data: dict, spec_schema: str) -> Optional[dict]: + """A named (`$ref`'d) schema is reusable only if spec-map.json already maps + it 1:1 to a single SDK symbol (no specPath, no fan-out) -- anything else + (a sub-object path, or a fan-out across several SDK variants) is a + judgment call this function refuses to make silently.""" + for e in map_data.get("types", []): + spec = e["spec"] if isinstance(e["spec"], list) else [e["spec"]] + if spec == [spec_schema] and not e.get("specPath") and len(e["sdk"]) == 1: + return e["sdk"][0] + return None + + +def _lookup_enum_map(map_data: dict, schema: str, prop: str, is_items: bool) -> Optional[dict]: + for e in map_data.get("enums", []): + loc = e["spec"] + if loc["schema"] == schema and loc["property"] == prop and bool(loc.get("items")) == is_items: + return e["sdk"] + return None + + +@dataclass +class FieldSpec: + name: str + annotation: str + reused_symbol: Optional[dict] = None # {"file": ..., "symbol": ...} if importing an existing type + + +def synth_field( + field_name: str, prop_schema: dict, required: bool, named_schema: Optional[str], map_data: dict +) -> FieldSpec: + """Synthesizes one TypedDict field from a spec property, reusing an + existing mapped SDK symbol for any `$ref` or enum-constrained property + (never inventing a new nested type or Literal -- that naming/placement + call stays with a human) and SCALAR_TYPE_MAP for everything else. Raises + SynthesisError with a precise, human-readable reason on anything it + cannot express.""" + _, nullable = coarse_type(prop_schema) + reused: Optional[dict] = None + + ref = prop_schema.get("$ref") + if isinstance(ref, str) and ref.startswith("#/components/schemas/"): + ref_name = ref.split("/")[-1] + site = _lookup_type_map(map_data, ref_name) + if site is None: + raise SynthesisError(f"property `{field_name}` references schema `{ref_name}` with no spec-map entry") + py_type = site["symbol"] + reused = site + elif find_enum_locator(prop_schema) is not None: + if named_schema is None: + raise SynthesisError(f"property `{field_name}` is enum-constrained on an inline (unnamed) schema") + is_items, _ = find_enum_locator(prop_schema) + site = _lookup_enum_map(map_data, named_schema, field_name, is_items) + if site is None: + raise SynthesisError(f"property `{field_name}` is enum-constrained with no mapped Literal in spec-map.json") + py_type = f"List[{site['symbol']}]" if is_items else site["symbol"] + reused = site + elif prop_schema.get("type") == "array" or ( + isinstance(prop_schema.get("type"), list) and "array" in prop_schema["type"] + ): + items = prop_schema.get("items", {}) if isinstance(prop_schema.get("items"), dict) else {} + item_ref = items.get("$ref") + if isinstance(item_ref, str) and item_ref.startswith("#/components/schemas/"): + ref_name = item_ref.split("/")[-1] + site = _lookup_type_map(map_data, ref_name) + if site is None: + raise SynthesisError(f"property `{field_name}` is an array of `{ref_name}` with no spec-map entry") + py_type = f"List[{site['symbol']}]" + reused = site + else: + t0, _ = coarse_type(items) + scalar = SCALAR_TYPE_MAP.get(t0) if t0 else None + if scalar is None: + raise SynthesisError(f"property `{field_name}` is an array of a type the synthesizer cannot express") + py_type = f"List[{scalar}]" + else: + t0, _ = coarse_type(prop_schema) + if t0 is None: + raise SynthesisError(f"property `{field_name}` has an ambiguous or multi-type spec shape") + py_type = SCALAR_TYPE_MAP.get(t0) + if py_type is None: + raise SynthesisError(f"property `{field_name}` has spec type `{t0}`, which the synthesizer cannot express") + + if nullable: + py_type = f"Optional[{py_type}]" + if not required: + py_type = f"NotRequired[{py_type}]" + return FieldSpec(name=field_name, annotation=py_type, reused_symbol=reused) + + +@dataclass +class BodySpec: + type_name: str + fields: list[FieldSpec] + named_schema: Optional[str] # the $ref'd schema name, so a spec-map entry can be added for it + + +def resolve_json_content(container: dict, ctx: str) -> Optional[dict]: + """Returns the `application/json` schema dict for a requestBody/response + object, or None if there is no content at all. Raises SynthesisError if + any *other* content type is present (multipart, binary, streaming, ...).""" + content = container.get("content") + if not content: + return None + other = sorted(set(content.keys()) - {"application/json"}) + if other: + raise SynthesisError(f"{ctx} uses unsupported content type(s): {other}") + return content.get("application/json", {}).get("schema") + + +def resolve_body_spec( + schema: Optional[dict], spec: dict, map_data: dict, type_name: str, ctx: str +) -> Optional[BodySpec]: + """Resolves a requestBody/response JSON schema (inline object, `$ref` to a + named object schema, or None) into the fields for a brand-new TypedDict + named `type_name`. Anything else a plain object -- oneOf/anyOf, a + top-level array, a bare scalar -- is out of scope for STANDARD generation + and raises SynthesisError with the exact shape that defeated it.""" + if schema is None: + return None + named_schema: Optional[str] = None + node = schema + if "$ref" in schema: + ref = schema["$ref"] + if not ref.startswith("#/components/schemas/"): + raise SynthesisError(f"{ctx}: unsupported $ref `{ref}`") + named_schema = ref.split("/")[-1] + node = get_schema(spec, named_schema) + if node is None: + raise SynthesisError(f"{ctx}: $ref `{named_schema}` does not resolve to a component schema") + if "oneOf" in node or "anyOf" in node: + kind = "oneOf" if "oneOf" in node else "anyOf" + raise SynthesisError(f"{ctx}: schema uses {kind}, which is unsupported") + if node.get("type") not in (None, "object") or "properties" not in node: + raise SynthesisError(f"{ctx}: schema is not a plain JSON object") + required = get_required(node) + fields = [ + synth_field(name, prop, name in required, named_schema, map_data) for name, prop in get_properties(node).items() + ] + return BodySpec(type_name=type_name, fields=fields, named_schema=named_schema) + + +@dataclass +class OperationInsertPlan: + path: str + verb: str + module: ResourceModule + method_name: str + path_args: list[str] # extra path params (instance_id already excluded), in call order + input_type_name: Optional[str] # None if this operation has no request body + input_spec: Optional[BodySpec] # non-None iff a brand-new TypedDict must be generated for it + output_type_name: str # "None" (as a literal type, for BlindpayApiResponse[None]) if no response body + output_spec: Optional[BodySpec] + + +def derive_method_and_output_name( + verb: str, template: str, prefix: str, drop_leading_param: bool +) -> tuple[str, bool, Optional[str]]: + """Returns (method_name, is_list, output_type_base_name). output_type_base + already has the "Get"/"List" prefix and "Response" suffix applied where + the verb implies a response type name; None means "no synthesized name + needed yet" (POST/PUT/PATCH/DELETE decide their own below). + + `drop_leading_param` discards the resource's own instance-id placeholder + (e.g. the `{}` in "/instances/{}/transfers") before reading off the last + segment(s) -- otherwise a plain collection GET right under the instance + (".../transfers") is misread as a single-item fetch, because the + instance-id placeholder immediately precedes it.""" + if verb == "get": + segments = [s for s in template.split("/") if s] + if drop_leading_param: + for i, s in enumerate(segments): + if s == "{}": + del segments[i] + break + if not segments or segments[-1] != "{}": + if len(segments) >= 2 and segments[-2] == "{}": + suffix = "".join(p.capitalize() for p in re.split(r"[-_]", segments[-1])) + return f"get_{segments[-1].replace('-', '_')}", False, f"Get{prefix}{suffix}Response" + return "list", True, f"List{prefix}sResponse" + return "get", False, f"Get{prefix}Response" + if verb == "post": + return "create", False, f"Create{prefix}Response" + if verb in ("put", "patch"): + return "update", False, f"Update{prefix}Response" + if verb == "delete": + return "delete", False, None + raise AssertionError(verb) + + +def classify_operation( + path: str, verb: str, op: dict, spec: dict, modules: list[ResourceModule], map_data: dict +) -> Union[OperationInsertPlan, NeedsHuman]: + norm = normalize_spec_path(path) + ctx = f"{verb.upper()} {path}" + match = find_matching_module(norm, modules) + if match is None: + return NeedsHuman("needs_human_operation", f"{ctx}: no existing resource matches this path's prefix") + if match == "ambiguous": + return NeedsHuman("needs_human_operation", f"{ctx}: more than one resource module matches this path's prefix") + module = match + assert isinstance(module, ResourceModule) + + all_params = path_param_names(path) + path_args = all_params[1:] if module.uses_instance_id and all_params else all_params + + method_name, is_list, output_name = derive_method_and_output_name( + verb, norm, module.prefix, drop_leading_param=module.uses_instance_id + ) + if any(m.name == method_name for m in module.methods): + return NeedsHuman( + "needs_human_operation", + f"{ctx}: derived method name `{method_name}` already exists on {module.async_class_name}", + ) + + try: + response_schema = None + for status, resp in sorted(op.get("responses", {}).items()): + if status.startswith("2") and isinstance(resp, dict): + response_schema = resolve_json_content(resp, f"{ctx} response {status}") + break + + output_spec: Optional[BodySpec] = None + if response_schema is None: + output_type_name = "None" + elif response_schema.get("type") == "array": + items = response_schema.get("items", {}) if isinstance(response_schema.get("items"), dict) else {} + item_ref = items.get("$ref") + if isinstance(item_ref, str) and item_ref.startswith("#/components/schemas/"): + ref_name = item_ref.split("/")[-1] + site = _lookup_type_map(map_data, ref_name) + if site is None: + raise SynthesisError(f"response is an array of `{ref_name}` with no spec-map entry") + output_type_name = f"List[{site['symbol']}]" + else: + t0, _ = coarse_type(items) + scalar = SCALAR_TYPE_MAP.get(t0) if t0 else None + if scalar is None: + raise SynthesisError("response is a top-level array of a type the synthesizer cannot express") + output_type_name = f"List[{scalar}]" + else: + output_type_name = output_name or f"{module.prefix}Response" + output_spec = resolve_body_spec(response_schema, spec, map_data, output_type_name, f"{ctx} response") + + input_type_name: Optional[str] = None + input_spec: Optional[BodySpec] = None + request_body = op.get("requestBody") + if request_body: + req_schema = resolve_json_content(request_body, f"{ctx} request body") + input_type_name = f"{'Create' if verb == 'post' else 'Update'}{module.prefix}Input" + input_spec = resolve_body_spec(req_schema, spec, map_data, input_type_name, f"{ctx} request body") + except SynthesisError as e: + return NeedsHuman("needs_human_operation", f"{ctx}: {e.reason}") + + return OperationInsertPlan( + path=path, + verb=verb, + module=module, + method_name=method_name, + path_args=path_args, + input_type_name=input_type_name, + input_spec=input_spec, + output_type_name=output_type_name, + output_spec=output_spec, + ) + + +def unmodeled_operation_allowed(unmodeled: list[dict], verb: str, path: str) -> bool: + return any(e.get("kind") == "operation" and e["method"].lower() == verb and e["path"] == path for e in unmodeled) + + +def reconcile_operations( + spec: dict, + map_data: dict, + unmodeled: Optional[list[dict]] = None, + modules: Optional[list[ResourceModule]] = None, +) -> tuple[list[OperationInsertPlan], list[NeedsHuman]]: + if modules is None: + modules = discover_resource_modules() + unmodeled = unmodeled or [] + covered = covered_operations(modules) + plans: list[OperationInsertPlan] = [] + problems: list[NeedsHuman] = [] + for path, item in sorted(spec.get("paths", {}).items()): + if not isinstance(item, dict): + continue + for verb, op in sorted(item.items()): + if verb not in HTTP_VERBS or not isinstance(op, dict): + continue + if (verb, normalize_spec_path(path)) in covered: + continue + if unmodeled_operation_allowed(unmodeled, verb, path): + continue + result = classify_operation(path, verb, op, spec, modules, map_data) + if isinstance(result, OperationInsertPlan): + plans.append(result) + else: + problems.append(result) + return plans, problems + + # --------------------------------------------------------------------------- # # Applying changes (text splicing) # --------------------------------------------------------------------------- # @@ -1019,6 +1521,122 @@ def _reparse_class(file_path: Path, symbol: str) -> ast.ClassDef: raise SystemExit(f"internal error: lost track of class {symbol} in {file_path} after edit") +# --------------------------------------------------------------------------- # +# Applying operation-insert changes +# --------------------------------------------------------------------------- # + + +def render_typeddict(body: BodySpec) -> str: + lines = [f"class {body.type_name}(TypedDict):"] + for f in body.fields: + lines.append(f" {f.name}: {f.annotation}") + if not body.fields: + lines.append(" pass") + return "\n".join(lines) + "\n" + + +def _find_class(source: str, name: str) -> ast.ClassDef: + tree = ast.parse(source) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == name: + return node + raise SystemExit(f"internal error: class {name} not found while applying an operation-insert") + + +def insert_typeddicts_before_class(source: str, class_name: str, blocks: list[str]) -> str: + if not blocks: + return source + class_node = _find_class(source, class_name) + lines = source.splitlines(keepends=True) + text = "\n\n".join(b.rstrip("\n") for b in blocks) + "\n\n\n" + lines[class_node.lineno - 1 : class_node.lineno - 1] = [text] + return "".join(lines) + + +def append_method(source: str, class_name: str, method_source: str) -> str: + class_node = _find_class(source, class_name) + last_stmt = class_node.body[-1] + last_line_no = last_stmt.end_lineno + assert last_line_no is not None + lines = source.splitlines(keepends=True) + indented = "\n".join(f" {ln}" if ln else ln for ln in method_source.rstrip("\n").split("\n")) + lines.insert(last_line_no, "\n" + indented + "\n") + return "".join(lines) + + +def build_path_fstring(path: str, module: ResourceModule) -> str: + path_wo_v1 = path[3:] if path.startswith("/v1/") else path + names = path_param_names(path) + if module.uses_instance_id and names: + path_wo_v1 = path_wo_v1.replace("{" + names[0] + "}", "{self._instance_id}", 1) + return path_wo_v1 + + +def render_method(plan: OperationInsertPlan, is_sync: bool) -> str: + args = [f"{a}: str" for a in plan.path_args] + if plan.input_type_name: + args.append(f"data: {plan.input_type_name}") + arg_str = ", ".join(["self", *args]) + fstring = build_path_fstring(plan.path, plan.module) + call_args = f'f"{fstring}"' + (", data" if plan.input_type_name else "") + awaited = "" if is_sync else "await " + async_kw = "" if is_sync else "async " + client_call = f"self._client.{plan.verb}({call_args})" + return ( + f"{async_kw}def {plan.method_name}({arg_str}) -> BlindpayApiResponse[{plan.output_type_name}]:\n" + f" return {awaited}{client_call}\n" + ) + + +def apply_operation_insert(plan: OperationInsertPlan, map_data: dict) -> AppliedChange: + module = plan.module + file_path = module.file + source = file_path.read_text() + + new_blocks = [render_typeddict(spec) for spec in (plan.input_spec, plan.output_spec) if spec is not None] + source = insert_typeddicts_before_class(source, module.async_class_name, new_blocks) + + source = append_method(source, module.async_class_name, render_method(plan, is_sync=False)) + if module.sync_class_name: + source = append_method(source, module.sync_class_name, render_method(plan, is_sync=True)) + + needed_wrappers = { + kw + for spec in (plan.input_spec, plan.output_spec) + if spec + for f in spec.fields + for kw in ("NotRequired", "Optional") + if f"{kw}[" in f.annotation + } + for kw in sorted(needed_wrappers): + source = ensure_name_imported(source, kw) + + for spec in (plan.input_spec, plan.output_spec): + if spec and spec.named_schema: + map_data.setdefault("types", []).append( + { + "spec": spec.named_schema, + "sdk": [{"file": str(file_path.relative_to(ROOT)), "symbol": spec.type_name}], + } + ) + + file_path.write_text(source) + try: + subprocess.run([sys.executable, "-m", "ruff", "format", "--quiet", str(file_path)], check=False) + subprocess.run( + [sys.executable, "-m", "ruff", "check", "--fix", "--quiet", "--select=I", str(file_path)], check=False + ) + except FileNotFoundError: + pass # ruff not available in this environment; the generated source is still valid Python + + return AppliedChange( + kind="operation-insert", + file=str(file_path.relative_to(ROOT)), + symbol=f"{module.async_class_name}.{plan.method_name}", + detail=f"added {plan.verb.upper()} {plan.path} as {module.async_class_name}.{plan.method_name}()", + ) + + # --------------------------------------------------------------------------- # # Coverage report (non-blocking) # --------------------------------------------------------------------------- # @@ -1098,6 +1716,11 @@ def cmd_check(report_path: Optional[Path]) -> int: prop_gaps = reconcile_types(spec, map_data, unmodeled, index) if not map_errors else [] enum_coverage_gaps = reconcile_enum_coverage(spec, map_data, unmodeled, index) if not map_errors else [] nested_gaps = reconcile_nested_coverage(spec, map_data, unmodeled) if not map_errors else [] + op_plans, op_needs_human = ( + reconcile_operations(spec, map_data, unmodeled, discover_resource_modules(SRC_ROOT)) + if not map_errors + else ([], []) + ) report = { "mode": "check", @@ -1110,17 +1733,26 @@ def cmd_check(report_path: Optional[Path]) -> int: {"schema": g.schema, "path": g.path, "property": g.property} for g in enum_coverage_gaps ], "nested_object_gaps": [{"schema": g.schema, "path": g.path} for g in nested_gaps], + "operation_gaps": [f"{p.verb.upper()} {p.path}" for p in op_plans], + "operation_needs_human": [{"kind": n.kind, "detail": n.detail} for n in op_needs_human], } if report_path: report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") - if not (map_errors or enum_gaps or prop_gaps or enum_coverage_gaps or nested_gaps): + if not (map_errors or enum_gaps or prop_gaps or enum_coverage_gaps or nested_gaps or op_plans or op_needs_human): return 0 if map_errors: print("spec-map.json validity -- FAILED:") for e in sorted(map_errors): print(f" {e}") + for p in op_plans: + print( + f"PENDING DRIFT (operation-insert): {p.verb.upper()} {p.path} has no SDK method yet and is " + f"auto-generatable; run --apply to add {p.module.async_class_name}.{p.method_name}()." + ) + for n in op_needs_human: + print(f"NEEDS_HUMAN (operation): {n.detail}") for g in enum_gaps: print( f"PENDING DRIFT (enum): {g.symbol} in {g.file} is missing {g.missing}; add to the Literal " @@ -1168,6 +1800,8 @@ def cmd_apply(spec_path: Path, report_path: Optional[Path]) -> int: enum_gaps = reconcile_enums(new_spec, map_data, unmodeled, index) prop_gaps = reconcile_types(new_spec, map_data, unmodeled, index) + op_plans, op_needs_human = reconcile_operations(new_spec, map_data, unmodeled, discover_resource_modules(SRC_ROOT)) + needs_human.extend(op_needs_human) applied: list[AppliedChange] = [] @@ -1179,10 +1813,14 @@ def cmd_apply(spec_path: Path, report_path: Optional[Path]) -> int: a, nh = apply_property_change(g, index) applied.extend(a) needs_human.extend(nh) + for p in sorted(op_plans, key=lambda p: (p.verb, p.path)): + applied.append(apply_operation_insert(p, map_data)) + if op_plans: + MAP_PATH.write_text(json.dumps(map_data, indent=2) + "\n") bump: Optional[str] = None if not needs_human: - if any(a.kind == "enum" for a in applied): + if any(a.kind in ("enum", "operation-insert") for a in applied): bump = "minor" elif applied: bump = "patch" diff --git a/.api-sync/unmodeled.json b/.api-sync/unmodeled.json index 3726ccc..17931f0 100644 --- a/.api-sync/unmodeled.json +++ b/.api-sync/unmodeled.json @@ -1984,5 +1984,33 @@ "path": "us", "reason": "Found by the new nested-object coverage check: VirtualAccountOut#us is an inline object/array-item shape reachable under a mapped schema. It is already modeled by the USBankDetails TypedDict, but that mapping is not registered as a specPath map entry, so this check cannot verify field parity. Needs a human pass to add the proper specPath map entry (or entries, if the shape recurs across multiple host schemas) rather than a blind mechanical add, since the right host TypedDict per specPath needs manual confirmation.", "owner": "eric@blindpay.com" + }, + { + "kind": "operation", + "method": "get", + "path": "/v1/instances/{instance_id}/rfi", + "reason": "Found by the new operation-coverage check: no resource module implements any RFI (request-for-information) endpoint yet, so there is no existing path prefix an auto-insert can attach to. Needs a human to design the RfiResource shape (module, class names, TypedDicts) before this and its sibling RFI operations can be added.", + "owner": "eric@blindpay.com" + }, + { + "kind": "operation", + "method": "post", + "path": "/v1/instances/{instance_id}/rfi", + "reason": "Same pre-existing gap as GET /v1/instances/{instance_id}/rfi above.", + "owner": "eric@blindpay.com" + }, + { + "kind": "operation", + "method": "get", + "path": "/v1/instances/{instance_id}/customers/{customer_id}/rfi", + "reason": "Same pre-existing gap as GET /v1/instances/{instance_id}/rfi above.", + "owner": "eric@blindpay.com" + }, + { + "kind": "operation", + "method": "post", + "path": "/v1/instances/{instance_id}/customers/{customer_id}/rfi", + "reason": "Same pre-existing gap as GET /v1/instances/{instance_id}/rfi above.", + "owner": "eric@blindpay.com" } ] diff --git a/tests/test_api_sync.py b/tests/test_api_sync.py index 9e55471..1312fd7 100644 --- a/tests/test_api_sync.py +++ b/tests/test_api_sync.py @@ -231,7 +231,9 @@ class in a file that imports Optional from `typing` but has never ], "ignore": {"schemas": []}, } - _write_map_and_unmodeled(repo, map_data) + # "/t" has no resource module behind it in this synthetic fixture -- suppress + # it as a known operation gap so this test stays focused on property splicing. + _write_map_and_unmodeled(repo, map_data, _op_unmodeled(("post", "/t"))) old_spec = base_schema( { "ThingIn": { @@ -300,6 +302,17 @@ def _write_map_and_unmodeled(root: Path, map_data: dict[str, Any], unmodeled: Op write(root, ".api-sync/unmodeled.json", json.dumps(unmodeled or [])) +def _op_unmodeled(*pairs: tuple[str, str]) -> list[dict[str, str]]: + """kind=operation unmodeled entries for (method, path) pairs. These fixture + repos' resource modules are TypedDict-only stand-ins with no real resource + class behind them, so their spec paths would otherwise show up as + operation-coverage gaps unrelated to whatever the test actually exercises.""" + return [ + {"kind": "operation", "method": m, "path": p, "reason": "test fixture has no resource module", "owner": "t"} + for m, p in pairs + ] + + class TestReconciliation: def test_enum_gap_detected_when_spec_has_extra_member(self, repo: Path): _write_fixture_repo(repo) @@ -926,14 +939,19 @@ def test_ambiguous_type_metadata_only_change_is_deliberately_compatible(self, re problems = sync.diff_removals_and_changes(old, new, self._map(), index) assert problems == [] - def test_new_operation_is_needs_human(self, repo: Path): + def test_new_operation_with_no_matching_resource_is_needs_human(self, repo: Path): + """New/uncovered operations are no longer caught by diffing path keys + between old and new specs (an operation can be "new" to the SDK + without its path key being new to the spec, e.g. right after a human + deletes a method by hand) -- that's reconcile_operations' job now, + checked against whatever spec is current, state-based like every + other reconcile_* check.""" _write_fixture_repo(repo) sync = load_sync(repo) - index = sync.build_sdk_index(sync.SRC_ROOT) - old = base_schema({}, {}) - new = base_schema({}, {"/new-path": {"get": op()}}) - problems = sync.diff_removals_and_changes(old, new, self._map(), index) - assert any(p.kind == "new_operation" and "/new-path" in p.detail for p in problems) + spec = base_schema({}, {"/new-path": {"get": op()}}) + plans, problems = sync.reconcile_operations(spec, self._map()) + assert plans == [] + assert any(p.kind == "needs_human_operation" and "/new-path" in p.detail for p in problems) def test_new_unmapped_schema_is_needs_human(self, repo: Path): _write_fixture_repo(repo) @@ -1031,7 +1049,7 @@ def _setup(self, repo: Path): "types": [{"spec": "PlainOut", "sdk": [{"file": "src/blindpay/resources/sample.py", "symbol": "Plain"}]}], "ignore": {"schemas": []}, } - _write_map_and_unmodeled(repo, map_data) + _write_map_and_unmodeled(repo, map_data, _op_unmodeled(("get", "/c"), ("get", "/p"))) old_spec = base_schema( { "ColorOut": {"properties": {"color": {"type": "string", "enum": ["red", "blue"]}}}, @@ -1154,7 +1172,7 @@ def build(root: Path) -> None: ], "ignore": {"schemas": []}, } - _write_map_and_unmodeled(root, map_data) + _write_map_and_unmodeled(root, map_data, _op_unmodeled(("get", "/c"), ("get", "/p"))) old_spec = base_schema( { "ColorOut": {"properties": {"color": {"type": "string", "enum": ["red", "blue"]}}}, diff --git a/tests/test_api_sync_golden.py b/tests/test_api_sync_golden.py new file mode 100644 index 0000000..89fd5ce --- /dev/null +++ b/tests/test_api_sync_golden.py @@ -0,0 +1,260 @@ +"""Golden self-test for the operation-insert generator in .api-sync/sync.py. + +Unlike test_api_sync.py (which never touches the real src/blindpay tree, only +synthetic fixture repos), this test deliberately operates on a scratch COPY of +the real repo: it deletes two real, currently-existing SDK methods (one GET, +one POST-with-body) along with their TypedDicts and spec-map entries, runs the +patcher with --apply against the repo's own committed spec snapshot, and +asserts the regenerated method's route, HTTP verb, and types match what was +deleted. It also runs pyright/mypy/pytest against the regenerated tree, and +checks that a second --apply is a no-op (idempotent). + +This is slow (spins up a second uv-managed virtualenv) and is skipped unless +`uv` is on PATH. +""" + +from __future__ import annotations + +import importlib.util +import json +import shutil +import subprocess +import sys +import types +from pathlib import Path +from typing import Any + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +SYNC_PATH = REPO_ROOT / ".api-sync" / "sync.py" + +pytestmark = pytest.mark.skipif(shutil.which("uv") is None, reason="uv is required to build the golden scratch repo") + +_load_counter = 0 + + +def load_sync(root: Path) -> types.ModuleType: + global _load_counter + _load_counter += 1 + module_name = f"sync_golden_under_test_{_load_counter}" + spec = importlib.util.spec_from_file_location(module_name, SYNC_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + module.ROOT = root # type: ignore[attr-defined] + module.API_SYNC = root / ".api-sync" # type: ignore[attr-defined] + module.SRC_ROOT = root / "src" / "blindpay" # type: ignore[attr-defined] + module.SNAPSHOT_PATH = module.API_SYNC / "spec-snapshot.json" # type: ignore[attr-defined] + module.DEFAULT_SPEC_PATH = module.API_SYNC / "spec-current.json" # type: ignore[attr-defined] + module.MAP_PATH = module.API_SYNC / "spec-map.json" # type: ignore[attr-defined] + module.UNMODELED_PATH = module.API_SYNC / "unmodeled.json" # type: ignore[attr-defined] + return module + + +def build_scratch_repo(dest: Path) -> None: + """Copies just enough of the real repo to run the patcher and its gauntlet + against: source, tests, and the packaging/tooling config the gauntlet + needs (pyproject.toml for pyright/ruff config, uv.lock for a fast, + reproducible `uv sync`).""" + for rel in ( + "src", + "tests", + ".api-sync", + "pyproject.toml", + "uv.lock", + "pyrightconfig.json", + "pytest.ini", + "py.typed", + "README.md", + "LICENSE", + ): + src = REPO_ROOT / rel + dst = dest / rel + if src.is_dir(): + # This file itself must never be copied in: the golden repo's own + # `uv run pytest` (invoked below by test_regenerated_tree_passes_pytest) + # would otherwise discover and re-run it, rebuilding another golden + # repo inside itself, recursively, forever. + shutil.copytree(src, dst, ignore=shutil.ignore_patterns("test_api_sync_golden.py")) + else: + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + # --apply reads --spec, never spec-current.json's *default* unless told to; + # give it a spec identical to the committed snapshot so the two real + # deletions below are the only drift it needs to reconcile. + shutil.copy2(dest / ".api-sync" / "spec-snapshot.json", dest / ".api-sync" / "spec-current.json") + + +def delete_get_secret(webhooks_py: Path) -> None: + """Deletes WebhookEndpointsResource(Sync).get_secret() -- a GET with a + single path param and an inline (unnamed) response schema -- and its + TypedDict.""" + src = webhooks_py.read_text() + for pattern in ( + "\n async def get_secret(self, id: str) -> BlindpayApiResponse[GetWebhookEndpointSecretResponse]:\n" + ' return await self._client.get(f"/instances/{self._instance_id}/webhook-endpoints/{id}/secret")\n', + "\n def get_secret(self, id: str) -> BlindpayApiResponse[GetWebhookEndpointSecretResponse]:\n" + ' return self._client.get(f"/instances/{self._instance_id}/webhook-endpoints/{id}/secret")\n', + "\n\nclass GetWebhookEndpointSecretResponse(TypedDict):\n key: str\n", + ): + assert pattern in src, f"fixture assumption broken, pattern not found:\n{pattern}" + src = src.replace(pattern, "\n" if pattern.startswith("\n ") else "") + webhooks_py.write_text(src) + + +def delete_create(webhooks_py: Path) -> None: + """Deletes WebhookEndpointsResource(Sync).create() -- a POST with a + request body and response, both $ref'd to named, spec-mapped schemas -- + and its TypedDicts.""" + src = webhooks_py.read_text() + for pattern in ( + "\n async def create(self, data: CreateWebhookEndpointInput) -> " + "BlindpayApiResponse[CreateWebhookEndpointResponse]:\n" + ' return await self._client.post(f"/instances/{self._instance_id}/webhook-endpoints", data)\n', + "\n def create(self, data: CreateWebhookEndpointInput) -> " + "BlindpayApiResponse[CreateWebhookEndpointResponse]:\n" + ' return self._client.post(f"/instances/{self._instance_id}/webhook-endpoints", data)\n', + "\n\nclass CreateWebhookEndpointInput(TypedDict):\n url: str\n events: List[WebhookEvents]\n", + "\n\nclass CreateWebhookEndpointResponse(TypedDict):\n id: str\n", + ): + assert pattern in src, f"fixture assumption broken, pattern not found:\n{pattern}" + src = src.replace(pattern, "\n" if pattern.startswith("\n ") else "") + webhooks_py.write_text(src) + + +def remove_spec_map_entries(map_path: Path, *spec_schemas: str) -> None: + data = json.loads(map_path.read_text()) + data["types"] = [e for e in data["types"] if e["spec"] not in spec_schemas] + map_path.write_text(json.dumps(data, indent=2) + "\n") + + +def run(cmd: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + + +@pytest.fixture(scope="module") +def golden_repo(tmp_path_factory: pytest.TempPathFactory) -> Path: + root = tmp_path_factory.mktemp("api-sync-golden") + build_scratch_repo(root) + + webhooks_py = root / "src/blindpay/resources/webhooks/webhooks.py" + delete_get_secret(webhooks_py) + delete_create(webhooks_py) + remove_spec_map_entries(root / ".api-sync/spec-map.json", "WebhookEndpointIn", "WebhookEndpointOut") + + sync_up_front = load_sync(root) + # Sanity: with both methods gone, --check must now report drift (point 5 + # of the brief: a pending, un-applied operation-insert fails --check). + assert sync_up_front.cmd_check(None) == 1 + + sync_setup = load_sync(root) + apply_result = sync_setup.cmd_apply(sync_setup.DEFAULT_SPEC_PATH, root / "apply-report.json") + assert apply_result == 0, "the patcher refused to regenerate the two deleted methods" + + sync_env = run(["uv", "sync", "--group", "dev", "--group", "test"], cwd=root) + assert sync_env.returncode == 0, sync_env.stderr + + return root + + +class TestGoldenRegeneration: + def test_apply_regenerated_both_deleted_methods(self, golden_repo: Path) -> None: + report = json.loads((golden_repo / "apply-report.json").read_text()) + assert report["bump"] == "minor" + symbols = {a["symbol"] for a in report["applied"]} + assert "WebhookEndpointsResource.get_secret" in symbols + assert "WebhookEndpointsResource.create" in symbols + + def test_regenerated_get_secret_matches_original_route_verb_and_types(self, golden_repo: Path) -> None: + src = (golden_repo / "src/blindpay/resources/webhooks/webhooks.py").read_text() + assert "async def get_secret(self, id: str) -> BlindpayApiResponse[GetWebhookEndpointSecretResponse]:" in src + assert 'self._client.get(f"/instances/{self._instance_id}/webhook-endpoints/{id}/secret")' in src + assert "class GetWebhookEndpointSecretResponse(TypedDict):" in src + assert " key: str" in src + + def test_regenerated_create_matches_original_route_verb_and_types(self, golden_repo: Path) -> None: + src = (golden_repo / "src/blindpay/resources/webhooks/webhooks.py").read_text() + assert ( + "async def create(self, data: CreateWebhookEndpointInput) -> " + "BlindpayApiResponse[CreateWebhookEndpointResponse]:" in src + ) + assert 'self._client.post(f"/instances/{self._instance_id}/webhook-endpoints", data)' in src + assert "class CreateWebhookEndpointInput(TypedDict):" in src + assert " url: str" in src + assert " events: List[WebhookEvents]" in src + assert "class CreateWebhookEndpointResponse(TypedDict):" in src + assert " id: str" in src + + def test_regenerated_type_map_entries_were_restored(self, golden_repo: Path) -> None: + map_data: dict[str, Any] = json.loads((golden_repo / ".api-sync/spec-map.json").read_text()) + mapped = {e["spec"]: e["sdk"][0]["symbol"] for e in map_data["types"] if isinstance(e["spec"], str)} + assert mapped.get("WebhookEndpointIn") == "CreateWebhookEndpointInput" + assert mapped.get("WebhookEndpointOut") == "CreateWebhookEndpointResponse" + + def test_regenerated_tree_passes_pyright(self, golden_repo: Path) -> None: + result = run(["uv", "run", "pyright"], cwd=golden_repo) + assert result.returncode == 0, result.stdout + result.stderr + + def test_regenerated_tree_passes_mypy(self, golden_repo: Path) -> None: + result = run(["uv", "run", "mypy", "."], cwd=golden_repo) + assert result.returncode == 0, result.stdout + result.stderr + + def test_regenerated_tree_passes_pytest(self, golden_repo: Path) -> None: + result = run(["uv", "run", "pytest", "-q"], cwd=golden_repo) + assert result.returncode == 0, result.stdout + result.stderr + + def test_second_apply_is_idempotent(self, golden_repo: Path) -> None: + before = { + p: (golden_repo / p).read_bytes() + for p in ("src/blindpay/resources/webhooks/webhooks.py", ".api-sync/spec-map.json") + } + sync = load_sync(golden_repo) + report_path = golden_repo / "second-apply-report.json" + assert sync.cmd_apply(sync.DEFAULT_SPEC_PATH, report_path) == 0 + report = json.loads(report_path.read_text()) + assert report["applied"] == [] + assert report["bump"] is None + for rel, content in before.items(): + assert (golden_repo / rel).read_bytes() == content, f"{rel} changed on a second, no-op apply" + + def test_check_is_green_after_apply(self, golden_repo: Path) -> None: + sync = load_sync(golden_repo) + assert sync.cmd_check(None) == 0 + + +class TestMultipartFixtureRoutesToNeedsHuman: + """A synthetic fixture operation with a multipart/form-data request body + must be routed to needs-human with a reason that specifically names the + unsupported content type -- never silently skipped, never misclassified + as a STANDARD operation-insert.""" + + def test_multipart_request_body_is_needs_human_with_precise_reason(self, golden_repo: Path) -> None: + sync = load_sync(golden_repo) + map_data = json.loads((golden_repo / ".api-sync/spec-map.json").read_text()) + # Two path params ("instance_id", "id") so this resolves to + # WebhookEndpointsResource by item-prefix match (shared with its + # sibling `delete(id)`), exactly like a real new sub-action would -- + # the point of this fixture is that content-type, not resource + # matching or method-name collision, is what must send it to + # needs-human. PUT (-> "update") is used rather than POST (-> "create") + # because this same golden_repo already has a generated create(). + spec: dict[str, Any] = { + "paths": { + "/v1/instances/{instance_id}/webhook-endpoints/{id}/attachment": { + "put": { + "requestBody": { + "content": {"multipart/form-data": {"schema": {"type": "object", "properties": {}}}} + }, + "responses": {"200": {"content": {}}}, + } + } + }, + "components": {"schemas": {}}, + } + plans, problems = sync.reconcile_operations(spec, map_data) + assert plans == [] + assert len(problems) == 1 + assert problems[0].kind == "needs_human_operation" + assert "multipart/form-data" in problems[0].detail