From 864ab145d255092f66a9bfe0c371a63c3f04254e Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 14:40:51 +0200 Subject: [PATCH 01/55] docs: KERN-IL/0.2 deterministic compiler design and walkthrough Design for replacing model-generated IL with deterministic AST/tree-sitter compilation: tiered emitter (measured 6.5x at L2 on 3.7k-line file), per-symbol source-map handles, verify verb for latent-fault trapping. Co-Authored-By: Claude Fable 5 --- docs/deterministic-compiler.md | 378 ++++++++++++++++++ ...-07-11-deterministic-il-compiler-design.md | 166 ++++++++ 2 files changed, 544 insertions(+) create mode 100644 docs/deterministic-compiler.md create mode 100644 docs/superpowers/specs/2026-07-11-deterministic-il-compiler-design.md diff --git a/docs/deterministic-compiler.md b/docs/deterministic-compiler.md new file mode 100644 index 0000000..f516bea --- /dev/null +++ b/docs/deterministic-compiler.md @@ -0,0 +1,378 @@ +# Deterministic compiler walkthrough (KERN-IL/0.2 proposal) + +This document walks one real file through every stage of the proposed deterministic +compiler, showing the artifact **before and after** each step. No model is involved +at any stage: the same source bytes always produce the same IL bytes. + +Status: design proposal. The Python frontend upgrades the existing +`python-ast-baseline` in `skills/kern/scripts/kern_cache.py`; other languages move +from the regex line-filter fallback to tree-sitter. + +--- + +## Step 0 — the input (authoritative source) + +`src/cache_loader.py`, 45 lines, ~343 tokens: + +```python +"""Entry loader with hash verification for the KERN cache.""" + +from __future__ import annotations + +import json +import re +from hashlib import sha256 +from pathlib import Path + +MANIFEST_NAME = "manifest.json" +ENTRY_PATTERN = re.compile(r"^[a-z0-9_/]+\.kern-il\.txt$") + + +class StaleSource(Exception): + """Raised when the on-disk source no longer matches the expected hash.""" + + +def load_entry(path: Path, expected_sha: str) -> dict: + """Read a cache entry, verify its hash, and parse it as JSON.""" + data = path.read_bytes() + current_sha = sha256(data).hexdigest() + if current_sha != expected_sha: + raise StaleSource(path) + return json.loads(data) + + +def find_entries(root: Path) -> list[Path]: + """Return every manifest-listed entry under the cache root.""" + manifest = load_entry(root / MANIFEST_NAME, read_expected(root)) + entries = [] + for name, record in manifest["files"].items(): + if not ENTRY_PATTERN.match(name): + continue + candidate = root / "ir" / name + if candidate.is_file(): + entries.append(candidate) + return entries + + +def read_expected(root: Path) -> str: + """Read the pinned manifest hash written by the last scan.""" + pin = (root / ".pin").read_text(encoding="utf-8").strip() + if len(pin) != 64: + raise ValueError(f"corrupt pin file in {root}") + return pin +``` + +--- + +## Step 1 — parse: source → syntax tree + +The file goes through a real parser, never a regex. + +| Language | Parser | Dependency | +|---|---|---| +| Python | `ast` (standard library) | none — already used today | +| TypeScript, JavaScript, Go, Rust, Java, … | tree-sitter grammar | `pip install tree-sitter tree-sitter-` | +| Parser unavailable or syntax error | current generic line baseline | none (fallback, clearly labeled) | + +**Before:** raw text. +**After:** a typed tree. For `load_entry` the parser produces (abridged): + +```text +FunctionDef name=load_entry lineno=18 end_lineno=24 +├── args: [path: Path, expected_sha: str] returns: dict +├── Assign data = Call(path.read_bytes) +├── Assign current_sha = Call(sha256(data).hexdigest) +├── If Compare(current_sha != expected_sha) +│ └── Raise Call(StaleSource, [path]) +└── Return Call(json.loads, [data]) +``` + +Every node carries exact line/column positions. This is where determinism comes +from: the tree is a fact about the bytes, not an interpretation of them. + +--- + +## Step 2 — extract: tree → symbol model + +A mechanical walk collects, per symbol: qualified name, signature, span, calls, +control-flow operations, raise sites, and a hash of the symbol's exact source slice. + +**Before:** syntax tree. +**After:** one record per symbol (internal form, never stored): + +```json +{ + "symbol": "load_entry", + "kind": "function", + "signature": "(path: Path, expected_sha: str) -> dict", + "span": [18, 24], + "slice_sha256": "a4f1c2e9…", + "calls": ["path.read_bytes", "sha256().hexdigest", "json.loads"], + "flow": [ + {"op": "CALL", "expr": "path.read_bytes()", "binds": "data"}, + {"op": "CALL", "expr": "sha256(data).hexdigest()", "binds": "current_sha"}, + {"op": "IF", "test": "current_sha != expected_sha", + "then": [{"op": "RAISE", "expr": "StaleSource(path)"}]}, + {"op": "RET", "expr": "json.loads(data)"} + ], + "raises": ["StaleSource"] +} +``` + +`slice_sha256` is the sha256 of source lines 18–24 exactly as they appear on disk. +It becomes the symbol's source-map handle (step 5). + +--- + +## Step 3 — effects and exception propagation + +Still no model. Two mechanical passes over the symbol models: + +**Effect table.** Known callees map to effect classes: + +| Callee pattern | Effect | +|---|---| +| `*.read_bytes`, `*.read_text`, `open(..., "r")`, `*.is_file` | `fs:read` | +| `*.write_bytes`, `*.write_text`, `os.replace` | `fs:write` | +| `requests.*`, `urllib.*`, `fetch` | `net` | +| `subprocess.*`, `os.system` | `proc` | +| `time.*`, `datetime.now` | `time` | +| `random.*`, `uuid.*` | `random` | + +**Propagation (intra-file fixpoint).** If `find_entries` calls `load_entry` and +`load_entry` has `fs:read`, then `find_entries` inherits `fs:read` — with the path +recorded. Same for raises. Calls to symbols outside the file that match no table +entry are listed explicitly as unknown, never silently dropped. + +**Before:** `find_entries` record has `calls` but no semantics. +**After:** + +```json +{ + "symbol": "find_entries", + "effects": ["fs:read (via load_entry, read_expected, is_file)"], + "raises": ["StaleSource (via load_entry)", "ValueError (via read_expected)"], + "unknown_calls": [] +} +``` + +Note: `find_entries` raising `ValueError` is a fact that appears **nowhere on any +single line of the source** — it emerges from propagation. The deterministic IL can +state things the raw text never states in one place. + +--- + +## Step 4 — emit: symbol models → KERN-IL page + +One shared emitter renders all languages. Two comparisons follow. + +### Today's output (`kern-il/0.1`, python-ast-baseline) — real output, ~531 tokens + +```text +KERN-IL/0.1 +source_rel=src/cache_loader.py +source_sha256=412e2610… +generator=deterministic-baseline/0.1 +mode=python-ast-baseline + +MODULE @L1-45 +IMPORT @L3 from __future__ import annotations +IMPORT @L5 import json +IMPORT @L6 import re +IMPORT @L7 from hashlib import sha256 +IMPORT @L8 from pathlib import Path +C @L10 MANIFEST_NAME='manifest.json' +C @L11 ENTRY_PATTERN=re.compile('^[a-z0-9_/]+\\.kern-il\\.txt$') + +CLASS StaleSource(Exception) @L14-15 + +F load_entry(path: Path, expected_sha: str)->dict @L18-24 + DOC Read a cache entry, verify its hash, and parse it as JSON. + CALLS path.read_bytes, sha256(data).hexdigest, json.loads, StaleSource, sha256 + 20|SET data=path.read_bytes() + 21|SET current_sha=sha256(data).hexdigest() + 22|IF current_sha != expected_sha + 23|ERR StaleSource(path) + 24|RET json.loads(data) + … +``` + +Problems: the `CALLS` line duplicates the flow lines below it; every statement +carries its own line number; no effects; no exception propagation; no per-symbol +hash; the omissions block is fixed boilerplate. + +### Proposed output (`kern-il/0.2`) — ~341 tokens + +```text +KERN-IL/0.2 +source_rel=src/cache_loader.py +source_sha256=412e261066069477 +repo_revision=d7e8242 +generator=kern-det/0.2 lang=python frontend=pyast + +IMPORTS json, re, hashlib.sha256, pathlib.Path @L3-8 +C MANIFEST_NAME='manifest.json' @L10 +C ENTRY_PATTERN=re.compile(r'^[a-z0-9_/]+\.kern-il\.txt$') @L11 !FAULT(regex) + +CLASS StaleSource(Exception) @L14-15 ^b8e2d1a4 + +F load_entry(path: Path, expected_sha: str) -> dict @L18-24 ^a4f1c2e9 + CALL path.read_bytes() -> data + CALL sha256(data).hexdigest() -> current_sha + IF current_sha != expected_sha: RAISE StaleSource(path) + RET json.loads(data) + EFFECTS fs:read + RAISES StaleSource + +F find_entries(root: Path) -> list[Path] @L27-37 ^c91d33f2 + CALL load_entry(root/MANIFEST_NAME, read_expected(root)) -> manifest + LOOP (name, record) in manifest['files'].items() + IF not ENTRY_PATTERN.match(name): CONTINUE + IF (root/'ir'/name).is_file(): CALL entries.append(...) + RET entries + EFFECTS fs:read (via load_entry, read_expected, is_file) + RAISES StaleSource, ValueError (via load_entry, read_expected) + +F read_expected(root: Path) -> str @L40-45 ^7d02ee18 + CALL (root/'.pin').read_text().strip() -> pin + IF len(pin) != 64: RAISE ValueError + RET pin + EFFECTS fs:read + RAISES ValueError + +OMIT docstrings=4 comments=0 blank=9; bodies verbatim-compressed +FAULT-BEFORE edit(any), regex(L11), exact-literals +``` + +What changed and why: + +- **`-> var` dataflow form.** `CALL path.read_bytes() -> data` replaces + `SET data=path.read_bytes()` plus the redundant `CALLS` list. +- **Per-symbol `^hash`.** sha256 prefix of the symbol's exact source slice — the + source-map handle. +- **`EFFECTS` / `RAISES`.** Computed in step 3, including propagated facts. +- **`!FAULT(reason)` risk tags.** The compiler stamps lines whose IL rendering is + known-lossy or high-stakes: regex literals, float/bit math, concurrency + primitives, truncated strings. Contract: a tagged line may not support a claim + or an edit without an exact-source fault first. +- **`OMIT` with real counts.** Mechanical per-file numbers instead of boilerplate. +- **One line-span per symbol** instead of per statement. + +--- + +## Step 5 — the source-map handle and the latent-fault trap + +Every symbol line is addressable as: + +```text +repo_revision : source_sha256 : symbol_path : span +d7e8242 : 412e2610… : load_entry : L18-24 (slice ^a4f1c2e9) +``` + +The failure this closes: an agent read the IL (or its image render) earlier, the +source has since changed — or the image was misread — and the agent edits from +stale context. Nothing traps, because unlike a CPU page fault, a *latent* fault +raises no signal. The defense is a program, not agent discipline: + +```bash +python3 kern_cache.py --repo . verify src/cache_loader.py \ + --symbol load_entry --hash a4f1c2e9 +``` + +| Result | Meaning | Required action | +|---|---|---| +| `ok` | Symbol bytes unchanged, same span | proceed | +| `moved` | Same bytes, new span (file shifted) | use returned span | +| `stale` | Symbol bytes changed | fault exact source; IL page invalid | + +The skill contract makes `verify` mandatory on the edit path: no write to a symbol +that was read from IL or an image without a passing `verify` (or a fresh `fault`) +for that symbol's handle. + +--- + +## Step 6 — cache, render, done + +Unchanged from today: the IL page is written to `.kern/ir/…`, the manifest records +`source_sha256` and `ir_sha256`, and the renderer may pack cold pages into lossless +WebP. The codec version bump (`kern-il/0.2`) automatically invalidates every 0.1 +page on first contact, exactly as the existing codec-invalidation path already does. + +--- + +## Measured token counts + +### Small file (this walkthrough's 45-line example) + +Approximate tokens (chars/4), measured on the artifacts above: + +| Artifact | Tokens | vs. source | +|---|---:|---:| +| Source (45 lines, dense, few comments) | ~343 | 1.00× | +| Current 0.1 baseline | ~531 | **1.55× larger** | +| Proposed 0.2 | ~341 | 1.00× — break-even | + +On a **small, dense** file the current baseline *expands* the source, and 0.2 only +breaks even — while adding effects, propagated exceptions, and per-symbol hashes. +The skill should skip IL for files below a size floor where source is already the +cheaper representation. + +### Large file (3,704-line production Python file, ~47,000 tokens) + +Run against a real 3,704-line document-processing module — the same size class as +the pilot file. Deterministic AST extraction was emitted at four detail tiers: + +| Tier | Contents per function | Tokens | Compression | +|---|---|---:|---:| +| L0 | Signatures, classes, constants, imports only | 1,891 | **24.9×** | +| L1 | L0 + deduplicated calls + raises | 4,567 | **10.3×** | +| L2 | L1 + control-flow skeleton (IF/LOOP/TRY/RET, no expressions) | 7,273 | **6.5×** | +| L3 | L2 + full expressions on every flow line | 10,095 | **4.7×** | +| — | Current 0.1 baseline (statement dump) on same file | 28,673 | 1.6× | + +Two conclusions: + +1. **Deterministic extraction reaches pilot-grade compression.** L2 (6.5×) matches + the model-enriched pilot result (6.33×) with zero model tokens and full + determinism. The current 0.1 baseline's weakness is not the AST approach — it + is the verbose per-statement dump (`CALLS` duplication, `SET` lines carrying + whole expressions, per-statement line numbers). +2. **Detail must be tiered per symbol.** The gap between L1 (10.3×) and L3 (4.7×) + is the cost of statement bodies. The 0.2 emitter should map tiers onto KERN's + existing temperature model: cold symbols get L1, warm symbols get L2/L3, hot + symbols get exact source. A tier marker on each function line + (e.g. `F name(...) ... ~L1`) declares what was omitted and makes a deeper + fault explicit and cheap. + +Redaction verified on the same run: a hardcoded `s2_…` API key present in the +source appears nowhere in the IL (6 `` markers emitted). + +### On micro-compressing keywords + +Replacing IL keywords with shorter codes (`CALLS` → `K`, `EFFECTS` → `E`) is +mostly a false economy and is out of scope for 0.2: + +- Modern BPE tokenizers already encode common short words as 1–2 tokens; a + single-letter opcode saves at most ~1 token per line — bounded by roughly 10% + of an L2 page, before subtracting the legend that must ship with every page. +- Identifiers, which dominate IL tokens, can never be mapped: the agent must + grep, quote, and edit them verbatim. +- A per-page dictionary is a new latent-fault source — a misremembered mapping is + exactly the "confidently wrong" failure this design exists to prevent. + +Tiering is the honest lever: L3→L2 alone saves 28% with no decode risk. + +--- + +## Verification plan + +1. **Determinism:** compile every corpus file twice; outputs must be byte-identical. +2. **Golden files:** fixture source → expected IL, per language, in CI. +3. **Fact fidelity:** for each corpus file, extract signatures, raise sites, and + call names independently from the AST and assert the IL contains each one — + machine-checked, replacing the pilot's manual five-fact retrieval check. +4. **Token benchmarks:** count source vs. IL tokens (Anthropic count-tokens API + when available, offline estimator otherwise) across a corpus bucketed by file + size; publish per-language results under `benchmarks/results/`. +5. **Fallback:** corpus runs with tree-sitter absent and with syntax-broken files + must degrade to the labeled generic baseline, never crash. diff --git a/docs/superpowers/specs/2026-07-11-deterministic-il-compiler-design.md b/docs/superpowers/specs/2026-07-11-deterministic-il-compiler-design.md new file mode 100644 index 0000000..88c213b --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-deterministic-il-compiler-design.md @@ -0,0 +1,166 @@ +# KERN-IL/0.2 — Deterministic IL Compiler + +Status: approved design, pending implementation plan. +Companion walkthrough with worked example: [`docs/deterministic-compiler.md`](../../deterministic-compiler.md). + +## Goal + +Replace model-generated IL summaries with a deterministic compiler that lowers +source into KERN-IL from AST information. Same source bytes always produce the +same IL bytes. The IL retains signatures, types, calls, control flow, side +effects, and exceptions, declares what it omits with real counts, and stamps +every symbol with a source-map handle so stale reads can be trapped by a +program instead of agent discipline. + +## Decisions (settled) + +| Question | Decision | +|---|---| +| Parser strategy | tree-sitter for non-Python languages; Python keeps stdlib `ast`. Graceful fallback to generic line baseline when tree-sitter is absent or parsing fails. | +| Model enrichment | Deterministic IL is primary and always present. Model may append clearly-marked `INTENT` lines on top; it never replaces deterministic facts. `prepare`/`commit` machinery stays, demoted to this role. | +| Language scope v1 | Python (upgrade existing frontend) + TypeScript/JavaScript (new tree-sitter frontend). Others stay on generic baseline until format stabilizes. | +| Latent-fault defense | Risk tags in IL + per-symbol slice hashes + new `verify` CLI verb, mandatory on the edit path. | +| Keyword micro-compression | Rejected. BPE already encodes the opcode vocabulary in 1–2 tokens; identifiers must stay verbatim; a per-page dictionary is a new latent-fault source. Tiering is the compression lever. | + +## Measured foundation (3,704-line production Python file, ~47k source tokens) + +| Tier | Per-function content | Tokens | Compression | +|---|---|---:|---:| +| L0 | signatures, classes, constants, imports | 1,891 | 24.9× | +| L1 | + deduplicated calls + raises | 4,567 | 10.3× | +| L2 | + control-flow skeleton, no expressions | 7,273 | 6.5× | +| L3 | + full expressions on flow lines | 10,095 | 4.7× | +| 0.1 today | statement dump | 28,673 | 1.6× | + +L2 matches the model-enriched pilot (6.33×) with zero model tokens. The 0.1 +baseline's weakness is the emitter, not the AST approach. On small dense files +(≤ ~50 lines) IL breaks even at best — a size floor applies. + +## Architecture + +```text +source bytes + │ frontend (per language) + ▼ +symbol models (common internal form) + │ effect/raise propagation (language tables + intra-file fixpoint) + ▼ +emitter (shared, tiered) ──► .kern/ir/.kern-il.txt +``` + +### Components + +1. **`skills/kern/scripts/kern_compile.py`** (new) — frontends + effect engine + + emitter. `kern_cache.py:baseline_for()` calls it; cache/manifest/render + machinery is unchanged. +2. **Frontend interface.** Each frontend consumes source text and produces + symbol models: qualified name, kind, signature, return type, decorators, + span, exact-slice sha256, calls, flow ops, raise sites, module-level + imports/constants. Python frontend uses stdlib `ast` (port + upgrade the + existing `python_ir` logic). TS/JS frontend uses `tree-sitter` + + `tree-sitter-typescript`/`tree-sitter-javascript` via a node-type mapping + table. +3. **Effect engine.** Per-language table mapping known callees to effect + classes: `fs:read`, `fs:write`, `net`, `proc`, `env`, `time`, `random`, + `console`, `thread`. Intra-file fixpoint propagation: a function calling an + effectful local function inherits its effects and raises, recorded with + `via` provenance. Unclassified external calls are listed as + `unknown-calls`, never silently dropped. +4. **Emitter (shared, tiered).** Renders symbol models at detail tiers: + - **L1** (cold default): signature line, `CALLS` (deduplicated), `EFFECTS`, + `RAISES`. + - **L2** (warm default): + control-flow skeleton (`IF`/`LOOP`/`WHILE`/`TRY`/ + `CATCH`/`RET`/`RAISE`/`WITH`), no expressions. + - **L3** (warm, on demand): + expressions, `CALL expr -> var` dataflow form. + Tier is chosen per file (config default) and per symbol (on-demand + re-ensure); each function line carries its tier marker (`~L1` etc.) so the + omission is declared. Hot symbols remain exact-source faults, unchanged. +5. **Secret redaction.** Reuse existing `SECRET_NAME`/`SECRET_VALUE` logic in + all frontends and the emitter (verified working: hardcoded `s2_…` key never + reached IL in testing). + +## KERN-IL/0.2 format + +```text +KERN-IL/0.2 +source_rel=src/loader.py +source_sha256=<64 hex> +repo_revision= | none> +generator=kern-det/0.2 lang=python frontend=pyast tier=L2 + +IMPORTS json, re, hashlib.sha256, pathlib.Path @L3-8 +C ENTRY_PATTERN=re.compile(…) @L11 !FAULT(regex) + +CLASS StaleSource(Exception) @L14-15 ^b8e2d1a4 + +F load_entry(path: Path, expected_sha: str) -> dict @L18-24 ^a4f1c2e9 ~L3 + CALL path.read_bytes() -> data + CALL sha256(data).hexdigest() -> current_sha + IF current_sha != expected_sha: RAISE StaleSource(path) + RET json.loads(data) + EFFECTS fs:read + RAISES StaleSource + +OMIT docstrings=4 comments=0 blank=9 bodies-tiered=2 +FAULT-BEFORE edit(any), regex(L11), exact-literals +``` + +Format rules: + +- Header keys are fixed and machine-parseable (existing `commit` validation + extends to `repo_revision` and `tier`). +- `^hash` = first 8 hex of sha256 of the symbol's exact source slice. +- Source-map handle = `repo_revision : source_sha256 : symbol_path : span`, + reconstructable from any symbol line plus the page header. +- `!FAULT(reason)` risk tags stamped on lines containing regex literals, + float/bit math, concurrency primitives, crypto calls, or truncated/elided + literals. Contract: a tagged line may not support a claim or an edit without + an exact-source fault. +- `OMIT` carries mechanical per-file counts, not boilerplate. +- Optional model enrichment appends only `INTENT : ` lines in + a dedicated trailing section marked `ENRICHMENT model=` — derived, + untrusted, never interleaved with deterministic facts. +- Codec bump `kern-il/0.2` invalidates all 0.1 pages via the existing + codec-invalidation path. + +## CLI changes (`kern_cache.py`) + +- `ensure` — unchanged interface; now emits 0.2 via `kern_compile.py`; accepts + `--tier {L1,L2,L3}` override. +- `verify --symbol --hash ` (new) — recomputes the + symbol slice against current source. Returns `ok` | `moved` (same bytes, new + span, span returned) | `stale` (bytes changed). SKILL.md makes a passing + `verify` or fresh `fault` mandatory before editing any symbol read from IL or + an image. +- Size floor: files whose source is below a configured token estimate + (default ~600 tokens) get a `mode=source-cheaper` stub IL pointing at exact + source instead of a full page. + +## Fallback chain + +1. Python: stdlib `ast` — always available. +2. TS/JS: tree-sitter if importable; otherwise generic line baseline with + `QA tree-sitter unavailable`. +3. Any parse error: generic line baseline with the error noted, never a crash. + +## Verification plan + +1. **Determinism:** compile corpus twice; byte-identical output required. +2. **Golden files:** fixture source → expected IL per language, per tier, in CI. +3. **Fact fidelity:** independently extract signatures, raise sites, and call + names from the AST and assert each appears in the IL (machine-checked). +4. **Token benchmarks:** `benchmarks/token_bench.py` counts source vs IL per + tier across a corpus bucketed by file size (Anthropic count-tokens API when + a key is present, chars/4 estimator otherwise); results published under + `benchmarks/results/`. Acceptance: ≥6× at L2 on the large-Python bucket + (measured 6.5×), reported honestly if lower. +5. **Fallback tests:** tree-sitter absent, syntax-broken files, binary input, + secret redaction. +6. **verify verb tests:** ok / moved / stale paths, including the + symbol-renamed and file-shifted cases. + +## Out of scope (0.2) + +LSP type resolution, cross-file call graph, Go/Rust/Java/C# frontends, image +renderer changes (it consumes IL text unchanged), incremental (per-edit) +recompilation. From 48e5e5921ed93428ed421093def44de1bc68556e Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:04:22 +0200 Subject: [PATCH 02/55] docs: implementation plan for KERN-IL/0.2 deterministic compiler Co-Authored-By: Claude Fable 5 --- .../2026-07-11-deterministic-il-compiler.md | 2027 +++++++++++++++++ 1 file changed, 2027 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-11-deterministic-il-compiler.md diff --git a/docs/superpowers/plans/2026-07-11-deterministic-il-compiler.md b/docs/superpowers/plans/2026-07-11-deterministic-il-compiler.md new file mode 100644 index 0000000..4f6cd9b --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-deterministic-il-compiler.md @@ -0,0 +1,2027 @@ +# KERN-IL/0.2 Deterministic Compiler Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace model-generated KERN-IL with a deterministic compiler (`kern_compile.py`) that lowers Python and TS/JS source into tiered KERN-IL/0.2 pages with effects, exceptions, per-symbol source-map hashes, and a `verify` CLI verb that traps stale reads. + +**Architecture:** A new module `skills/kern/scripts/kern_compile.py` holds language frontends (Python stdlib `ast`; TS/JS via optional tree-sitter) that produce a common symbol model, an effect/raise propagation engine, and a shared tiered emitter (L1/L2/L3). `kern_cache.py` keeps all cache/manifest machinery and calls the compiler from `baseline_for()`; it gains a codec bump to `kern-il/0.2`, a size floor, a `--tier` flag, and a `verify` verb. A benchmark script measures compression per tier and file-size bucket. + +**Tech Stack:** Python 3.10+ stdlib only, except optional `tree-sitter`, `tree-sitter-javascript`, `tree-sitter-typescript` (graceful fallback when absent). Tests use stdlib `unittest` (repo has no pytest). + +## Global Constraints + +- Spec: `docs/superpowers/specs/2026-07-11-deterministic-il-compiler-design.md`. Walkthrough with format examples: `docs/deterministic-compiler.md`. +- Determinism is a hard invariant: same source bytes → byte-identical IL. Never put timestamps, absolute paths, or environment data in IL output. +- Codec string is exactly `kern-il/0.2`; generator string is exactly `kern-det/0.2`; IL first line is exactly `KERN-IL/0.2`. +- Tier names are exactly `L1`, `L2`, `L3`. Default tier `L2`. Size floor config key `min_ir_tokens`, default `600` (chars/4 estimate). +- `kern_compile.py` must not import `kern_cache.py` (the reverse dependency only, and only as a local import inside functions). +- Secret redaction rules are copied from the existing `kern_cache.py` (`SECRET_NAME`, `SECRET_VALUE`); a likely credential must never appear in IL. +- All scripts must keep passing `python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/render_ir.py skills/kern/scripts/kern_compile.py`. +- Run all tests with `python3 -m unittest discover -s tests -v` from repo root. +- Work on branch `design/deterministic-il-compiler` (already exists, has the spec). + +## File Structure + +- `skills/kern/scripts/kern_compile.py` — NEW. Symbol model, Python frontend, TS/JS frontend, effect engine, tiered emitter. Self-contained. +- `skills/kern/scripts/kern_cache.py` — MODIFY. Codec bump, `baseline_for` dispatch, size-floor stub, `git_revision`, `--tier`, `verify` verb; delete the old AST baseline (`python_ir`, `function_card`, `outline`, `expr`, `LiteralSanitizer`, `sanitize_string`, `call_name`, `target_text`). Keep `generic_ir` + `redact_line` + `SECRET_*`. +- `benchmarks/token_bench.py` — NEW. Per-tier token benchmark + fidelity check. +- `tests/__init__.py`, `tests/test_kern_compile.py`, `tests/test_effects.py`, `tests/test_emitter.py`, `tests/test_tsjs.py`, `tests/test_cache_integration.py`, `tests/test_verify.py`, `tests/test_token_bench.py` — NEW. +- `skills/kern/SKILL.md`, `docs/architecture.md`, `CHANGELOG.md`, `README.md` — MODIFY (docs, Task 9). + +--- + +### Task 1: Symbol model + Python frontend (signatures, spans, slice hashes) + +**Files:** +- Create: `skills/kern/scripts/kern_compile.py` +- Create: `tests/__init__.py` (empty) +- Test: `tests/test_kern_compile.py` + +**Interfaces:** +- Consumes: nothing (first task). +- Produces: `kern_compile.parse_python(text: str) -> ModuleIR`; dataclasses `FlowOp(op, detail, binds, depth, line, risk)`, `Symbol(kind, name, span, signature, returns, decorators, slice8, calls, raises, flow, is_async, bases, detail, risk, effects, raises_all, unknown_calls)`, `ModuleIR(lang, frontend, symbols, omit, parse_error)`; helpers `slice_sha8(text, start, end) -> str` (8 hex), `expr_text(node, max_length, secret_hint) -> str`, `sanitize_string(value, secret_hint) -> str`; constants `CODEC_VERSION = "kern-il/0.2"`, `GENERATOR = "kern-det/0.2"`. Later tasks fill `flow` (Task 2) and `effects`/`raises_all` (Task 3). + +- [ ] **Step 1: Write the failing test** + +Create `tests/__init__.py` (empty file) and `tests/test_kern_compile.py`: + +```python +import hashlib +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + +SAMPLE = '''"""Module docstring.""" + +import json +from hashlib import sha256 +from pathlib import Path + +MANIFEST_NAME = "manifest.json" +API_KEY = "s2_abcdefghijklmnop1234" + + +class StaleSource(Exception): + """Raised on hash mismatch.""" + + +def load_entry(path: Path, expected_sha: str) -> dict: + """Read, verify, parse.""" + data = path.read_bytes() + current_sha = sha256(data).hexdigest() + if current_sha != expected_sha: + raise StaleSource(path) + return json.loads(data) + + +class Loader: + async def fetch(self, url): + return await self.client.get(url) +''' + + +class TestPythonFrontend(unittest.TestCase): + def setUp(self): + self.mod = kern_compile.parse_python(SAMPLE) + + def sym(self, name): + return next(s for s in self.mod.symbols if s.name == name) + + def test_module_metadata(self): + self.assertEqual(self.mod.lang, "python") + self.assertEqual(self.mod.frontend, "pyast") + self.assertEqual(self.mod.parse_error, "") + + def test_function_symbol(self): + f = self.sym("load_entry") + self.assertEqual(f.kind, "function") + self.assertIn("path: Path", f.signature) + self.assertIn("expected_sha: str", f.signature) + self.assertEqual(f.returns, "dict") + self.assertFalse(f.is_async) + self.assertIn("path.read_bytes", f.calls) + self.assertIn("json.loads", f.calls) + self.assertEqual(f.raises, ["StaleSource"]) + + def test_slice_hash_matches_exact_source_lines(self): + f = self.sym("load_entry") + start, end = f.span + lines = SAMPLE.splitlines(keepends=True) + expected = hashlib.sha256("".join(lines[start - 1:end]).encode()).hexdigest()[:8] + self.assertEqual(f.slice8, expected) + self.assertEqual(SAMPLE.splitlines()[start - 1].strip(), "def load_entry(path: Path, expected_sha: str) -> dict:") + + def test_method_is_qualified_and_async(self): + m = self.sym("Loader.fetch") + self.assertTrue(m.is_async) + + def test_class_symbol(self): + c = self.sym("StaleSource") + self.assertEqual(c.kind, "class") + self.assertEqual(c.bases, "Exception") + self.assertEqual(len(c.slice8), 8) + + def test_secret_const_redacted(self): + consts = [s for s in self.mod.symbols if s.kind == "const"] + api = next(s for s in consts if s.name == "API_KEY") + self.assertNotIn("s2_abcdefghijklmnop1234", api.detail) + self.assertIn("REDACTED", api.detail) + + def test_imports_collected(self): + imports = [s for s in self.mod.symbols if s.kind == "import"] + details = " ".join(s.detail for s in imports) + self.assertIn("json", details) + self.assertIn("sha256", details) + + def test_omit_counts(self): + self.assertGreaterEqual(self.mod.omit["docstrings"], 3) + self.assertGreaterEqual(self.mod.omit["blank"], 5) + + def test_parse_error_reported(self): + bad = kern_compile.parse_python("def broken(:\n") + self.assertNotEqual(bad.parse_error, "") + self.assertEqual(bad.symbols, []) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python3 -m unittest tests.test_kern_compile -v` +Expected: FAIL — `ModuleNotFoundError: No module named 'kern_compile'` + +- [ ] **Step 3: Write minimal implementation** + +Create `skills/kern/scripts/kern_compile.py`: + +```python +#!/usr/bin/env python3 +"""Deterministic KERN-IL/0.2 compiler: language frontends, effect engine, tiered emitter.""" + +from __future__ import annotations + +import ast +import copy +import hashlib +import re +from dataclasses import dataclass, field + +CODEC_VERSION = "kern-il/0.2" +GENERATOR = "kern-det/0.2" + +SECRET_NAME = re.compile( + r"(?i)(api[_-]?key|access[_-]?key|auth|bearer|credential|passwd|password|private[_-]?key|secret|token)" +) +SECRET_VALUE = re.compile( + r"(?i)(?:sk|rk|pk|s2)[_-][A-Za-z0-9_-]{12,}|-----BEGIN [A-Z ]*PRIVATE KEY-----|" + r"(?:aws|ghp|github_pat)_[A-Za-z0-9_-]{12,}" +) +SPACE = re.compile(r"\s+") + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def slice_sha8(source_text: str, start: int, end: int) -> str: + lines = source_text.splitlines(keepends=True) + return sha256_hex("".join(lines[start - 1:end]).encode("utf-8", "surrogatepass"))[:8] + + +def sanitize_string(value: str, secret_hint: bool = False) -> str: + digest = sha256_hex(value.encode("utf-8", "surrogatepass"))[:12] + if secret_hint or SECRET_VALUE.search(value): + return f"" + if len(value) > 160: + return f"" + return value + + +class _LiteralSanitizer(ast.NodeTransformer): + def visit_Constant(self, node: ast.Constant): + if isinstance(node.value, str): + return ast.copy_location(ast.Constant(sanitize_string(node.value)), node) + return node + + +def expr_text(node: ast.AST | None, max_length: int = 200, secret_hint: bool = False) -> str: + if node is None: + return "None" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + rendered = repr(sanitize_string(node.value, secret_hint)) + else: + clone = _LiteralSanitizer().visit(copy.deepcopy(node)) + ast.fix_missing_locations(clone) + try: + rendered = ast.unparse(clone) + except Exception: + rendered = f"<{node.__class__.__name__}>" + rendered = SPACE.sub(" ", rendered).strip() + if secret_hint and not rendered.startswith("'" + if len(rendered) > max_length: + digest = sha256_hex(rendered.encode())[:12] + rendered = rendered[: max_length - 24] + f"…" + return rendered + + +def _target(node: ast.AST) -> str: + try: + return SPACE.sub(" ", ast.unparse(node)).strip() + except Exception: + return f"<{node.__class__.__name__}>" + + +@dataclass +class FlowOp: + op: str + detail: str = "" + binds: str = "" + depth: int = 0 + line: int = 0 + risk: str = "" + + +@dataclass +class Symbol: + kind: str # function | class | const | import + name: str + span: tuple = (0, 0) + signature: str = "" + returns: str = "" + decorators: list = field(default_factory=list) + slice8: str = "" + calls: list = field(default_factory=list) + raises: list = field(default_factory=list) + flow: list = field(default_factory=list) + is_async: bool = False + bases: str = "" + detail: str = "" + risk: str = "" + effects: dict = field(default_factory=dict) # effect -> list of via names ([] = direct) + raises_all: dict = field(default_factory=dict) # exception -> list of via names + unknown_calls: int = 0 + + +@dataclass +class ModuleIR: + lang: str + frontend: str + symbols: list + omit: dict + parse_error: str = "" + + +def _import_detail(node: ast.Import | ast.ImportFrom) -> str: + if isinstance(node, ast.Import): + return ", ".join(a.name for a in node.names) + names = ",".join(a.name for a in node.names) + return f"{node.module or '.'}:{names}" + + +def _omit_counts(text: str, tree: ast.Module) -> dict: + lines = text.splitlines() + comments = sum(1 for l in lines if l.strip().startswith("#")) + blank = sum(1 for l in lines if not l.strip()) + docstrings = 0 + for node in ast.walk(tree): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + if ast.get_docstring(node) is not None: + docstrings += 1 + assigns = sum(1 for n in ast.walk(tree) if isinstance(n, (ast.Assign, ast.AnnAssign, ast.AugAssign))) + return {"docstrings": docstrings, "comments": comments, "blank": blank, "assignments": assigns} + + +def _function_symbol(node, qualified: str, text: str) -> Symbol: + calls: list[str] = [] + raises: list[str] = [] + for ch in ast.walk(node): + if isinstance(ch, ast.Call): + try: + name = expr_text(ch.func, 80) + except Exception: + name = "" + if name not in calls: + calls.append(name) + elif isinstance(ch, ast.Raise) and ch.exc is not None: + name = expr_text(ch.exc, 60).split("(")[0] + if name not in raises: + raises.append(name) + start = min([d.lineno for d in node.decorator_list] + [node.lineno]) + end = node.end_lineno or node.lineno + return Symbol( + kind="function", + name=qualified, + span=(start, end), + signature=expr_text(node.args, 200), + returns=expr_text(node.returns, 60) if node.returns else "", + decorators=[expr_text(d, 60) for d in node.decorator_list], + slice8=slice_sha8(text, start, end), + calls=calls, + raises=raises, + flow=[], # filled by flow_ops in Task 2 + is_async=isinstance(node, ast.AsyncFunctionDef), + ) + + +def parse_python(text: str) -> ModuleIR: + try: + tree = ast.parse(text) + except SyntaxError as exc: + return ModuleIR("python", "pyast", [], {}, parse_error=f"L{exc.lineno}: {exc.msg}") + symbols: list[Symbol] = [] + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + symbols.append(Symbol(kind="import", name="", detail=_import_detail(node), + span=(node.lineno, node.end_lineno or node.lineno))) + elif isinstance(node, (ast.Assign, ast.AnnAssign)): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + names = ",".join(_target(t) for t in targets) + hint = bool(SECRET_NAME.search(names)) + symbols.append(Symbol(kind="const", name=names, + detail=expr_text(node.value, 100, hint), + span=(node.lineno, node.end_lineno or node.lineno))) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + symbols.append(_function_symbol(node, node.name, text)) + elif isinstance(node, ast.ClassDef): + end = node.end_lineno or node.lineno + symbols.append(Symbol(kind="class", name=node.name, + bases=",".join(expr_text(b, 60) for b in node.bases), + span=(node.lineno, end), slice8=slice_sha8(text, node.lineno, end))) + for member in node.body: + if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): + symbols.append(_function_symbol(member, f"{node.name}.{member.name}", text)) + return ModuleIR("python", "pyast", symbols, _omit_counts(text, tree)) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python3 -m unittest tests.test_kern_compile -v` +Expected: all tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add skills/kern/scripts/kern_compile.py tests/__init__.py tests/test_kern_compile.py +git commit -m "feat: kern_compile symbol model and Python frontend" +``` + +--- + +### Task 2: Control-flow extraction with risk tags + +**Files:** +- Modify: `skills/kern/scripts/kern_compile.py` (add `flow_ops`, `expr_risk`; wire into `_function_symbol`) +- Test: `tests/test_kern_compile.py` (append test class) + +**Interfaces:** +- Consumes: Task 1 dataclasses. +- Produces: `flow_ops(statements: list[ast.stmt], depth=0, budget=200) -> list[FlowOp]`; `expr_risk(node: ast.AST) -> str` returning one of `"regex" | "crypto" | "concurrency" | "math" | ""`. `_function_symbol` now fills `Symbol.flow`. Ops emitted: `CALL` (bare calls and assign-from-call, with `binds`), `IF/ELSE/LOOP/WHILE/WITH/TRY/CATCH/FINALLY/RET/RAISE/AWAIT/YIELD/MATCH/CASE/NESTED/BREAK/CONTINUE`. Plain assignments are intentionally NOT emitted (they are counted in `omit["assignments"]`). + +- [ ] **Step 1: Write the failing test** + +Append to `tests/test_kern_compile.py`: + +```python +FLOW_SAMPLE = ''' +import re +import threading + +PATTERN = re.compile(r"^x+$") + + +def process(path, items): + data = path.read_bytes() + total = 0 + for item in items: + if item.bad: + raise ValueError(item) + total += 1 + try: + result = transform(data) + except KeyError as exc: + log(exc) + finally: + cleanup() + with threading.Lock(): + shared.append(total) + return result + + +def transform(data): + return re.sub(r"a+", "b", data.decode()) +''' + + +class TestFlowOps(unittest.TestCase): + def setUp(self): + self.mod = kern_compile.parse_python(FLOW_SAMPLE) + self.proc = next(s for s in self.mod.symbols if s.name == "process") + + def ops(self): + return [(o.op, o.depth) for o in self.proc.flow] + + def test_call_with_binds(self): + first = self.proc.flow[0] + self.assertEqual(first.op, "CALL") + self.assertEqual(first.binds, "data") + self.assertIn("path.read_bytes", first.detail) + self.assertGreater(first.line, 0) + + def test_plain_assignment_not_emitted(self): + details = " ".join(o.detail for o in self.proc.flow) + self.assertNotIn("total = 0", details) + + def test_structure(self): + ops = self.ops() + self.assertIn(("LOOP", 0), ops) + self.assertIn(("IF", 1), ops) + self.assertIn(("RAISE", 2), ops) + self.assertIn(("TRY", 0), ops) + self.assertIn(("CATCH", 0), ops) + self.assertIn(("FINALLY", 0), ops) + self.assertIn(("WITH", 0), ops) + self.assertIn(("RET", 0), ops) + + def test_regex_risk_tagged(self): + trans = next(s for s in self.mod.symbols if s.name == "transform") + ret = next(o for o in trans.flow if o.op == "RET") + self.assertEqual(ret.risk, "regex") + + def test_concurrency_risk_tagged(self): + withs = [o for o in self.proc.flow if o.op == "WITH"] + self.assertEqual(withs[0].risk, "concurrency") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python3 -m unittest tests.test_kern_compile.TestFlowOps -v` +Expected: FAIL — flow list empty (`IndexError` / assertion failures). + +- [ ] **Step 3: Write implementation** + +Add to `kern_compile.py` (above `_function_symbol`), and change `_function_symbol` so `flow=flow_ops(node.body)`: + +```python +_RISK_CALL = [ + ("regex", re.compile(r"^re\.(compile|match|search|sub|split|fullmatch|findall|finditer)$")), + ("crypto", re.compile(r"^(hashlib|hmac|secrets)\.")), + ("concurrency", re.compile(r"^(threading|asyncio|multiprocessing|concurrent)\.")), +] +_RISK_MATH = re.compile(r"(\*\*|<<|>>)") +_TRY_TYPES = (ast.Try, getattr(ast, "TryStar", ast.Try)) + + +def expr_risk(node: ast.AST | None) -> str: + if node is None: + return "" + for ch in ast.walk(node): + if isinstance(ch, ast.Call): + try: + fn = ast.unparse(ch.func) + except Exception: + continue + for name, rx in _RISK_CALL: + if rx.search(fn): + return name + elif isinstance(ch, ast.withitem): + try: + fn = ast.unparse(ch.context_expr) + except Exception: + continue + if fn.startswith(("threading.", "asyncio.", "multiprocessing.")): + return "concurrency" + try: + if _RISK_MATH.search(ast.unparse(node)): + return "math" + except Exception: + pass + return "" + + +def flow_ops(statements: list, depth: int = 0, budget: int = 200) -> list: + ops: list[FlowOp] = [] + + def add(node, op, detail="", binds="", risk=""): + if len(ops) < budget: + ops.append(FlowOp(op=op, detail=detail, binds=binds, depth=depth, + line=getattr(node, "lineno", 0), risk=risk)) + + def sub(body): + return flow_ops(body, depth + 1, budget - len(ops)) + + for s in statements: + if len(ops) >= budget: + break + if isinstance(s, ast.Expr) and isinstance(s.value, ast.Constant) and isinstance(s.value.value, str): + continue # docstring + if isinstance(s, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + add(s, "NESTED", s.name) + elif isinstance(s, (ast.Assign, ast.AnnAssign)) and isinstance(getattr(s, "value", None), (ast.Call, ast.Await)): + targets = s.targets if isinstance(s, ast.Assign) else [s.target] + names = ",".join(_target(t) for t in targets) + call = s.value.value if isinstance(s.value, ast.Await) else s.value + hint = bool(SECRET_NAME.search(names)) + add(s, "CALL", expr_text(call, 120, hint), binds=names, risk=expr_risk(call)) + elif isinstance(s, ast.If): + add(s, "IF", expr_text(s.test, 100), risk=expr_risk(s.test)) + ops.extend(sub(s.body)) + if s.orelse: + add(s, "ELSE") + ops.extend(sub(s.orelse)) + elif isinstance(s, (ast.For, ast.AsyncFor)): + add(s, "LOOP", f"{_target(s.target)} in {expr_text(s.iter, 100)}") + ops.extend(sub(s.body)) + elif isinstance(s, ast.While): + add(s, "WHILE", expr_text(s.test, 100), risk=expr_risk(s.test)) + ops.extend(sub(s.body)) + elif isinstance(s, (ast.With, ast.AsyncWith)): + detail = ", ".join(expr_text(i.context_expr, 80) for i in s.items) + risk = "" + for i in s.items: + risk = risk or expr_risk(i.context_expr) + add(s, "WITH", detail, risk=risk) + ops.extend(sub(s.body)) + elif isinstance(s, _TRY_TYPES): + add(s, "TRY") + ops.extend(sub(s.body)) + for h in s.handlers: + add(h, "CATCH", expr_text(h.type, 60)) + ops.extend(sub(h.body)) + if s.finalbody: + add(s, "FINALLY") + ops.extend(sub(s.finalbody)) + elif isinstance(s, ast.Return): + add(s, "RET", expr_text(s.value, 100), risk=expr_risk(s.value)) + elif isinstance(s, ast.Raise): + add(s, "RAISE", expr_text(s.exc, 80)) + elif isinstance(s, ast.Match): + add(s, "MATCH", expr_text(s.subject, 80)) + for case in s.cases: + add(case, "CASE", expr_text(case.pattern, 60)) + ops.extend(sub(case.body)) + elif isinstance(s, ast.Expr): + v = s.value + if isinstance(v, ast.Await): + add(s, "AWAIT", expr_text(v.value, 120)) + elif isinstance(v, ast.Call): + add(s, "CALL", expr_text(v, 120), risk=expr_risk(v)) + elif isinstance(v, (ast.Yield, ast.YieldFrom)): + add(s, "YIELD", expr_text(getattr(v, "value", None), 80)) + elif isinstance(s, (ast.Break, ast.Continue)): + add(s, s.__class__.__name__.upper()) + return ops[:budget] +``` + +Note: `ast.Match` requires Python 3.10+ (global constraint). In `_function_symbol`, replace `flow=[]` with `flow=flow_ops(node.body)`. + +- [ ] **Step 4: Run tests** + +Run: `python3 -m unittest tests.test_kern_compile -v` +Expected: all PASS (Task 1 tests still green). + +- [ ] **Step 5: Commit** + +```bash +git add skills/kern/scripts/kern_compile.py tests/test_kern_compile.py +git commit -m "feat: control-flow extraction with risk tags" +``` + +--- + +### Task 3: Effect tables and effect/raise propagation + +**Files:** +- Modify: `skills/kern/scripts/kern_compile.py` (add `EFFECT_RULES`, `classify_call`, `propagate`) +- Test: `tests/test_effects.py` + +**Interfaces:** +- Consumes: `ModuleIR`, `Symbol` from Task 1. +- Produces: `classify_call(call_name: str) -> list[str]`; `propagate(module: ModuleIR) -> None` filling `Symbol.effects: dict[str, list[str]]` (effect → sorted via-names, `[]` = direct), `Symbol.raises_all: dict[str, list[str]]`, `Symbol.unknown_calls: int`. Idempotent: calling twice changes nothing (emitter calls it on every emit). + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_effects.py`: + +```python +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + +SAMPLE = ''' +def read_expected(root): + pin = (root / ".pin").read_text().strip() + if len(pin) != 64: + raise ValueError(root) + return pin + + +def load_entry(path, expected_sha): + data = path.read_bytes() + if not data: + raise StaleSource(path) + return data + + +def find_entries(root): + manifest = load_entry(root / "m.json", read_expected(root)) + frobnicate(manifest) + return manifest +''' + + +class TestClassify(unittest.TestCase): + def test_fs_read(self): + self.assertIn("fs:read", kern_compile.classify_call("path.read_bytes")) + self.assertIn("fs:read", kern_compile.classify_call("open")) + def test_fs_write(self): + self.assertIn("fs:write", kern_compile.classify_call("os.replace")) + def test_proc(self): + self.assertIn("proc", kern_compile.classify_call("subprocess.run")) + def test_net(self): + self.assertIn("net", kern_compile.classify_call("requests.post")) + def test_unknown(self): + self.assertEqual(kern_compile.classify_call("frobnicate"), []) + + +class TestPropagate(unittest.TestCase): + def setUp(self): + self.mod = kern_compile.parse_python(SAMPLE) + kern_compile.propagate(self.mod) + + def sym(self, name): + return next(s for s in self.mod.symbols if s.name == name) + + def test_direct_effect(self): + self.assertEqual(self.sym("load_entry").effects.get("fs:read"), []) + + def test_inherited_effect_with_via(self): + eff = self.sym("find_entries").effects + self.assertIn("fs:read", eff) + self.assertIn("load_entry", eff["fs:read"]) + + def test_raises_propagate(self): + ra = self.sym("find_entries").raises_all + self.assertIn("StaleSource", ra) + self.assertIn("ValueError", ra) + self.assertIn("read_expected", ra["ValueError"]) + + def test_unknown_counted(self): + self.assertGreaterEqual(self.sym("find_entries").unknown_calls, 1) + + def test_idempotent(self): + before = {s.name: dict(s.effects) for s in self.mod.symbols} + kern_compile.propagate(self.mod) + after = {s.name: dict(s.effects) for s in self.mod.symbols} + self.assertEqual(before, after) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python3 -m unittest tests.test_effects -v` +Expected: FAIL — `AttributeError: module 'kern_compile' has no attribute 'classify_call'` + +- [ ] **Step 3: Write implementation** + +Add to `kern_compile.py`: + +```python +EFFECT_RULES = [ + ("fs:read", re.compile( + r"^(open|fitz\.open|os\.(walk|listdir|stat|scandir)|os\.path\.(exists|isfile|isdir|getsize|getmtime|basename)" + r")$|\.(read|readlines|read_text|read_bytes|exists|is_file|is_dir|stat|iterdir|glob)$")), + ("fs:write", re.compile( + r"^(os\.(replace|remove|makedirs|mkdir|rename|unlink|chmod)|shutil\.\w+|tempfile\.\w+)$" + r"|\.(write|writelines|write_text|write_bytes|save|unlink|mkdir|touch|chmod)$")), + ("net", re.compile(r"^(requests|urllib|socket|http)\.")), + ("proc", re.compile(r"^(subprocess\.\w+|os\.(system|popen|execv|execve|spawnl))$")), + ("env", re.compile(r"^os\.(getenv|putenv|environ\.get)$")), + ("time", re.compile(r"^(time\.(time|sleep|monotonic|time_ns)|datetime\.(now|utcnow|datetime\.now))$|\.sleep$")), + ("random", re.compile(r"^(random|uuid|secrets)\.")), + ("console", re.compile(r"^(print|input)$|^logging\.")), + ("thread", re.compile(r"^(threading|asyncio|multiprocessing|concurrent)\.|^ThreadPoolExecutor$")), +] + + +def classify_call(call_name: str) -> list: + name = call_name.split("(")[0].strip() + return [effect for effect, rx in EFFECT_RULES if rx.search(name)] + + +def propagate(module: ModuleIR) -> None: + funcs = [s for s in module.symbols if s.kind == "function"] + by_tail: dict[str, list] = {} + for s in funcs: + by_tail.setdefault(s.name.split(".")[-1], []).append(s) + for s in funcs: + if not s.effects: + s.effects = {e: [] for c in s.calls for e in classify_call(c)} + if not s.raises_all: + s.raises_all = {r: [] for r in s.raises} + unknown = 0 + for c in s.calls: + tail = c.split("(")[0].split(".")[-1] + if not classify_call(c) and tail not in by_tail: + unknown += 1 + s.unknown_calls = unknown + changed, rounds = True, 0 + while changed and rounds < 32: + changed, rounds = False, rounds + 1 + for s in funcs: + for c in s.calls: + tail = c.split("(")[0].split(".")[-1] + cands = by_tail.get(tail, []) + if len(cands) != 1 or cands[0] is s: + continue + callee = cands[0] + for eff in callee.effects: + if eff not in s.effects: + s.effects[eff] = [tail] + changed = True + elif s.effects[eff] and tail not in s.effects[eff]: + s.effects[eff] = sorted(s.effects[eff] + [tail]) + for exc in callee.raises_all: + if exc not in s.raises_all: + s.raises_all[exc] = [tail] + changed = True + elif s.raises_all[exc] and tail not in s.raises_all[exc]: + s.raises_all[exc] = sorted(s.raises_all[exc] + [tail]) +``` + +Note the `if not s.effects:` guards — they make `propagate` idempotent. + +- [ ] **Step 4: Run tests** + +Run: `python3 -m unittest tests.test_effects tests.test_kern_compile -v` +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +git add skills/kern/scripts/kern_compile.py tests/test_effects.py +git commit -m "feat: effect tables and effect/raise propagation" +``` + +--- + +### Task 4: Tiered emitter + +**Files:** +- Modify: `skills/kern/scripts/kern_compile.py` (add `emit_il`, `_function_lines`, `_render_provenanced`) +- Test: `tests/test_emitter.py` + +**Interfaces:** +- Consumes: `ModuleIR`, `propagate` (called inside `emit_il`). +- Produces: `emit_il(module: ModuleIR, source_rel: str, source_sha256: str, repo_revision: str = "none", tier: str = "L2") -> str`. Output format per spec: 5 header lines + blank, `IMPORTS`, `C`, `CLASS`, `F` blocks, `OMIT`, `FAULT-BEFORE`. Task 6 calls this from `kern_cache.baseline_for`. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_emitter.py`: + +```python +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + +SAMPLE = ''' +import re +from pathlib import Path + +TOKEN = "ghp_abcdefghijklmnop1234" +PATTERN = re.compile(r"^[a-z]+$") + + +def load_entry(path: Path, expected_sha: str) -> dict: + data = path.read_bytes() + if not data: + raise ValueError(path) + return parse(data) + + +def parse(data): + return data.decode() +''' + + +def emit(tier): + mod = kern_compile.parse_python(SAMPLE) + return kern_compile.emit_il(mod, "src/x.py", "a" * 64, "d7e8242", tier) + + +class TestEmitter(unittest.TestCase): + def test_header(self): + il = emit("L2").splitlines() + self.assertEqual(il[0], "KERN-IL/0.2") + self.assertEqual(il[1], "source_rel=src/x.py") + self.assertEqual(il[2], "source_sha256=" + "a" * 64) + self.assertEqual(il[3], "repo_revision=d7e8242") + self.assertIn("generator=kern-det/0.2 lang=python frontend=pyast tier=L2", il[4]) + + def test_function_line_format(self): + il = emit("L2") + fline = next(l for l in il.splitlines() if l.startswith("F load_entry")) + self.assertRegex(fline, r"^F load_entry\(.+\) -> dict @L\d+-\d+ \^[0-9a-f]{8} ~L2$") + + def test_tier_l1_has_no_flow(self): + il = emit("L1") + self.assertNotIn(" IF", il) + self.assertIn("CALLS", il) + self.assertIn("EFFECTS fs:read", il) + self.assertIn("RAISES ValueError", il) + + def test_tier_l2_flow_without_expressions(self): + il = emit("L2") + body = [l for l in il.splitlines() if l.startswith(" ")] + joined = "\n".join(body) + self.assertIn("IF", joined) + self.assertIn("RAISE", joined) + self.assertNotIn("not data", joined) + + def test_tier_l3_flow_with_expressions_and_binds(self): + il = emit("L3") + self.assertIn("CALL path.read_bytes() -> data", il) + self.assertIn("IF not data", il) + + def test_l3_is_larger_than_l2_is_larger_than_l1(self): + self.assertGreater(len(emit("L3")), len(emit("L2"))) + self.assertGreater(len(emit("L2")), len(emit("L1"))) + + def test_secret_never_in_output(self): + for tier in ("L1", "L2", "L3"): + self.assertNotIn("ghp_abcdefghijklmnop1234", emit(tier)) + + def test_regex_const_fault_tagged(self): + il = emit("L2") + cline = next(l for l in il.splitlines() if l.startswith("C PATTERN")) + self.assertIn("!FAULT(regex)", cline) + self.assertIn("regex(L", il.splitlines()[-1]) + + def test_omit_counts_and_fault_before(self): + lines = emit("L2").splitlines() + self.assertTrue(lines[-2].startswith("OMIT ")) + self.assertIn("bodies-tier=L2", lines[-2]) + self.assertTrue(lines[-1].startswith("FAULT-BEFORE edit(any), exact-literals")) + + def test_deterministic(self): + self.assertEqual(emit("L2"), emit("L2")) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python3 -m unittest tests.test_emitter -v` +Expected: FAIL — `AttributeError: ... no attribute 'emit_il'` + +- [ ] **Step 3: Write implementation** + +Add to `kern_compile.py`. Also extend `parse_python` const handling to set `risk`: after building the const `Symbol`, set `symbol.risk = expr_risk(node.value)` (requires `expr_risk` from Task 2 — it is already defined above `parse_python` after Task 2 reordering; if not, move it above). + +```python +_TIER_LEVEL = {"L1": 1, "L2": 2, "L3": 3} +_ELIDED = (" str: + parts = [] + for key in sorted(mapping): + vias = mapping[key] + parts.append(key + (f" (via {', '.join(sorted(vias))})" if vias else "")) + if unknown: + parts.append(f"unknown-calls={unknown}") + return ", ".join(parts) + + +def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: + head = "ASYNC F" if s.is_async else "F" + lines = [f"{head} {s.name}({s.signature}) -> {s.returns or 'Any'} " + f"@L{s.span[0]}-{s.span[1]} ^{s.slice8} ~{tier}"] + if s.decorators: + lines.append(" DECORATORS " + ", ".join(s.decorators)) + if s.calls: + shown = s.calls[:25] + extra = f" …+{len(s.calls) - 25}" if len(s.calls) > 25 else "" + lines.append(" CALLS " + ", ".join(shown) + extra) + effects = _render_provenanced(s.effects, s.unknown_calls) + if effects: + lines.append(" EFFECTS " + effects) + raises = _render_provenanced(s.raises_all, 0) + if raises: + lines.append(" RAISES " + raises) + if level >= 2: + for op in s.flow: + pad = " " * (op.depth + 2) + piece = op.op + if level == 3 and op.detail: + piece += f" {op.detail}" + elif level == 2 and op.op in ("CATCH", "NESTED", "CASE") and op.detail: + piece += f" {op.detail}" + if level == 3 and op.binds: + piece += f" -> {op.binds}" + risk = op.risk + if not risk and level == 3 and any(m in op.detail for m in _ELIDED): + risk = "elided-literal" + if risk: + piece += f" !FAULT({risk})" + faults.append(f"{risk}(L{op.line})") + lines.append(pad + piece) + return lines + + +def emit_il(module: ModuleIR, source_rel: str, source_sha256: str, + repo_revision: str = "none", tier: str = "L2") -> str: + level = _TIER_LEVEL[tier] + propagate(module) + out = [ + "KERN-IL/0.2", + f"source_rel={source_rel}", + f"source_sha256={source_sha256}", + f"repo_revision={repo_revision}", + f"generator={GENERATOR} lang={module.lang} frontend={module.frontend} tier={tier}", + "", + ] + faults: list[str] = [] + imports = [s for s in module.symbols if s.kind == "import"] + if imports: + lo = min(s.span[0] for s in imports) + hi = max(s.span[1] for s in imports) + out.append(f"IMPORTS {'; '.join(s.detail for s in imports)} @L{lo}-{hi}") + for s in module.symbols: + if s.kind == "const": + tag = "" + if s.risk: + tag = f" !FAULT({s.risk})" + faults.append(f"{s.risk}(L{s.span[0]})") + out.append(f"C {s.name}={s.detail} @L{s.span[0]}{tag}") + for s in module.symbols: + if s.kind == "class": + out.extend(["", f"CLASS {s.name}({s.bases}) @L{s.span[0]}-{s.span[1]} ^{s.slice8}"]) + elif s.kind == "function": + out.append("") + out.extend(_function_lines(s, level, tier, faults)) + omit = " ".join(f"{k}={v}" for k, v in sorted(module.omit.items())) + out.extend([ + "", + f"OMIT {omit} bodies-tier={tier}", + "FAULT-BEFORE edit(any), exact-literals" + + "".join(f", {f}" for f in dict.fromkeys(faults)), + ]) + return "\n".join(out).rstrip() + "\n" +``` + +- [ ] **Step 4: Run all tests** + +Run: `python3 -m unittest discover -s tests -v` +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +git add skills/kern/scripts/kern_compile.py tests/test_emitter.py +git commit -m "feat: tiered KERN-IL/0.2 emitter with fault tags and omit counts" +``` + +--- + +### Task 5: TS/JS tree-sitter frontend (optional dependency) + +**Files:** +- Modify: `skills/kern/scripts/kern_compile.py` (add `tsjs_available`, `parse_tsjs`) +- Test: `tests/test_tsjs.py` + +**Interfaces:** +- Consumes: `Symbol`, `FlowOp`, `ModuleIR`, `emit_il`. +- Produces: `tsjs_available() -> bool`; `parse_tsjs(text: str, typescript: bool = False) -> ModuleIR` with `frontend="tree-sitter"`, `lang="typescript"|"javascript"`. Tests skip when tree-sitter absent. Task 6 dispatches on file suffix. + +- [ ] **Step 1: Try installing the optional dependency** + +Run: `python3 -m pip install --user tree-sitter tree-sitter-javascript tree-sitter-typescript` +If install fails (no network/permission), proceed anyway — tests skip and the fallback path is exercised instead. Do not add these to any requirements file; they stay optional. + +- [ ] **Step 2: Write the failing test** + +Create `tests/test_tsjs.py`: + +```python +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + +TS_SAMPLE = ''' +import { readFile } from "fs/promises"; + +const MAX = 10; + +export class Loader { + async fetch(url: string): Promise { + const data = await readFile(url); + if (!data) { + throw new Error(url); + } + return data.toString(); + } +} + +export function parse(raw: string): number { + return Number(raw); +} +''' + + +class TestAvailability(unittest.TestCase): + def test_available_returns_bool(self): + self.assertIsInstance(kern_compile.tsjs_available(), bool) + + +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestTsFrontend(unittest.TestCase): + def setUp(self): + self.mod = kern_compile.parse_tsjs(TS_SAMPLE, typescript=True) + + def sym(self, name): + return next(s for s in self.mod.symbols if s.name == name) + + def test_module_metadata(self): + self.assertEqual(self.mod.lang, "typescript") + self.assertEqual(self.mod.frontend, "tree-sitter") + + def test_function_and_method(self): + f = self.sym("parse") + self.assertEqual(f.kind, "function") + self.assertIn("raw: string", f.signature) + m = self.sym("Loader.fetch") + self.assertTrue(m.is_async) + self.assertIn("readFile", " ".join(m.calls)) + + def test_flow_and_raises(self): + m = self.sym("Loader.fetch") + ops = [o.op for o in m.flow] + self.assertIn("IF", ops) + self.assertIn("RAISE", ops) + self.assertIn("RET", ops) + self.assertIn("Error", m.raises) + + def test_slice_hash_present(self): + self.assertEqual(len(self.sym("parse").slice8), 8) + + def test_emit_works(self): + il = kern_compile.emit_il(self.mod, "src/x.ts", "b" * 64, "none", "L2") + self.assertTrue(il.startswith("KERN-IL/0.2")) + self.assertIn("F parse", il) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `python3 -m unittest tests.test_tsjs -v` +Expected: FAIL — `AttributeError: ... no attribute 'tsjs_available'` (or, with tree-sitter installed, missing `parse_tsjs`). + +- [ ] **Step 4: Write implementation** + +Add to `kern_compile.py`: + +```python +def tsjs_available() -> bool: + try: + import tree_sitter # noqa: F401 + import tree_sitter_javascript # noqa: F401 + import tree_sitter_typescript # noqa: F401 + return True + except ImportError: + return False + + +_TS_FLOW = { + "if_statement": "IF", "for_statement": "LOOP", "for_in_statement": "LOOP", + "while_statement": "WHILE", "do_statement": "WHILE", "try_statement": "TRY", + "return_statement": "RET", "throw_statement": "RAISE", "switch_statement": "MATCH", +} +_TS_FUNC_NODES = {"function_declaration", "generator_function_declaration", "method_definition"} + + +def parse_tsjs(text: str, typescript: bool = False) -> ModuleIR: + from tree_sitter import Language, Parser + if typescript: + import tree_sitter_typescript as ts_lang + language = Language(ts_lang.language_typescript()) + lang_name = "typescript" + else: + import tree_sitter_javascript as js_lang + language = Language(js_lang.language()) + lang_name = "javascript" + parser = Parser(language) + tree = parser.parse(text.encode("utf-8")) + raw = text.encode("utf-8") + + def ntext(n, cap=120): + piece = raw[n.start_byte:n.end_byte].decode("utf-8", "replace") + piece = SPACE.sub(" ", piece).strip() + if SECRET_VALUE.search(piece): + return sanitize_string(piece, secret_hint=True) + return piece[:cap] + + def span(n): + return (n.start_point[0] + 1, n.end_point[0] + 1) + + def field_node(n, name): + return n.child_by_field_name(name) + + def collect_calls(n, acc): + if n.type == "call_expression": + fn = field_node(n, "function") + if fn is not None: + name = ntext(fn, 80) + if name not in acc: + acc.append(name) + for c in n.children: + collect_calls(c, acc) + + def collect_raises(n, acc): + if n.type == "throw_statement": + body = ntext(n, 80) + name = body.removeprefix("throw").strip().removeprefix("new").strip() + name = name.split("(")[0].rstrip(";").strip() + if name and name not in acc: + acc.append(name) + for c in n.children: + collect_raises(c, acc) + + def flow(n, depth=0, budget=200): + ops = [] + + def add(node, op, detail="", binds=""): + if len(ops) < budget: + ops.append(FlowOp(op=op, detail=detail, binds=binds, depth=depth, + line=node.start_point[0] + 1)) + + for c in n.named_children: + if len(ops) >= budget: + break + t = c.type + if t in _TS_FLOW: + cond = field_node(c, "condition") + detail = ntext(cond, 100).strip("()") if cond is not None else "" + if t == "return_statement": + detail = ntext(c, 100).removeprefix("return").strip().rstrip(";") + if t == "throw_statement": + detail = ntext(c, 80).removeprefix("throw").strip().rstrip(";") + add(c, _TS_FLOW[t], detail) + for name in ("body", "consequence"): + inner = field_node(c, name) + if inner is not None: + ops.extend(flow(inner, depth + 1, budget - len(ops))) + alt = field_node(c, "alternative") + if alt is not None: + add(c, "ELSE") + ops.extend(flow(alt, depth + 1, budget - len(ops))) + handler = field_node(c, "handler") + if handler is not None: + add(handler, "CATCH", ntext(field_node(handler, "parameter") or handler, 40)) + hbody = field_node(handler, "body") + if hbody is not None: + ops.extend(flow(hbody, depth + 1, budget - len(ops))) + elif t == "expression_statement" and c.named_children and c.named_children[0].type in ("call_expression", "await_expression"): + add(c, "CALL", ntext(c.named_children[0], 120)) + elif t in ("lexical_declaration", "variable_declaration"): + for d in c.named_children: + if d.type == "variable_declarator": + value = field_node(d, "value") + if value is not None and value.type in ("call_expression", "await_expression"): + name = field_node(d, "name") + add(d, "CALL", ntext(value, 120), binds=ntext(name, 40) if name is not None else "") + else: + ops.extend(flow(c, depth, budget - len(ops))) + return ops[:budget] + + def function_symbol(node, qualified): + calls, raises = [], [] + collect_calls(node, calls) + collect_raises(node, raises) + params = field_node(node, "parameters") + rtype = field_node(node, "return_type") + body = field_node(node, "body") + a, b = span(node) + is_async = any(ch.type == "async" for ch in node.children) + return Symbol( + kind="function", name=qualified, span=(a, b), + signature=ntext(params, 200).strip("()") if params is not None else "", + returns=ntext(rtype, 60).lstrip(": ") if rtype is not None else "", + slice8=slice_sha8(text, a, b), calls=calls, raises=raises, + flow=flow(body) if body is not None else [], is_async=is_async, + ) + + symbols: list[Symbol] = [] + + def top(n, class_prefix=""): + for c in n.named_children: + t = c.type + if t in ("import_statement",): + symbols.append(Symbol(kind="import", name="", detail=ntext(c, 120), span=span(c))) + elif t in ("lexical_declaration", "variable_declaration") and not class_prefix: + for d in c.named_children: + if d.type == "variable_declarator": + name = field_node(d, "name") + value = field_node(d, "value") + if value is not None and value.type in ("arrow_function", "function_expression"): + symbols.append(function_symbol(value, ntext(name, 60))) + elif name is not None: + detail = ntext(value, 100) if value is not None else "" + hint = bool(SECRET_NAME.search(ntext(name, 60))) + if hint: + detail = sanitize_string(detail, secret_hint=True) + symbols.append(Symbol(kind="const", name=ntext(name, 60), detail=detail, span=span(c))) + elif t in _TS_FUNC_NODES: + name = field_node(c, "name") + qual = (class_prefix + ntext(name, 60)) if name is not None else class_prefix + "" + symbols.append(function_symbol(c, qual)) + elif t == "class_declaration": + name = field_node(c, "name") + cname = ntext(name, 60) if name is not None else "" + a, b = span(c) + symbols.append(Symbol(kind="class", name=cname, span=(a, b), slice8=slice_sha8(text, a, b))) + body = field_node(c, "body") + if body is not None: + top(body, class_prefix=cname + ".") + elif t in ("export_statement", "program", "statement_block"): + top(c, class_prefix) + top(tree.root_node) + + lines = text.splitlines() + omit = { + "docstrings": 0, + "comments": sum(1 for l in lines if l.strip().startswith("//")), + "blank": sum(1 for l in lines if not l.strip()), + "assignments": 0, + } + return ModuleIR(lang_name, "tree-sitter", symbols, omit) +``` + +- [ ] **Step 5: Run tests** + +Run: `python3 -m unittest tests.test_tsjs -v` +Expected: PASS (or SKIP with "tree-sitter not installed" — both acceptable; `TestAvailability` must pass either way). If tree-sitter IS installed and node-type names mismatch the grammar version, debug with `python3 -c "import kern_compile; m = kern_compile.parse_tsjs(open('tests/test_tsjs.py').read().split(\"'''\")[1], True); print([s.name for s in m.symbols])"` and adjust `_TS_FLOW`/node names — grammar node types are stable across recent versions but `parameter` vs `parameters` on catch clauses varies. + +- [ ] **Step 6: Commit** + +```bash +git add skills/kern/scripts/kern_compile.py tests/test_tsjs.py +git commit -m "feat: TS/JS tree-sitter frontend with graceful absence" +``` + +--- + +### Task 6: Integrate compiler into kern_cache (codec bump, dispatch, size floor, --tier) + +**Files:** +- Modify: `skills/kern/scripts/kern_cache.py` +- Test: `tests/test_cache_integration.py` + +**Interfaces:** +- Consumes: `kern_compile.parse_python`, `parse_tsjs`, `tsjs_available`, `emit_il` (local import inside `baseline_for`). +- Produces: `baseline_for(root, source, relative, digest, config, tier=None) -> str` (new signature); `git_revision(root: Path) -> str`; `ensure_file(root, paths, relative, source, config, tier=None)`; `prepare_file(root, paths, relative, source, config)`; constants `CODEC_VERSION = "kern-il/0.2"`, `BASELINE_GENERATOR = "kern-det/0.2"`, `TSJS_SUFFIXES = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"}`; config keys `min_ir_tokens: 600`, `default_tier: "L2"`; manifest field `ir_tier`. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_cache_integration.py`: + +```python +import json +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_cache # noqa: E402 + +BIG_PY = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( + f'def fn_{i}(path, n):\n' + f' """Doc {i}."""\n' + f' data = path.read_bytes()\n' + f' if not data:\n' + f' raise ValueError(n)\n' + f' return json.loads(data)\n' + for i in range(30) +) + + +class TestCacheIntegration(unittest.TestCase): + def setUp(self): + self.root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + (self.root / "big.py").write_text(BIG_PY) + (self.root / "tiny.py").write_text("X = 1\n") + self.paths, self.config = kern_cache.initialize(self.root) + + def ensure(self, name, tier=None): + rel, src = kern_cache.normalize_rel(self.root, name) + return kern_cache.ensure_file(self.root, self.paths, rel, src, self.config, tier=tier) + + def test_codec_is_0_2(self): + self.assertEqual(kern_cache.CODEC_VERSION, "kern-il/0.2") + + def test_big_python_file_gets_deterministic_il(self): + result = self.ensure("big.py") + il = Path(result["ir"]).read_text() + self.assertTrue(il.startswith("KERN-IL/0.2")) + self.assertIn("tier=L2", il) + self.assertIn("F fn_0(", il) + self.assertIn("EFFECTS fs:read", il) + self.assertIn("RAISES ValueError", il) + + def test_tier_override(self): + result = self.ensure("big.py", tier="L1") + il = Path(result["ir"]).read_text() + self.assertIn("tier=L1", il) + self.assertNotIn(" IF", il) + manifest = json.loads((self.paths["manifest"]).read_text()) + self.assertEqual(manifest["files"]["big.py"]["ir_tier"], "L1") + + def test_tiny_file_gets_source_cheaper_stub(self): + result = self.ensure("tiny.py") + il = Path(result["ir"]).read_text() + self.assertIn("mode=source-cheaper", il) + self.assertNotIn("F ", il) + + def test_syntax_error_falls_back_to_generic(self): + (self.root / "broken.py").write_text("def broken(:\n pass\n" * 300) + il = Path(self.ensure("broken.py")["ir"]).read_text() + self.assertIn("mode=generic-line-baseline", il) + + def test_repo_revision_header_present(self): + il = Path(self.ensure("big.py")["ir"]).read_text() + self.assertIn("repo_revision=", il) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python3 -m unittest tests.test_cache_integration -v` +Expected: FAIL — `TypeError: ensure_file() got an unexpected keyword argument` and codec assertion failure. + +- [ ] **Step 3: Modify `kern_cache.py`** + +Apply these changes: + +1. Constants (`kern_cache.py:22-24`): +```python +CODEC_VERSION = "kern-il/0.2" +BASELINE_GENERATOR = "kern-det/0.2" +TSJS_SUFFIXES = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"} +``` +2. `DEFAULT_CONFIG`: add `"min_ir_tokens": 600,` and `"default_tier": "L2",` after `"image_profile": "dense",`. +3. Add after `git_files`: +```python +def git_revision(root: Path) -> str: + try: + head = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--short", "HEAD"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, text=True, + ) + if head.returncode != 0: + return "none" + sha = head.stdout.strip() + dirty = subprocess.run( + ["git", "-C", str(root), "status", "--porcelain"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, text=True, + ) + return f"dirty:{sha}" if dirty.stdout.strip() else sha + except OSError: + return "none" +``` +4. Delete `sanitize_string`, `LiteralSanitizer`, `expr`, `target_text`, `call_name`, `outline`, `function_card`, `python_ir` (lines ~366-568). Keep `SECRET_NAME`, `SECRET_VALUE`, `SPACE`, `redact_line`, `generic_ir`, `GENERIC_KEEP`. +5. Replace `baseline_for` with: +```python +def stub_ir(text: str, relative: str, digest: str) -> str: + lines = [ + CODEC_VERSION.upper(), + f"source_rel={relative}", + f"source_sha256={digest}", + f"generator={BASELINE_GENERATOR}", + "mode=source-cheaper", + f"QA source is ~{max(1, len(text) // 4)} tokens ({len(text.splitlines())} lines), below the IL floor; fault exact source.", + ] + return "\n".join(lines) + "\n" + + +def baseline_for(root: Path, source: Path, relative: str, digest: str, + config: dict[str, Any], tier: str | None = None) -> tuple[str, str]: + """Return (il_text, tier_used).""" + text = source.read_text(encoding="utf-8", errors="replace") + if max(1, len(text) // 4) < int(config.get("min_ir_tokens", 600)): + return stub_ir(text, relative, digest), "stub" + selected = tier or str(config.get("default_tier", "L2")) + note = "generic language fallback" + try: + import kern_compile + suffix = source.suffix.lower() + module = None + if suffix == ".py": + module = kern_compile.parse_python(text) + elif suffix in TSJS_SUFFIXES and kern_compile.tsjs_available(): + module = kern_compile.parse_tsjs(text, typescript=suffix in {".ts", ".tsx"}) + if module is not None: + if module.parse_error: + note = f"parse failed: {module.parse_error}" + else: + revision = git_revision(root) + return kern_compile.emit_il(module, relative, digest, revision, selected), selected + except Exception as exc: + note = f"deterministic compiler failed: {exc}" + return generic_ir(text, relative, digest, note), "generic" +``` +Note: the IL first line is `KERN-IL/0.2` from both `emit_il` and `generic_ir`/`stub_ir` (they use `CODEC_VERSION.upper()`), so `commit_file`'s existing first-line check keeps working unchanged. +6. `ensure_file`: change signature to `def ensure_file(root, paths, relative, source, config, tier=None)`; replace the two lines +```python + ir = baseline_for(source, relative, digest) +``` +with +```python + ir, tier_used = baseline_for(root, source, relative, digest, config, tier) +``` +and inside the manifest update dict add `"ir_tier": tier_used,` next to `"ir_generator": BASELINE_GENERATOR,`. Also: when `tier` is explicitly passed and differs from the recorded `record.get("ir_tier")`, treat the cache as unusable (add `and (tier is None or record.get("ir_tier") == tier)` to the `usable = (...)` expression). +7. `prepare_file`: change signature to `(root, paths, relative, source, config)` and its call to `ensure_file(root, paths, relative, source, config)`. +8. `sync_cache`: change its `ensure_file(root, paths, relative, source)` call to `ensure_file(root, paths, relative, source, config)`. +9. `parse_args`: on the `ensure` subparser add `command.add_argument("--tier", choices=("L1", "L2", "L3"))` — restructure the `for name in ("ensure", "prepare", "paths"):` loop into individual subparser blocks so only `ensure` gets `--tier`. +10. `main`: `ensure` branch becomes `result = ensure_file(root, paths, relative, source, config, tier=getattr(args, "tier", None))`; `prepare` branch passes `config`. + +- [ ] **Step 4: Run all tests + compile check** + +Run: `python3 -m unittest discover -s tests -v && python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/kern_compile.py skills/kern/scripts/render_ir.py` +Expected: all PASS, no compile errors. + +- [ ] **Step 5: Smoke-test on this repo** + +Run: `python3 skills/kern/scripts/kern_cache.py --repo . scan && python3 skills/kern/scripts/kern_cache.py --repo . ensure skills/kern/scripts/kern_cache.py && head -20 .kern/ir/skills/kern/scripts/kern_cache.py.kern-il.txt && rm -rf .kern` +Expected: JSON success output; IL starts with `KERN-IL/0.2`, contains `tier=L2` and `F ` lines. + +- [ ] **Step 6: Commit** + +```bash +git add skills/kern/scripts/kern_cache.py tests/test_cache_integration.py +git commit -m "feat: kern-il/0.2 codec, deterministic baseline dispatch, size floor, --tier" +``` + +--- + +### Task 7: `verify` CLI verb + +**Files:** +- Modify: `skills/kern/scripts/kern_cache.py` (add `verify_symbol`, CLI wiring) +- Test: `tests/test_verify.py` + +**Interfaces:** +- Consumes: `kern_compile.parse_python` / `parse_tsjs` (local import), Task 6 constants. +- Produces: `verify_symbol(root, paths, relative, source, symbol: str, expected_hash: str, expected_span: str | None) -> dict` with `result` ∈ `"ok" | "moved" | "stale"`; CLI `kern_cache.py --repo R verify FILE --symbol NAME --hash HHHHHHHH [--span La-b]`. Task 9 documents it in SKILL.md. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_verify.py`: + +```python +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_cache # noqa: E402 +import kern_compile # noqa: E402 + +SRC = '''import json + + +def load_entry(path, expected_sha): + data = path.read_bytes() + if not data: + raise ValueError(path) + return json.loads(data) +''' + + +class TestVerify(unittest.TestCase): + def setUp(self): + self.root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + self.file = self.root / "mod.py" + self.file.write_text(SRC) + self.paths, self.config = kern_cache.initialize(self.root) + mod = kern_compile.parse_python(SRC) + self.sym = next(s for s in mod.symbols if s.name == "load_entry") + + def verify(self, expected_hash, span=None): + rel, src = kern_cache.normalize_rel(self.root, "mod.py") + return kern_cache.verify_symbol(self.root, self.paths, rel, src, + "load_entry", expected_hash, span) + + def test_ok(self): + r = self.verify(self.sym.slice8, f"L{self.sym.span[0]}-{self.sym.span[1]}") + self.assertEqual(r["result"], "ok") + + def test_moved_when_file_shifts(self): + self.file.write_text("# new comment line\n" + SRC) + r = self.verify(self.sym.slice8, f"L{self.sym.span[0]}-{self.sym.span[1]}") + self.assertEqual(r["result"], "moved") + self.assertIn("current_span", r) + + def test_stale_when_body_changes(self): + self.file.write_text(SRC.replace("json.loads(data)", "json.loads(data.strip())")) + r = self.verify(self.sym.slice8) + self.assertEqual(r["result"], "stale") + self.assertEqual(r["reason"], "symbol-bytes-changed") + + def test_stale_when_symbol_deleted(self): + self.file.write_text("import json\n") + r = self.verify(self.sym.slice8) + self.assertEqual(r["result"], "stale") + self.assertEqual(r["reason"], "symbol-not-found") + + def test_unsupported_suffix_raises(self): + (self.root / "x.rb").write_text("def x; end\n" * 200) + rel, src = kern_cache.normalize_rel(self.root, "x.rb") + with self.assertRaises(ValueError): + kern_cache.verify_symbol(self.root, self.paths, rel, src, "x", "deadbeef", None) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python3 -m unittest tests.test_verify -v` +Expected: FAIL — `AttributeError: ... no attribute 'verify_symbol'` + +- [ ] **Step 3: Write implementation** + +Add to `kern_cache.py` (after `fault_source`): + +```python +def verify_symbol(root: Path, paths: dict[str, Path], relative: str, source: Path, + symbol: str, expected_hash: str, expected_span: str | None = None) -> dict[str, Any]: + import kern_compile + text = source.read_text(encoding="utf-8", errors="replace") + suffix = source.suffix.lower() + if suffix == ".py": + module = kern_compile.parse_python(text) + elif suffix in TSJS_SUFFIXES and kern_compile.tsjs_available(): + module = kern_compile.parse_tsjs(text, typescript=suffix in {".ts", ".tsx"}) + else: + raise ValueError(f"verify does not support {suffix or 'this file type'}; use fault with --expect-sha") + if module.parse_error: + raise RuntimeError(f"current source does not parse ({module.parse_error}); fault exact source") + base = {"ok": True, "operation": "verify", "source_rel": relative, "symbol": symbol, + "source_sha256": sha256_bytes(text.encode("utf-8", "surrogatepass"))} + matches = [s for s in module.symbols if s.kind in {"function", "class"} and s.name == symbol] + if not matches: + return {**base, "result": "stale", "reason": "symbol-not-found"} + found = matches[0] + current_span = f"L{found.span[0]}-{found.span[1]}" + if found.slice8 != expected_hash: + return {**base, "result": "stale", "reason": "symbol-bytes-changed", + "current_hash": found.slice8, "current_span": current_span} + if expected_span and expected_span != current_span: + return {**base, "result": "moved", "current_span": current_span} + return {**base, "result": "ok", "current_span": current_span} +``` + +CLI wiring in `parse_args`: +```python + verify = sub.add_parser("verify") + verify.add_argument("file") + verify.add_argument("--symbol", required=True) + verify.add_argument("--hash", required=True) + verify.add_argument("--span") +``` +and in `main`, inside the file-command branch: +```python + elif args.command == "verify": + result = verify_symbol(root, paths, relative, source, args.symbol, args.hash, args.span) +``` + +- [ ] **Step 4: Run all tests** + +Run: `python3 -m unittest discover -s tests -v` +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +git add skills/kern/scripts/kern_cache.py tests/test_verify.py +git commit -m "feat: verify verb traps stale symbol reads" +``` + +--- + +### Task 8: Token benchmark harness + +**Files:** +- Create: `benchmarks/token_bench.py` +- Test: `tests/test_token_bench.py` + +**Interfaces:** +- Consumes: `kern_compile.parse_python`, `emit_il`. +- Produces: `benchmarks/token_bench.py` CLI: `python3 benchmarks/token_bench.py FILE [FILE...] [--out results.json]`. Report schema: `{"schema": "kern-bench/0.2", "estimator": "chars/4", "files": [{"file", "source_tokens", "bucket", "tiers": {"L1": {"tokens", "ratio"}, ...}, "fidelity_missing": [...]}]}`. Importable functions `estimate_tokens(text)`, `bench_file(path)`. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_token_bench.py`: + +```python +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "skills" / "kern" / "scripts")) + +spec = importlib.util.spec_from_file_location("token_bench", REPO / "benchmarks" / "token_bench.py") +token_bench = importlib.util.module_from_spec(spec) +spec.loader.exec_module(token_bench) + +BIG = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( + f'def fn_{i}(path):\n' + f' """Doc {i} with a somewhat longer explanatory sentence to add bulk."""\n' + f' # a comment line adding source-only weight\n' + f' data = path.read_bytes()\n' + f' if not data:\n' + f' raise ValueError(path)\n' + f' return json.loads(data)\n' + for i in range(40) +) + + +class TestTokenBench(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.f = self.tmp / "big.py" + self.f.write_text(BIG) + + def test_bench_file_shape(self): + row = token_bench.bench_file(self.f) + self.assertIn("source_tokens", row) + self.assertEqual(set(row["tiers"]), {"L1", "L2", "L3"}) + for tier in row["tiers"].values(): + self.assertGreater(tier["ratio"], 1.0) + + def test_tier_ordering(self): + row = token_bench.bench_file(self.f) + self.assertGreater(row["tiers"]["L1"]["ratio"], row["tiers"]["L2"]["ratio"]) + self.assertGreater(row["tiers"]["L2"]["ratio"], row["tiers"]["L3"]["ratio"]) + + def test_fidelity_no_missing_functions(self): + row = token_bench.bench_file(self.f) + self.assertEqual(row["fidelity_missing"], []) + + def test_parse_error_reported_not_raised(self): + bad = self.tmp / "bad.py" + bad.write_text("def broken(:\n") + row = token_bench.bench_file(bad) + self.assertIn("error", row) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python3 -m unittest tests.test_token_bench -v` +Expected: FAIL — `FileNotFoundError` for `benchmarks/token_bench.py`. + +- [ ] **Step 3: Write implementation** + +Create `benchmarks/token_bench.py`: + +```python +#!/usr/bin/env python3 +"""Token benchmark: source vs deterministic KERN-IL per tier, bucketed by size.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + + +def estimate_tokens(text: str) -> int: + return max(1, len(text) // 4) + + +def bucket(tokens: int) -> str: + if tokens < 2_000: + return "small(<2k)" + if tokens < 10_000: + return "medium(2k-10k)" + return "large(>10k)" + + +def fidelity_missing(module, il: str) -> list[str]: + missing = [] + for symbol in module.symbols: + if symbol.kind == "function" and symbol.name.split(".")[-1] not in il: + missing.append(symbol.name) + return missing + + +def bench_file(path: Path) -> dict: + text = path.read_text(encoding="utf-8", errors="replace") + module = kern_compile.parse_python(text) + if module.parse_error: + return {"file": str(path), "error": module.parse_error} + source_tokens = estimate_tokens(text) + row = {"file": str(path), "source_tokens": source_tokens, + "bucket": bucket(source_tokens), "tiers": {}, "fidelity_missing": []} + for tier in ("L1", "L2", "L3"): + il = kern_compile.emit_il(module, path.name, "0" * 64, "none", tier) + il_tokens = estimate_tokens(il) + row["tiers"][tier] = {"tokens": il_tokens, "ratio": round(source_tokens / il_tokens, 2)} + if tier == "L2": + row["fidelity_missing"] = fidelity_missing(module, il) + return row + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("files", nargs="+", type=Path) + parser.add_argument("--out", type=Path, help="Write JSON report here as well as stdout") + args = parser.parse_args() + report = {"schema": "kern-bench/0.2", "estimator": "chars/4", + "files": [bench_file(f) for f in args.files]} + payload = json.dumps(report, indent=2, sort_keys=True) + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(payload + "\n") + print(payload) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 4: Run tests and a real benchmark** + +Run: `python3 -m unittest tests.test_token_bench -v` +Expected: PASS. +Run: `python3 benchmarks/token_bench.py skills/kern/scripts/kern_cache.py --out benchmarks/results/python-det-v2.json` +Expected: JSON report; L2 ratio on `kern_cache.py` should be ≥ 3× (it is a medium file; the ≥6× acceptance target applies to the large bucket only). + +- [ ] **Step 5: Commit** + +```bash +git add benchmarks/token_bench.py benchmarks/results/python-det-v2.json tests/test_token_bench.py +git commit -m "feat: per-tier token benchmark with fidelity check" +``` + +--- + +### Task 9: Documentation and skill contract + +**Files:** +- Modify: `skills/kern/SKILL.md` +- Modify: `docs/architecture.md` +- Modify: `README.md` (Development section + Language coverage sentence) +- Modify: `CHANGELOG.md` + +**Interfaces:** +- Consumes: everything shipped in Tasks 1-8. +- Produces: user-facing contract for `verify`, tiers, and the deterministic pipeline. + +- [ ] **Step 1: Update `skills/kern/SKILL.md`** + +In the "JIT a requested file" section, after the `ensure` code block, add: + +```markdown +`ensure` accepts `--tier L1|L2|L3` (default from config `default_tier`, `L2`). +L1 = signatures + calls + effects + raises; L2 = + control-flow skeleton; +L3 = + expressions and dataflow. Files below the `min_ir_tokens` floor get a +`mode=source-cheaper` stub — read exact source instead. +``` + +In the "Fault exact source before edits" section, add before the existing code block: + +```markdown +Before editing any symbol read from IL or an image, verify its source-map handle: + + python3 /scripts/kern_cache.py --repo verify path/to/file \ + --symbol --hash [--span L-L] + +`ok` — proceed. `moved` — same bytes at a new span; use the returned span. +`stale` — the symbol changed; the IL page is invalid, fault exact source. +Lines tagged `!FAULT(reason)` (regex, math, concurrency, elided-literal) may not +support a claim or an edit without an exact-source fault, regardless of verify. +``` + +- [ ] **Step 2: Update `docs/architecture.md`** + +Add invariant `7. Every IL symbol carries a slice hash; a passing verify or fresh fault is required before that symbol is edited.` and change the Lifecycle line to `scan → hash → invalidate → compile(tiered) → render → page in → verify/fault → write → invalidate`. + +- [ ] **Step 3: Update `README.md`** + +In "Language and runtime coverage", replace the first sentence with: + +```markdown +Python and (when tree-sitter is installed) JavaScript/TypeScript receive a deterministic +AST-based compiler with tiered detail, computed side effects, and exception propagation. +Other recognized source formats receive a deterministic generic baseline. +``` + +In "Development", replace the commands block with: + +```bash +npm ci +npm run build +python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/render_ir.py skills/kern/scripts/kern_compile.py +python3 -m unittest discover -s tests +python3 skills/kern/scripts/kern_cache.py --repo . scan +``` + +- [ ] **Step 4: Update `CHANGELOG.md`** + +Add an entry at the top following the file's existing format: codec `kern-il/0.2`, deterministic tiered compiler, effects/raises, verify verb, size floor, token benchmark. Bump the minor version consistent with the file's convention. + +- [ ] **Step 5: Full check** + +Run: `python3 -m unittest discover -s tests -v && python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/render_ir.py skills/kern/scripts/kern_compile.py` +Expected: all PASS. + +- [ ] **Step 6: Commit** + +```bash +git add skills/kern/SKILL.md docs/architecture.md README.md CHANGELOG.md +git commit -m "docs: kern-il/0.2 tiers, verify contract, dev commands" +``` + +--- + +### Task 10: Enrichment is append-only (commit_file contract) + +**Files:** +- Modify: `skills/kern/scripts/kern_cache.py:691-764` (`commit_file`) +- Modify: `skills/kern/references/compiler-worker.md` +- Test: `tests/test_enrichment.py` + +**Interfaces:** +- Consumes: Task 6 `ensure_file`/`baseline_for` (deterministic IL on disk before enrichment). +- Produces: `commit_file` rejects any staging IL that does not consist of the current committed deterministic IL followed by an `ENRICHMENT model=` section containing only `INTENT : ` lines. Spec rule: model output never replaces deterministic facts. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_enrichment.py`: + +```python +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_cache # noqa: E402 + +BIG_PY = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( + f'def fn_{i}(path):\n' + f' data = path.read_bytes()\n' + f' return json.loads(data)\n' + for i in range(40) +) + + +class TestEnrichmentAppendOnly(unittest.TestCase): + def setUp(self): + self.root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + (self.root / "mod.py").write_text(BIG_PY) + self.paths, self.config = kern_cache.initialize(self.root) + rel, src = kern_cache.normalize_rel(self.root, "mod.py") + self.rel, self.src = rel, src + ensured = kern_cache.ensure_file(self.root, self.paths, rel, src, self.config) + self.digest = ensured["source_sha256"] + self.baseline = Path(ensured["ir"]).read_text() + + def commit(self, staging_text): + staging = self.root / "staging.kern-il.txt" + staging.write_text(staging_text) + return kern_cache.commit_file(self.root, self.paths, self.rel, self.src, + staging, self.digest) + + def test_valid_append_accepted(self): + staged = self.baseline + "\nENRICHMENT model=test-model\nINTENT fn_0: reads and parses a JSON file\n" + result = self.commit(staged) + self.assertEqual(result["status"], "ready") + + def test_replacement_rejected(self): + rogue = self.baseline.replace("F fn_0(", "F totally_different(") + with self.assertRaises(ValueError): + self.commit(rogue + "\nENRICHMENT model=test-model\nINTENT fn_0: x\n") + + def test_missing_enrichment_header_rejected(self): + with self.assertRaises(ValueError): + self.commit(self.baseline + "\nINTENT fn_0: no header line\n") + + def test_non_intent_lines_rejected(self): + staged = self.baseline + "\nENRICHMENT model=test-model\nF injected_fact() -> Any @L1-1 ^deadbeef ~L2\n" + with self.assertRaises(ValueError): + self.commit(staged) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python3 -m unittest tests.test_enrichment -v` +Expected: FAIL — `test_replacement_rejected` and the others fail because `commit_file` currently accepts any well-headed IL. + +- [ ] **Step 3: Modify `commit_file`** + +In `commit_file`, after the existing header validation (`generator` check) and before `atomic_write`, add: + +```python + baseline_path = artifact_paths(paths, relative)["ir"] + if not baseline_path.is_file(): + raise ValueError("No deterministic baseline IL exists; run ensure before commit") + baseline_text = baseline_path.read_text(encoding="utf-8") + if not text.startswith(baseline_text.rstrip("\n")): + raise ValueError("Enrichment must preserve the deterministic IL verbatim as a prefix") + appended = text[len(baseline_text.rstrip("\n")):].strip("\n") + if appended: + appended_lines = appended.splitlines() + if not appended_lines[0].startswith("ENRICHMENT model="): + raise ValueError("Appended section must start with 'ENRICHMENT model='") + for line in appended_lines[1:]: + if line.strip() and not line.startswith("INTENT "): + raise ValueError(f"Enrichment may only append INTENT lines, found: {line[:60]!r}") +``` + +Note: `artifact_paths` is already imported in scope; `text` is the decoded staging payload already available in `commit_file`. + +- [ ] **Step 4: Rewrite `skills/kern/references/compiler-worker.md`** + +Replace the enrichment contract with the new one (keep the file's existing tone/format): + +```markdown +# Compiler worker contract (enrichment) + +The deterministic IL is authoritative and already committed. Your job is ONLY to +append intent summaries. Output the baseline IL verbatim, then: + + ENRICHMENT model= + INTENT : + +Rules: +- Never modify, reorder, or omit any deterministic line. The commit will be + rejected if the baseline is not a byte-exact prefix of your output. +- Only `INTENT` lines may follow the `ENRICHMENT` header. +- One INTENT line per symbol, at most; skip symbols whose purpose is obvious. +- Never include secrets, credentials, or long literals. +``` + +- [ ] **Step 5: Run all tests** + +Run: `python3 -m unittest discover -s tests -v` +Expected: all PASS. + +- [ ] **Step 6: Commit** + +```bash +git add skills/kern/scripts/kern_cache.py skills/kern/references/compiler-worker.md tests/test_enrichment.py +git commit -m "feat: enrichment is append-only INTENT lines over deterministic IL" +``` From e9018601c5e31aaeb0f0c18486a4e22d6a5c6bfe Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:10:46 +0200 Subject: [PATCH 03/55] feat: kern_compile symbol model and Python frontend Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 198 ++++++++++++++++++++++++++++ tests/__init__.py | 0 tests/test_kern_compile.py | 102 ++++++++++++++ 3 files changed, 300 insertions(+) create mode 100644 skills/kern/scripts/kern_compile.py create mode 100644 tests/__init__.py create mode 100644 tests/test_kern_compile.py diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py new file mode 100644 index 0000000..39918ad --- /dev/null +++ b/skills/kern/scripts/kern_compile.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Deterministic KERN-IL/0.2 compiler: language frontends, effect engine, tiered emitter.""" + +from __future__ import annotations + +import ast +import copy +import hashlib +import re +from dataclasses import dataclass, field + +CODEC_VERSION = "kern-il/0.2" +GENERATOR = "kern-det/0.2" + +SECRET_NAME = re.compile( + r"(?i)(api[_-]?key|access[_-]?key|auth|bearer|credential|passwd|password|private[_-]?key|secret|token)" +) +SECRET_VALUE = re.compile( + r"(?i)(?:sk|rk|pk|s2)[_-][A-Za-z0-9_-]{12,}|-----BEGIN [A-Z ]*PRIVATE KEY-----|" + r"(?:aws|ghp|github_pat)_[A-Za-z0-9_-]{12,}" +) +SPACE = re.compile(r"\s+") + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def slice_sha8(source_text: str, start: int, end: int) -> str: + lines = source_text.splitlines(keepends=True) + return sha256_hex("".join(lines[start - 1:end]).encode("utf-8", "surrogatepass"))[:8] + + +def sanitize_string(value: str, secret_hint: bool = False) -> str: + digest = sha256_hex(value.encode("utf-8", "surrogatepass"))[:12] + if secret_hint or SECRET_VALUE.search(value): + return f"" + if len(value) > 160: + return f"" + return value + + +class _LiteralSanitizer(ast.NodeTransformer): + def visit_Constant(self, node: ast.Constant): + if isinstance(node.value, str): + return ast.copy_location(ast.Constant(sanitize_string(node.value)), node) + return node + + +def expr_text(node: ast.AST | None, max_length: int = 200, secret_hint: bool = False) -> str: + if node is None: + return "None" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + rendered = repr(sanitize_string(node.value, secret_hint)) + else: + clone = _LiteralSanitizer().visit(copy.deepcopy(node)) + ast.fix_missing_locations(clone) + try: + rendered = ast.unparse(clone) + except Exception: + rendered = f"<{node.__class__.__name__}>" + rendered = SPACE.sub(" ", rendered).strip() + if secret_hint and not rendered.startswith("'" + if len(rendered) > max_length: + digest = sha256_hex(rendered.encode())[:12] + rendered = rendered[: max_length - 24] + f"…" + return rendered + + +def _target(node: ast.AST) -> str: + try: + return SPACE.sub(" ", ast.unparse(node)).strip() + except Exception: + return f"<{node.__class__.__name__}>" + + +@dataclass +class FlowOp: + op: str + detail: str = "" + binds: str = "" + depth: int = 0 + line: int = 0 + risk: str = "" + + +@dataclass +class Symbol: + kind: str # function | class | const | import + name: str + span: tuple = (0, 0) + signature: str = "" + returns: str = "" + decorators: list = field(default_factory=list) + slice8: str = "" + calls: list = field(default_factory=list) + raises: list = field(default_factory=list) + flow: list = field(default_factory=list) + is_async: bool = False + bases: str = "" + detail: str = "" + risk: str = "" + effects: dict = field(default_factory=dict) # effect -> list of via names ([] = direct) + raises_all: dict = field(default_factory=dict) # exception -> list of via names + unknown_calls: int = 0 + + +@dataclass +class ModuleIR: + lang: str + frontend: str + symbols: list + omit: dict + parse_error: str = "" + + +def _import_detail(node: ast.Import | ast.ImportFrom) -> str: + if isinstance(node, ast.Import): + return ", ".join(a.name for a in node.names) + names = ",".join(a.name for a in node.names) + return f"{node.module or '.'}:{names}" + + +def _omit_counts(text: str, tree: ast.Module) -> dict: + lines = text.splitlines() + comments = sum(1 for l in lines if l.strip().startswith("#")) + blank = sum(1 for l in lines if not l.strip()) + docstrings = 0 + for node in ast.walk(tree): + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + if ast.get_docstring(node) is not None: + docstrings += 1 + assigns = sum(1 for n in ast.walk(tree) if isinstance(n, (ast.Assign, ast.AnnAssign, ast.AugAssign))) + return {"docstrings": docstrings, "comments": comments, "blank": blank, "assignments": assigns} + + +def _function_symbol(node, qualified: str, text: str) -> Symbol: + calls: list[str] = [] + raises: list[str] = [] + for ch in ast.walk(node): + if isinstance(ch, ast.Call): + try: + name = expr_text(ch.func, 80) + except Exception: + name = "" + if name not in calls: + calls.append(name) + elif isinstance(ch, ast.Raise) and ch.exc is not None: + name = expr_text(ch.exc, 60).split("(")[0] + if name not in raises: + raises.append(name) + start = min([d.lineno for d in node.decorator_list] + [node.lineno]) + end = node.end_lineno or node.lineno + return Symbol( + kind="function", + name=qualified, + span=(start, end), + signature=expr_text(node.args, 200), + returns=expr_text(node.returns, 60) if node.returns else "", + decorators=[expr_text(d, 60) for d in node.decorator_list], + slice8=slice_sha8(text, start, end), + calls=calls, + raises=raises, + flow=[], # filled by flow_ops in Task 2 + is_async=isinstance(node, ast.AsyncFunctionDef), + ) + + +def parse_python(text: str) -> ModuleIR: + try: + tree = ast.parse(text) + except SyntaxError as exc: + return ModuleIR("python", "pyast", [], {}, parse_error=f"L{exc.lineno}: {exc.msg}") + symbols: list[Symbol] = [] + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + symbols.append(Symbol(kind="import", name="", detail=_import_detail(node), + span=(node.lineno, node.end_lineno or node.lineno))) + elif isinstance(node, (ast.Assign, ast.AnnAssign)): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + names = ",".join(_target(t) for t in targets) + hint = bool(SECRET_NAME.search(names)) + symbols.append(Symbol(kind="const", name=names, + detail=expr_text(node.value, 100, hint), + span=(node.lineno, node.end_lineno or node.lineno))) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + symbols.append(_function_symbol(node, node.name, text)) + elif isinstance(node, ast.ClassDef): + end = node.end_lineno or node.lineno + symbols.append(Symbol(kind="class", name=node.name, + bases=",".join(expr_text(b, 60) for b in node.bases), + span=(node.lineno, end), slice8=slice_sha8(text, node.lineno, end))) + for member in node.body: + if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): + symbols.append(_function_symbol(member, f"{node.name}.{member.name}", text)) + return ModuleIR("python", "pyast", symbols, _omit_counts(text, tree)) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_kern_compile.py b/tests/test_kern_compile.py new file mode 100644 index 0000000..1d1cf7c --- /dev/null +++ b/tests/test_kern_compile.py @@ -0,0 +1,102 @@ +import hashlib +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + +SAMPLE = '''"""Module docstring.""" + +import json +from hashlib import sha256 +from pathlib import Path + +MANIFEST_NAME = "manifest.json" +API_KEY = "s2_abcdefghijklmnop1234" + + +class StaleSource(Exception): + """Raised on hash mismatch.""" + + +def load_entry(path: Path, expected_sha: str) -> dict: + """Read, verify, parse.""" + data = path.read_bytes() + current_sha = sha256(data).hexdigest() + if current_sha != expected_sha: + raise StaleSource(path) + return json.loads(data) + + +class Loader: + async def fetch(self, url): + return await self.client.get(url) +''' + + +class TestPythonFrontend(unittest.TestCase): + def setUp(self): + self.mod = kern_compile.parse_python(SAMPLE) + + def sym(self, name): + return next(s for s in self.mod.symbols if s.name == name) + + def test_module_metadata(self): + self.assertEqual(self.mod.lang, "python") + self.assertEqual(self.mod.frontend, "pyast") + self.assertEqual(self.mod.parse_error, "") + + def test_function_symbol(self): + f = self.sym("load_entry") + self.assertEqual(f.kind, "function") + self.assertIn("path: Path", f.signature) + self.assertIn("expected_sha: str", f.signature) + self.assertEqual(f.returns, "dict") + self.assertFalse(f.is_async) + self.assertIn("path.read_bytes", f.calls) + self.assertIn("json.loads", f.calls) + self.assertEqual(f.raises, ["StaleSource"]) + + def test_slice_hash_matches_exact_source_lines(self): + f = self.sym("load_entry") + start, end = f.span + lines = SAMPLE.splitlines(keepends=True) + expected = hashlib.sha256("".join(lines[start - 1:end]).encode()).hexdigest()[:8] + self.assertEqual(f.slice8, expected) + self.assertEqual(SAMPLE.splitlines()[start - 1].strip(), "def load_entry(path: Path, expected_sha: str) -> dict:") + + def test_method_is_qualified_and_async(self): + m = self.sym("Loader.fetch") + self.assertTrue(m.is_async) + + def test_class_symbol(self): + c = self.sym("StaleSource") + self.assertEqual(c.kind, "class") + self.assertEqual(c.bases, "Exception") + self.assertEqual(len(c.slice8), 8) + + def test_secret_const_redacted(self): + consts = [s for s in self.mod.symbols if s.kind == "const"] + api = next(s for s in consts if s.name == "API_KEY") + self.assertNotIn("s2_abcdefghijklmnop1234", api.detail) + self.assertIn("REDACTED", api.detail) + + def test_imports_collected(self): + imports = [s for s in self.mod.symbols if s.kind == "import"] + details = " ".join(s.detail for s in imports) + self.assertIn("json", details) + self.assertIn("sha256", details) + + def test_omit_counts(self): + self.assertGreaterEqual(self.mod.omit["docstrings"], 3) + self.assertGreaterEqual(self.mod.omit["blank"], 5) + + def test_parse_error_reported(self): + bad = kern_compile.parse_python("def broken(:\n") + self.assertNotEqual(bad.parse_error, "") + self.assertEqual(bad.symbols, []) + + +if __name__ == "__main__": + unittest.main() From 7eaa5a53be4371c431ce389d688e2b72a923ce17 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:15:42 +0200 Subject: [PATCH 04/55] fix: decorated classes carry decorator span and list ClassDef branch in parse_python now computes decorator span consistently with function handling: start includes decorator lines, slice_sha8 spans from decorator to end, and Symbol.decorators field is populated. Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 4 +++- tests/test_kern_compile.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 39918ad..37ed89a 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -188,10 +188,12 @@ def parse_python(text: str) -> ModuleIR: elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): symbols.append(_function_symbol(node, node.name, text)) elif isinstance(node, ast.ClassDef): + start = min([d.lineno for d in node.decorator_list] + [node.lineno]) end = node.end_lineno or node.lineno symbols.append(Symbol(kind="class", name=node.name, bases=",".join(expr_text(b, 60) for b in node.bases), - span=(node.lineno, end), slice8=slice_sha8(text, node.lineno, end))) + span=(start, end), slice8=slice_sha8(text, start, end), + decorators=[expr_text(d, 60) for d in node.decorator_list])) for member in node.body: if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): symbols.append(_function_symbol(member, f"{node.name}.{member.name}", text)) diff --git a/tests/test_kern_compile.py b/tests/test_kern_compile.py index 1d1cf7c..de1b172 100644 --- a/tests/test_kern_compile.py +++ b/tests/test_kern_compile.py @@ -97,6 +97,13 @@ def test_parse_error_reported(self): self.assertNotEqual(bad.parse_error, "") self.assertEqual(bad.symbols, []) + def test_decorated_class_span_includes_decorator(self): + src = "import functools\n\n@functools.total_ordering\nclass Ordered:\n pass\n" + mod = kern_compile.parse_python(src) + c = next(s for s in mod.symbols if s.name == "Ordered") + self.assertEqual(c.span[0], 3) + self.assertEqual(c.decorators, ["functools.total_ordering"]) + if __name__ == "__main__": unittest.main() From 5b81a92c87ca5e39cf5c1dd97bfca6a1e00d98f4 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:18:32 +0200 Subject: [PATCH 05/55] feat: control-flow extraction with risk tags Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 112 +++++++++++++++++++++++++++- tests/test_kern_compile.py | 70 +++++++++++++++++ 2 files changed, 181 insertions(+), 1 deletion(-) diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 37ed89a..fc20673 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -136,6 +136,116 @@ def _omit_counts(text: str, tree: ast.Module) -> dict: return {"docstrings": docstrings, "comments": comments, "blank": blank, "assignments": assigns} +_RISK_CALL = [ + ("regex", re.compile(r"^re\.(compile|match|search|sub|split|fullmatch|findall|finditer)$")), + ("crypto", re.compile(r"^(hashlib|hmac|secrets)\.")), + ("concurrency", re.compile(r"^(threading|asyncio|multiprocessing|concurrent)\.")), +] +_RISK_MATH = re.compile(r"(\*\*|<<|>>)") +_TRY_TYPES = (ast.Try, getattr(ast, "TryStar", ast.Try)) + + +def expr_risk(node: ast.AST | None) -> str: + if node is None: + return "" + for ch in ast.walk(node): + if isinstance(ch, ast.Call): + try: + fn = ast.unparse(ch.func) + except Exception: + continue + for name, rx in _RISK_CALL: + if rx.search(fn): + return name + elif isinstance(ch, ast.withitem): + try: + fn = ast.unparse(ch.context_expr) + except Exception: + continue + if fn.startswith(("threading.", "asyncio.", "multiprocessing.")): + return "concurrency" + try: + if _RISK_MATH.search(ast.unparse(node)): + return "math" + except Exception: + pass + return "" + + +def flow_ops(statements: list, depth: int = 0, budget: int = 200) -> list: + ops: list[FlowOp] = [] + + def add(node, op, detail="", binds="", risk=""): + if len(ops) < budget: + ops.append(FlowOp(op=op, detail=detail, binds=binds, depth=depth, + line=getattr(node, "lineno", 0), risk=risk)) + + def sub(body): + return flow_ops(body, depth + 1, budget - len(ops)) + + for s in statements: + if len(ops) >= budget: + break + if isinstance(s, ast.Expr) and isinstance(s.value, ast.Constant) and isinstance(s.value.value, str): + continue # docstring + if isinstance(s, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + add(s, "NESTED", s.name) + elif isinstance(s, (ast.Assign, ast.AnnAssign)) and isinstance(getattr(s, "value", None), (ast.Call, ast.Await)): + targets = s.targets if isinstance(s, ast.Assign) else [s.target] + names = ",".join(_target(t) for t in targets) + call = s.value.value if isinstance(s.value, ast.Await) else s.value + hint = bool(SECRET_NAME.search(names)) + add(s, "CALL", expr_text(call, 120, hint), binds=names, risk=expr_risk(call)) + elif isinstance(s, ast.If): + add(s, "IF", expr_text(s.test, 100), risk=expr_risk(s.test)) + ops.extend(sub(s.body)) + if s.orelse: + add(s, "ELSE") + ops.extend(sub(s.orelse)) + elif isinstance(s, (ast.For, ast.AsyncFor)): + add(s, "LOOP", f"{_target(s.target)} in {expr_text(s.iter, 100)}") + ops.extend(sub(s.body)) + elif isinstance(s, ast.While): + add(s, "WHILE", expr_text(s.test, 100), risk=expr_risk(s.test)) + ops.extend(sub(s.body)) + elif isinstance(s, (ast.With, ast.AsyncWith)): + detail = ", ".join(expr_text(i.context_expr, 80) for i in s.items) + risk = "" + for i in s.items: + risk = risk or expr_risk(i.context_expr) + add(s, "WITH", detail, risk=risk) + ops.extend(sub(s.body)) + elif isinstance(s, _TRY_TYPES): + add(s, "TRY") + ops.extend(sub(s.body)) + for h in s.handlers: + add(h, "CATCH", expr_text(h.type, 60)) + ops.extend(sub(h.body)) + if s.finalbody: + add(s, "FINALLY") + ops.extend(sub(s.finalbody)) + elif isinstance(s, ast.Return): + add(s, "RET", expr_text(s.value, 100), risk=expr_risk(s.value)) + elif isinstance(s, ast.Raise): + add(s, "RAISE", expr_text(s.exc, 80)) + elif isinstance(s, ast.Match): + add(s, "MATCH", expr_text(s.subject, 80)) + for case in s.cases: + add(case, "CASE", expr_text(case.pattern, 60)) + ops.extend(sub(case.body)) + elif isinstance(s, ast.Expr): + v = s.value + if isinstance(v, ast.Await): + add(s, "AWAIT", expr_text(v.value, 120)) + elif isinstance(v, ast.Call): + add(s, "CALL", expr_text(v, 120), risk=expr_risk(v)) + elif isinstance(v, (ast.Yield, ast.YieldFrom)): + add(s, "YIELD", expr_text(getattr(v, "value", None), 80)) + elif isinstance(s, (ast.Break, ast.Continue)): + add(s, s.__class__.__name__.upper()) + return ops[:budget] + + def _function_symbol(node, qualified: str, text: str) -> Symbol: calls: list[str] = [] raises: list[str] = [] @@ -163,7 +273,7 @@ def _function_symbol(node, qualified: str, text: str) -> Symbol: slice8=slice_sha8(text, start, end), calls=calls, raises=raises, - flow=[], # filled by flow_ops in Task 2 + flow=flow_ops(node.body), is_async=isinstance(node, ast.AsyncFunctionDef), ) diff --git a/tests/test_kern_compile.py b/tests/test_kern_compile.py index de1b172..49f194c 100644 --- a/tests/test_kern_compile.py +++ b/tests/test_kern_compile.py @@ -105,5 +105,75 @@ def test_decorated_class_span_includes_decorator(self): self.assertEqual(c.decorators, ["functools.total_ordering"]) +FLOW_SAMPLE = ''' +import re +import threading + +PATTERN = re.compile(r"^x+$") + + +def process(path, items): + data = path.read_bytes() + total = 0 + for item in items: + if item.bad: + raise ValueError(item) + total += 1 + try: + result = transform(data) + except KeyError as exc: + log(exc) + finally: + cleanup() + with threading.Lock(): + shared.append(total) + return result + + +def transform(data): + return re.sub(r"a+", "b", data.decode()) +''' + + +class TestFlowOps(unittest.TestCase): + def setUp(self): + self.mod = kern_compile.parse_python(FLOW_SAMPLE) + self.proc = next(s for s in self.mod.symbols if s.name == "process") + + def ops(self): + return [(o.op, o.depth) for o in self.proc.flow] + + def test_call_with_binds(self): + first = self.proc.flow[0] + self.assertEqual(first.op, "CALL") + self.assertEqual(first.binds, "data") + self.assertIn("path.read_bytes", first.detail) + self.assertGreater(first.line, 0) + + def test_plain_assignment_not_emitted(self): + details = " ".join(o.detail for o in self.proc.flow) + self.assertNotIn("total = 0", details) + + def test_structure(self): + ops = self.ops() + self.assertIn(("LOOP", 0), ops) + self.assertIn(("IF", 1), ops) + self.assertIn(("RAISE", 2), ops) + self.assertIn(("TRY", 0), ops) + self.assertIn(("CATCH", 0), ops) + self.assertIn(("FINALLY", 0), ops) + self.assertIn(("WITH", 0), ops) + self.assertIn(("RET", 0), ops) + + def test_regex_risk_tagged(self): + trans = next(s for s in self.mod.symbols if s.name == "transform") + ret = next(o for o in trans.flow if o.op == "RET") + self.assertEqual(ret.risk, "regex") + + def test_concurrency_risk_tagged(self): + withs = [o for o in self.proc.flow if o.op == "WITH"] + self.assertEqual(withs[0].risk, "concurrency") + + if __name__ == "__main__": unittest.main() From 1534fcc84753b4ee8ba82aa737c796e00c4291ad Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:24:27 +0200 Subject: [PATCH 06/55] fix: else/finally line attribution and AST-based math risk Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 14 +++++-------- tests/test_kern_compile.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index fc20673..8fffda0 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -141,7 +141,6 @@ def _omit_counts(text: str, tree: ast.Module) -> dict: ("crypto", re.compile(r"^(hashlib|hmac|secrets)\.")), ("concurrency", re.compile(r"^(threading|asyncio|multiprocessing|concurrent)\.")), ] -_RISK_MATH = re.compile(r"(\*\*|<<|>>)") _TRY_TYPES = (ast.Try, getattr(ast, "TryStar", ast.Try)) @@ -164,21 +163,18 @@ def expr_risk(node: ast.AST | None) -> str: continue if fn.startswith(("threading.", "asyncio.", "multiprocessing.")): return "concurrency" - try: - if _RISK_MATH.search(ast.unparse(node)): + elif isinstance(ch, ast.BinOp) and isinstance(ch.op, (ast.Pow, ast.LShift, ast.RShift)): return "math" - except Exception: - pass return "" def flow_ops(statements: list, depth: int = 0, budget: int = 200) -> list: ops: list[FlowOp] = [] - def add(node, op, detail="", binds="", risk=""): + def add(node, op, detail="", binds="", risk="", line=None): if len(ops) < budget: ops.append(FlowOp(op=op, detail=detail, binds=binds, depth=depth, - line=getattr(node, "lineno", 0), risk=risk)) + line=line if line is not None else getattr(node, "lineno", 0), risk=risk)) def sub(body): return flow_ops(body, depth + 1, budget - len(ops)) @@ -200,7 +196,7 @@ def sub(body): add(s, "IF", expr_text(s.test, 100), risk=expr_risk(s.test)) ops.extend(sub(s.body)) if s.orelse: - add(s, "ELSE") + add(s, "ELSE", line=s.orelse[0].lineno) ops.extend(sub(s.orelse)) elif isinstance(s, (ast.For, ast.AsyncFor)): add(s, "LOOP", f"{_target(s.target)} in {expr_text(s.iter, 100)}") @@ -222,7 +218,7 @@ def sub(body): add(h, "CATCH", expr_text(h.type, 60)) ops.extend(sub(h.body)) if s.finalbody: - add(s, "FINALLY") + add(s, "FINALLY", line=s.finalbody[0].lineno) ops.extend(sub(s.finalbody)) elif isinstance(s, ast.Return): add(s, "RET", expr_text(s.value, 100), risk=expr_risk(s.value)) diff --git a/tests/test_kern_compile.py b/tests/test_kern_compile.py index 49f194c..1d8d625 100644 --- a/tests/test_kern_compile.py +++ b/tests/test_kern_compile.py @@ -174,6 +174,37 @@ def test_concurrency_risk_tagged(self): withs = [o for o in self.proc.flow if o.op == "WITH"] self.assertEqual(withs[0].risk, "concurrency") + def test_else_and_finally_line_attribution(self): + src = ( + "def f(a):\n" + " if a:\n" + " run(a)\n" + " else:\n" + " halt(a)\n" + " try:\n" + " run(a)\n" + " finally:\n" + " halt(a)\n" + ) + mod = kern_compile.parse_python(src) + flow = next(s for s in mod.symbols if s.name == "f").flow + else_op = next(o for o in flow if o.op == "ELSE") + fin_op = next(o for o in flow if o.op == "FINALLY") + self.assertEqual(else_op.line, 5) + self.assertEqual(fin_op.line, 9) + + def test_math_risk_not_triggered_by_string_contents(self): + src = 'def f():\n return "a ** b << c"\n' + mod = kern_compile.parse_python(src) + ret = next(o for o in next(s for s in mod.symbols if s.name == "f").flow if o.op == "RET") + self.assertEqual(ret.risk, "") + + def test_math_risk_triggered_by_real_operators(self): + src = "def f(a, b):\n return a ** b\n" + mod = kern_compile.parse_python(src) + ret = next(o for o in next(s for s in mod.symbols if s.name == "f").flow if o.op == "RET") + self.assertEqual(ret.risk, "math") + if __name__ == "__main__": unittest.main() From 649b5c4e09b33fb2c5a1e77de5c571da4da962b8 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:30:17 +0200 Subject: [PATCH 07/55] fix: call-based risk tags take priority over math Restructure expr_risk into two passes: first check Call nodes (against _RISK_CALL) and withitem nodes for call-based risks (regex/crypto/ concurrency) and return immediately if found. Only after verify no call-based risks, check for ast.BinOp math risks. This ensures that foo(a) ** hashlib.sha256(a).digest() correctly returns "crypto" instead of "math", since call-based security risks must outrank math risks. Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 8 +++++++- tests/test_kern_compile.py | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 8fffda0..228bda9 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -147,6 +147,8 @@ def _omit_counts(text: str, tree: ast.Module) -> dict: def expr_risk(node: ast.AST | None) -> str: if node is None: return "" + + # First pass: check for call-based risks (regex, crypto, concurrency) for ch in ast.walk(node): if isinstance(ch, ast.Call): try: @@ -163,8 +165,12 @@ def expr_risk(node: ast.AST | None) -> str: continue if fn.startswith(("threading.", "asyncio.", "multiprocessing.")): return "concurrency" - elif isinstance(ch, ast.BinOp) and isinstance(ch.op, (ast.Pow, ast.LShift, ast.RShift)): + + # Second pass: check for math risks (only if no call-based risks found) + for ch in ast.walk(node): + if isinstance(ch, ast.BinOp) and isinstance(ch.op, (ast.Pow, ast.LShift, ast.RShift)): return "math" + return "" diff --git a/tests/test_kern_compile.py b/tests/test_kern_compile.py index 1d8d625..78cd4ac 100644 --- a/tests/test_kern_compile.py +++ b/tests/test_kern_compile.py @@ -205,6 +205,12 @@ def test_math_risk_triggered_by_real_operators(self): ret = next(o for o in next(s for s in mod.symbols if s.name == "f").flow if o.op == "RET") self.assertEqual(ret.risk, "math") + def test_call_risk_outranks_math(self): + src = "import hashlib\n\ndef f(a):\n return foo(a) ** hashlib.sha256(a).digest()\n" + mod = kern_compile.parse_python(src) + ret = next(o for o in next(s for s in mod.symbols if s.name == "f").flow if o.op == "RET") + self.assertEqual(ret.risk, "crypto") + if __name__ == "__main__": unittest.main() From 897011f8e983e625922607ed3c8dbbb6f4ea5a25 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:32:30 +0200 Subject: [PATCH 08/55] feat: effect tables and effect/raise propagation Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 61 +++++++++++++++++++++++ tests/test_effects.py | 77 +++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 tests/test_effects.py diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 228bda9..ed586f2 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -143,6 +143,67 @@ def _omit_counts(text: str, tree: ast.Module) -> dict: ] _TRY_TYPES = (ast.Try, getattr(ast, "TryStar", ast.Try)) +EFFECT_RULES = [ + ("fs:read", re.compile( + r"^(open|fitz\.open|os\.(walk|listdir|stat|scandir)|os\.path\.(exists|isfile|isdir|getsize|getmtime|basename)" + r")$|\.(read|readlines|read_text|read_bytes|exists|is_file|is_dir|stat|iterdir|glob)$")), + ("fs:write", re.compile( + r"^(os\.(replace|remove|makedirs|mkdir|rename|unlink|chmod)|shutil\.\w+|tempfile\.\w+)$" + r"|\.(write|writelines|write_text|write_bytes|save|unlink|mkdir|touch|chmod)$")), + ("net", re.compile(r"^(requests|urllib|socket|http)\.")), + ("proc", re.compile(r"^(subprocess\.\w+|os\.(system|popen|execv|execve|spawnl))$")), + ("env", re.compile(r"^os\.(getenv|putenv|environ\.get)$")), + ("time", re.compile(r"^(time\.(time|sleep|monotonic|time_ns)|datetime\.(now|utcnow|datetime\.now))$|\.sleep$")), + ("random", re.compile(r"^(random|uuid|secrets)\.")), + ("console", re.compile(r"^(print|input)$|^logging\.")), + ("thread", re.compile(r"^(threading|asyncio|multiprocessing|concurrent)\.|^ThreadPoolExecutor$")), +] + + +def classify_call(call_name: str) -> list: + name = call_name.split("(")[0].strip() + return [effect for effect, rx in EFFECT_RULES if rx.search(name)] + + +def propagate(module: ModuleIR) -> None: + funcs = [s for s in module.symbols if s.kind == "function"] + by_tail: dict[str, list] = {} + for s in funcs: + by_tail.setdefault(s.name.split(".")[-1], []).append(s) + for s in funcs: + if not s.effects: + s.effects = {e: [] for c in s.calls for e in classify_call(c)} + if not s.raises_all: + s.raises_all = {r: [] for r in s.raises} + unknown = 0 + for c in s.calls: + tail = c.split("(")[0].split(".")[-1] + if not classify_call(c) and tail not in by_tail: + unknown += 1 + s.unknown_calls = unknown + changed, rounds = True, 0 + while changed and rounds < 32: + changed, rounds = False, rounds + 1 + for s in funcs: + for c in s.calls: + tail = c.split("(")[0].split(".")[-1] + cands = by_tail.get(tail, []) + if len(cands) != 1 or cands[0] is s: + continue + callee = cands[0] + for eff in callee.effects: + if eff not in s.effects: + s.effects[eff] = [tail] + changed = True + elif s.effects[eff] and tail not in s.effects[eff]: + s.effects[eff] = sorted(s.effects[eff] + [tail]) + for exc in callee.raises_all: + if exc not in s.raises_all: + s.raises_all[exc] = [tail] + changed = True + elif s.raises_all[exc] and tail not in s.raises_all[exc]: + s.raises_all[exc] = sorted(s.raises_all[exc] + [tail]) + def expr_risk(node: ast.AST | None) -> str: if node is None: diff --git a/tests/test_effects.py b/tests/test_effects.py new file mode 100644 index 0000000..20771aa --- /dev/null +++ b/tests/test_effects.py @@ -0,0 +1,77 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + +SAMPLE = ''' +def read_expected(root): + pin = (root / ".pin").read_text().strip() + if len(pin) != 64: + raise ValueError(root) + return pin + + +def load_entry(path, expected_sha): + data = path.read_bytes() + if not data: + raise StaleSource(path) + return data + + +def find_entries(root): + manifest = load_entry(root / "m.json", read_expected(root)) + frobnicate(manifest) + return manifest +''' + + +class TestClassify(unittest.TestCase): + def test_fs_read(self): + self.assertIn("fs:read", kern_compile.classify_call("path.read_bytes")) + self.assertIn("fs:read", kern_compile.classify_call("open")) + def test_fs_write(self): + self.assertIn("fs:write", kern_compile.classify_call("os.replace")) + def test_proc(self): + self.assertIn("proc", kern_compile.classify_call("subprocess.run")) + def test_net(self): + self.assertIn("net", kern_compile.classify_call("requests.post")) + def test_unknown(self): + self.assertEqual(kern_compile.classify_call("frobnicate"), []) + + +class TestPropagate(unittest.TestCase): + def setUp(self): + self.mod = kern_compile.parse_python(SAMPLE) + kern_compile.propagate(self.mod) + + def sym(self, name): + return next(s for s in self.mod.symbols if s.name == name) + + def test_direct_effect(self): + self.assertEqual(self.sym("load_entry").effects.get("fs:read"), []) + + def test_inherited_effect_with_via(self): + eff = self.sym("find_entries").effects + self.assertIn("fs:read", eff) + self.assertIn("load_entry", eff["fs:read"]) + + def test_raises_propagate(self): + ra = self.sym("find_entries").raises_all + self.assertIn("StaleSource", ra) + self.assertIn("ValueError", ra) + self.assertIn("read_expected", ra["ValueError"]) + + def test_unknown_counted(self): + self.assertGreaterEqual(self.sym("find_entries").unknown_calls, 1) + + def test_idempotent(self): + before = {s.name: dict(s.effects) for s in self.mod.symbols} + kern_compile.propagate(self.mod) + after = {s.name: dict(s.effects) for s in self.mod.symbols} + self.assertEqual(before, after) + + +if __name__ == "__main__": + unittest.main() From 8e807621fa71dc304e548d16785a0db5ffb2aa74 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:37:40 +0200 Subject: [PATCH 09/55] feat: tiered KERN-IL/0.2 emitter with fault tags and omit counts Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 99 ++++++++++++++++++++++++++++- tests/test_emitter.py | 92 +++++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 tests/test_emitter.py diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index ed586f2..56be201 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -355,9 +355,11 @@ def parse_python(text: str) -> ModuleIR: targets = node.targets if isinstance(node, ast.Assign) else [node.target] names = ",".join(_target(t) for t in targets) hint = bool(SECRET_NAME.search(names)) - symbols.append(Symbol(kind="const", name=names, - detail=expr_text(node.value, 100, hint), - span=(node.lineno, node.end_lineno or node.lineno))) + symbol = Symbol(kind="const", name=names, + detail=expr_text(node.value, 100, hint), + span=(node.lineno, node.end_lineno or node.lineno)) + symbol.risk = expr_risk(node.value) + symbols.append(symbol) elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): symbols.append(_function_symbol(node, node.name, text)) elif isinstance(node, ast.ClassDef): @@ -371,3 +373,94 @@ def parse_python(text: str) -> ModuleIR: if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): symbols.append(_function_symbol(member, f"{node.name}.{member.name}", text)) return ModuleIR("python", "pyast", symbols, _omit_counts(text, tree)) + + +_TIER_LEVEL = {"L1": 1, "L2": 2, "L3": 3} +_ELIDED = (" str: + parts = [] + for key in sorted(mapping): + vias = mapping[key] + parts.append(key + (f" (via {', '.join(sorted(vias))})" if vias else "")) + if unknown: + parts.append(f"unknown-calls={unknown}") + return ", ".join(parts) + + +def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: + head = "ASYNC F" if s.is_async else "F" + lines = [f"{head} {s.name}({s.signature}) -> {s.returns or 'Any'} " + f"@L{s.span[0]}-{s.span[1]} ^{s.slice8} ~{tier}"] + if s.decorators: + lines.append(" DECORATORS " + ", ".join(s.decorators)) + if s.calls: + shown = s.calls[:25] + extra = f" …+{len(s.calls) - 25}" if len(s.calls) > 25 else "" + lines.append(" CALLS " + ", ".join(shown) + extra) + effects = _render_provenanced(s.effects, s.unknown_calls) + if effects: + lines.append(" EFFECTS " + effects) + raises = _render_provenanced(s.raises_all, 0) + if raises: + lines.append(" RAISES " + raises) + if level >= 2: + for op in s.flow: + pad = " " * (op.depth + 2) + piece = op.op + if level == 3 and op.detail: + piece += f" {op.detail}" + elif level == 2 and op.op in ("CATCH", "NESTED", "CASE") and op.detail: + piece += f" {op.detail}" + if level == 3 and op.binds: + piece += f" -> {op.binds}" + risk = op.risk + if not risk and level == 3 and any(m in op.detail for m in _ELIDED): + risk = "elided-literal" + if risk: + piece += f" !FAULT({risk})" + faults.append(f"{risk}(L{op.line})") + lines.append(pad + piece) + return lines + + +def emit_il(module: ModuleIR, source_rel: str, source_sha256: str, + repo_revision: str = "none", tier: str = "L2") -> str: + level = _TIER_LEVEL[tier] + propagate(module) + out = [ + "KERN-IL/0.2", + f"source_rel={source_rel}", + f"source_sha256={source_sha256}", + f"repo_revision={repo_revision}", + f"generator={GENERATOR} lang={module.lang} frontend={module.frontend} tier={tier}", + "", + ] + faults: list[str] = [] + imports = [s for s in module.symbols if s.kind == "import"] + if imports: + lo = min(s.span[0] for s in imports) + hi = max(s.span[1] for s in imports) + out.append(f"IMPORTS {'; '.join(s.detail for s in imports)} @L{lo}-{hi}") + for s in module.symbols: + if s.kind == "const": + tag = "" + if s.risk: + tag = f" !FAULT({s.risk})" + faults.append(f"{s.risk}(L{s.span[0]})") + out.append(f"C {s.name}={s.detail} @L{s.span[0]}{tag}") + for s in module.symbols: + if s.kind == "class": + out.extend(["", f"CLASS {s.name}({s.bases}) @L{s.span[0]}-{s.span[1]} ^{s.slice8}"]) + elif s.kind == "function": + out.append("") + out.extend(_function_lines(s, level, tier, faults)) + omit = " ".join(f"{k}={v}" for k, v in sorted(module.omit.items())) + out.extend([ + "", + f"OMIT {omit} bodies-tier={tier}", + "FAULT-BEFORE edit(any), exact-literals" + + "".join(f", {f}" for f in dict.fromkeys(faults)), + ]) + return "\n".join(out).rstrip() + "\n" diff --git a/tests/test_emitter.py b/tests/test_emitter.py new file mode 100644 index 0000000..280e842 --- /dev/null +++ b/tests/test_emitter.py @@ -0,0 +1,92 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + +SAMPLE = ''' +import re +from pathlib import Path + +TOKEN = "ghp_abcdefghijklmnop1234" +PATTERN = re.compile(r"^[a-z]+$") + + +def load_entry(path: Path, expected_sha: str) -> dict: + data = path.read_bytes() + if not data: + raise ValueError(path) + return parse(data) + + +def parse(data): + return data.decode() +''' + + +def emit(tier): + mod = kern_compile.parse_python(SAMPLE) + return kern_compile.emit_il(mod, "src/x.py", "a" * 64, "d7e8242", tier) + + +class TestEmitter(unittest.TestCase): + def test_header(self): + il = emit("L2").splitlines() + self.assertEqual(il[0], "KERN-IL/0.2") + self.assertEqual(il[1], "source_rel=src/x.py") + self.assertEqual(il[2], "source_sha256=" + "a" * 64) + self.assertEqual(il[3], "repo_revision=d7e8242") + self.assertIn("generator=kern-det/0.2 lang=python frontend=pyast tier=L2", il[4]) + + def test_function_line_format(self): + il = emit("L2") + fline = next(l for l in il.splitlines() if l.startswith("F load_entry")) + self.assertRegex(fline, r"^F load_entry\(.+\) -> dict @L\d+-\d+ \^[0-9a-f]{8} ~L2$") + + def test_tier_l1_has_no_flow(self): + il = emit("L1") + self.assertNotIn(" IF", il) + self.assertIn("CALLS", il) + self.assertIn("EFFECTS fs:read", il) + self.assertIn("RAISES ValueError", il) + + def test_tier_l2_flow_without_expressions(self): + il = emit("L2") + body = [l for l in il.splitlines() if l.startswith(" ")] + joined = "\n".join(body) + self.assertIn("IF", joined) + self.assertIn("RAISE", joined) + self.assertNotIn("not data", joined) + + def test_tier_l3_flow_with_expressions_and_binds(self): + il = emit("L3") + self.assertIn("CALL path.read_bytes() -> data", il) + self.assertIn("IF not data", il) + + def test_l3_is_larger_than_l2_is_larger_than_l1(self): + self.assertGreater(len(emit("L3")), len(emit("L2"))) + self.assertGreater(len(emit("L2")), len(emit("L1"))) + + def test_secret_never_in_output(self): + for tier in ("L1", "L2", "L3"): + self.assertNotIn("ghp_abcdefghijklmnop1234", emit(tier)) + + def test_regex_const_fault_tagged(self): + il = emit("L2") + cline = next(l for l in il.splitlines() if l.startswith("C PATTERN")) + self.assertIn("!FAULT(regex)", cline) + self.assertIn("regex(L", il.splitlines()[-1]) + + def test_omit_counts_and_fault_before(self): + lines = emit("L2").splitlines() + self.assertTrue(lines[-2].startswith("OMIT ")) + self.assertIn("bodies-tier=L2", lines[-2]) + self.assertTrue(lines[-1].startswith("FAULT-BEFORE edit(any), exact-literals")) + + def test_deterministic(self): + self.assertEqual(emit("L2"), emit("L2")) + + +if __name__ == "__main__": + unittest.main() From 64008b55aa96f7983bc5e96ba60e388a9a610654 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:44:37 +0200 Subject: [PATCH 10/55] fix: emit one IMPORTS line per contiguous import run Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 21 ++++++++++++++++++--- tests/test_emitter.py | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 56be201..eedf00e 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -440,9 +440,24 @@ def emit_il(module: ModuleIR, source_rel: str, source_sha256: str, faults: list[str] = [] imports = [s for s in module.symbols if s.kind == "import"] if imports: - lo = min(s.span[0] for s in imports) - hi = max(s.span[1] for s in imports) - out.append(f"IMPORTS {'; '.join(s.detail for s in imports)} @L{lo}-{hi}") + # Group imports into contiguous runs + sorted_imports = sorted(imports, key=lambda s: s.span[0]) + runs = [] + current_run = [sorted_imports[0]] + for imp in sorted_imports[1:]: + # Start new run if gap > 1 line after previous import's end + if imp.span[0] - current_run[-1].span[1] > 1: + runs.append(current_run) + current_run = [imp] + else: + current_run.append(imp) + runs.append(current_run) + + # Emit one IMPORTS line per run + for run in runs: + lo = min(s.span[0] for s in run) + hi = max(s.span[1] for s in run) + out.append(f"IMPORTS {'; '.join(s.detail for s in run)} @L{lo}-{hi}") for s in module.symbols: if s.kind == "const": tag = "" diff --git a/tests/test_emitter.py b/tests/test_emitter.py index 280e842..280c3ab 100644 --- a/tests/test_emitter.py +++ b/tests/test_emitter.py @@ -87,6 +87,20 @@ def test_omit_counts_and_fault_before(self): def test_deterministic(self): self.assertEqual(emit("L2"), emit("L2")) + def test_scattered_imports_do_not_balloon_span(self): + src = ( + "import os\n\n\n" + "def a():\n return 1\n\n\n" + "import sys\n\n\n" + "def b():\n return 2\n" + ) + mod = kern_compile.parse_python(src) + il = kern_compile.emit_il(mod, "src/x.py", "a" * 64, "none", "L2") + import_lines = [l for l in il.splitlines() if l.startswith("IMPORTS")] + self.assertEqual(len(import_lines), 2) + self.assertIn("os @L1-1", import_lines[0]) + self.assertIn("sys @L8-8", import_lines[1]) + if __name__ == "__main__": unittest.main() From 8c26d1f0d9630996eee78822e94606e7346c339d Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:48:51 +0200 Subject: [PATCH 11/55] feat: TS/JS tree-sitter frontend with graceful absence Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 178 ++++++++++++++++++++++++++++ tests/test_tsjs.py | 72 +++++++++++ 2 files changed, 250 insertions(+) create mode 100644 tests/test_tsjs.py diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index eedf00e..666e89b 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -425,6 +425,184 @@ def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: return lines +def tsjs_available() -> bool: + try: + import tree_sitter # noqa: F401 + import tree_sitter_javascript # noqa: F401 + import tree_sitter_typescript # noqa: F401 + return True + except ImportError: + return False + + +_TS_FLOW = { + "if_statement": "IF", "for_statement": "LOOP", "for_in_statement": "LOOP", + "while_statement": "WHILE", "do_statement": "WHILE", "try_statement": "TRY", + "return_statement": "RET", "throw_statement": "RAISE", "switch_statement": "MATCH", +} +_TS_FUNC_NODES = {"function_declaration", "generator_function_declaration", "method_definition"} + + +def parse_tsjs(text: str, typescript: bool = False) -> ModuleIR: + from tree_sitter import Language, Parser + if typescript: + import tree_sitter_typescript as ts_lang + language = Language(ts_lang.language_typescript()) + lang_name = "typescript" + else: + import tree_sitter_javascript as js_lang + language = Language(js_lang.language()) + lang_name = "javascript" + parser = Parser(language) + tree = parser.parse(text.encode("utf-8")) + raw = text.encode("utf-8") + + def ntext(n, cap=120): + piece = raw[n.start_byte:n.end_byte].decode("utf-8", "replace") + piece = SPACE.sub(" ", piece).strip() + if SECRET_VALUE.search(piece): + return sanitize_string(piece, secret_hint=True) + return piece[:cap] + + def span(n): + return (n.start_point[0] + 1, n.end_point[0] + 1) + + def field_node(n, name): + return n.child_by_field_name(name) + + def collect_calls(n, acc): + if n.type == "call_expression": + fn = field_node(n, "function") + if fn is not None: + name = ntext(fn, 80) + if name not in acc: + acc.append(name) + for c in n.children: + collect_calls(c, acc) + + def collect_raises(n, acc): + if n.type == "throw_statement": + body = ntext(n, 80) + name = body.removeprefix("throw").strip().removeprefix("new").strip() + name = name.split("(")[0].rstrip(";").strip() + if name and name not in acc: + acc.append(name) + for c in n.children: + collect_raises(c, acc) + + def flow(n, depth=0, budget=200): + ops = [] + + def add(node, op, detail="", binds=""): + if len(ops) < budget: + ops.append(FlowOp(op=op, detail=detail, binds=binds, depth=depth, + line=node.start_point[0] + 1)) + + for c in n.named_children: + if len(ops) >= budget: + break + t = c.type + if t in _TS_FLOW: + cond = field_node(c, "condition") + detail = ntext(cond, 100).strip("()") if cond is not None else "" + if t == "return_statement": + detail = ntext(c, 100).removeprefix("return").strip().rstrip(";") + if t == "throw_statement": + detail = ntext(c, 80).removeprefix("throw").strip().rstrip(";") + add(c, _TS_FLOW[t], detail) + for name in ("body", "consequence"): + inner = field_node(c, name) + if inner is not None: + ops.extend(flow(inner, depth + 1, budget - len(ops))) + alt = field_node(c, "alternative") + if alt is not None: + add(c, "ELSE") + ops.extend(flow(alt, depth + 1, budget - len(ops))) + handler = field_node(c, "handler") + if handler is not None: + param = field_node(handler, "parameter") + if param is None: + param = field_node(handler, "parameters") + add(handler, "CATCH", ntext(param or handler, 40)) + hbody = field_node(handler, "body") + if hbody is not None: + ops.extend(flow(hbody, depth + 1, budget - len(ops))) + elif t == "expression_statement" and c.named_children and c.named_children[0].type in ("call_expression", "await_expression"): + add(c, "CALL", ntext(c.named_children[0], 120)) + elif t in ("lexical_declaration", "variable_declaration"): + for d in c.named_children: + if d.type == "variable_declarator": + value = field_node(d, "value") + if value is not None and value.type in ("call_expression", "await_expression"): + name = field_node(d, "name") + add(d, "CALL", ntext(value, 120), binds=ntext(name, 40) if name is not None else "") + else: + ops.extend(flow(c, depth, budget - len(ops))) + return ops[:budget] + + def function_symbol(node, qualified): + calls, raises = [], [] + collect_calls(node, calls) + collect_raises(node, raises) + params = field_node(node, "parameters") + rtype = field_node(node, "return_type") + body = field_node(node, "body") + a, b = span(node) + is_async = any(ch.type == "async" for ch in node.children) + return Symbol( + kind="function", name=qualified, span=(a, b), + signature=ntext(params, 200).strip("()") if params is not None else "", + returns=ntext(rtype, 60).lstrip(": ") if rtype is not None else "", + slice8=slice_sha8(text, a, b), calls=calls, raises=raises, + flow=flow(body) if body is not None else [], is_async=is_async, + ) + + symbols: list[Symbol] = [] + + def top(n, class_prefix=""): + for c in n.named_children: + t = c.type + if t in ("import_statement",): + symbols.append(Symbol(kind="import", name="", detail=ntext(c, 120), span=span(c))) + elif t in ("lexical_declaration", "variable_declaration") and not class_prefix: + for d in c.named_children: + if d.type == "variable_declarator": + name = field_node(d, "name") + value = field_node(d, "value") + if value is not None and value.type in ("arrow_function", "function_expression"): + symbols.append(function_symbol(value, ntext(name, 60))) + elif name is not None: + detail = ntext(value, 100) if value is not None else "" + hint = bool(SECRET_NAME.search(ntext(name, 60))) + if hint: + detail = sanitize_string(detail, secret_hint=True) + symbols.append(Symbol(kind="const", name=ntext(name, 60), detail=detail, span=span(c))) + elif t in _TS_FUNC_NODES: + name = field_node(c, "name") + qual = (class_prefix + ntext(name, 60)) if name is not None else class_prefix + "" + symbols.append(function_symbol(c, qual)) + elif t == "class_declaration": + name = field_node(c, "name") + cname = ntext(name, 60) if name is not None else "" + a, b = span(c) + symbols.append(Symbol(kind="class", name=cname, span=(a, b), slice8=slice_sha8(text, a, b))) + body = field_node(c, "body") + if body is not None: + top(body, class_prefix=cname + ".") + elif t in ("export_statement", "program", "statement_block"): + top(c, class_prefix) + top(tree.root_node) + + lines = text.splitlines() + omit = { + "docstrings": 0, + "comments": sum(1 for l in lines if l.strip().startswith("//")), + "blank": sum(1 for l in lines if not l.strip()), + "assignments": 0, + } + return ModuleIR(lang_name, "tree-sitter", symbols, omit) + + def emit_il(module: ModuleIR, source_rel: str, source_sha256: str, repo_revision: str = "none", tier: str = "L2") -> str: level = _TIER_LEVEL[tier] diff --git a/tests/test_tsjs.py b/tests/test_tsjs.py new file mode 100644 index 0000000..22661ef --- /dev/null +++ b/tests/test_tsjs.py @@ -0,0 +1,72 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + +TS_SAMPLE = ''' +import { readFile } from "fs/promises"; + +const MAX = 10; + +export class Loader { + async fetch(url: string): Promise { + const data = await readFile(url); + if (!data) { + throw new Error(url); + } + return data.toString(); + } +} + +export function parse(raw: string): number { + return Number(raw); +} +''' + + +class TestAvailability(unittest.TestCase): + def test_available_returns_bool(self): + self.assertIsInstance(kern_compile.tsjs_available(), bool) + + +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestTsFrontend(unittest.TestCase): + def setUp(self): + self.mod = kern_compile.parse_tsjs(TS_SAMPLE, typescript=True) + + def sym(self, name): + return next(s for s in self.mod.symbols if s.name == name) + + def test_module_metadata(self): + self.assertEqual(self.mod.lang, "typescript") + self.assertEqual(self.mod.frontend, "tree-sitter") + + def test_function_and_method(self): + f = self.sym("parse") + self.assertEqual(f.kind, "function") + self.assertIn("raw: string", f.signature) + m = self.sym("Loader.fetch") + self.assertTrue(m.is_async) + self.assertIn("readFile", " ".join(m.calls)) + + def test_flow_and_raises(self): + m = self.sym("Loader.fetch") + ops = [o.op for o in m.flow] + self.assertIn("IF", ops) + self.assertIn("RAISE", ops) + self.assertIn("RET", ops) + self.assertIn("Error", m.raises) + + def test_slice_hash_present(self): + self.assertEqual(len(self.sym("parse").slice8), 8) + + def test_emit_works(self): + il = kern_compile.emit_il(self.mod, "src/x.ts", "b" * 64, "none", "L2") + self.assertTrue(il.startswith("KERN-IL/0.2")) + self.assertIn("F parse", il) + + +if __name__ == "__main__": + unittest.main() From 388a51aa2e03702536d239f129416d15b31ec6bf Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 15:53:17 +0200 Subject: [PATCH 12/55] fix: parse_tsjs reports tree-sitter syntax errors via parse_error Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 18 +++++++++++++++++- tests/test_tsjs.py | 8 ++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 666e89b..3d233bc 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -600,7 +600,23 @@ def top(n, class_prefix=""): "blank": sum(1 for l in lines if not l.strip()), "assignments": 0, } - return ModuleIR(lang_name, "tree-sitter", symbols, omit) + + parse_error = "" + if tree.root_node.has_error: + def first_error_line(n): + if n.type == "ERROR" or n.is_missing: + return n.start_point[0] + 1 + for c in n.children: + found = first_error_line(c) + if found is not None: + return found + return None + line = first_error_line(tree.root_node) + if line is None: + line = 1 + parse_error = f"tree-sitter reported syntax errors (first at L{line})" + + return ModuleIR(lang_name, "tree-sitter", symbols, omit, parse_error=parse_error) def emit_il(module: ModuleIR, source_rel: str, source_sha256: str, diff --git a/tests/test_tsjs.py b/tests/test_tsjs.py index 22661ef..c6fe699 100644 --- a/tests/test_tsjs.py +++ b/tests/test_tsjs.py @@ -67,6 +67,14 @@ def test_emit_works(self): self.assertTrue(il.startswith("KERN-IL/0.2")) self.assertIn("F parse", il) + def test_broken_source_sets_parse_error(self): + broken = "export function f( {\n return 1;\n}\n" + mod = kern_compile.parse_tsjs(broken, typescript=True) + self.assertNotEqual(mod.parse_error, "") + + def test_clean_source_has_no_parse_error(self): + self.assertEqual(self.mod.parse_error, "") + if __name__ == "__main__": unittest.main() From 439faacf50e74e4cea39bcc105d1a1bc5fe15970 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:00:54 +0200 Subject: [PATCH 13/55] feat: kern-il/0.2 codec, deterministic baseline dispatch, size floor, --tier Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 301 ++++++++---------------------- tests/test_cache_integration.py | 71 +++++++ 2 files changed, 150 insertions(+), 222 deletions(-) create mode 100644 tests/test_cache_integration.py diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 3217f84..0c34391 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -4,7 +4,6 @@ from __future__ import annotations import argparse -import ast import copy import hashlib import json @@ -20,8 +19,9 @@ SCHEMA = "kern-cache/0.1" -CODEC_VERSION = "kern-il/0.1" -BASELINE_GENERATOR = "deterministic-baseline/0.1" +CODEC_VERSION = "kern-il/0.2" +BASELINE_GENERATOR = "kern-det/0.2" +TSJS_SUFFIXES = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"} CACHE_DIRNAME = ".kern" DEFAULT_CONFIG: dict[str, Any] = { "schema": SCHEMA, @@ -39,6 +39,8 @@ ], "max_file_bytes": 2_000_000, "image_profile": "dense", + "min_ir_tokens": 600, + "default_tier": "L2", } SECRET_NAME = re.compile( @@ -193,7 +195,7 @@ def load_manifest(path: Path, root: Path) -> dict[str, Any]: raise RuntimeError(f"Invalid KERN manifest {path}: {exc}") from exc if manifest.get("schema") != SCHEMA: raise RuntimeError(f"Unsupported manifest schema: {manifest.get('schema')!r}") - if Path(manifest.get("repo_root", "")).resolve() != root: + if Path(manifest.get("repo_root", "")).resolve() != root.resolve(): raise RuntimeError("Manifest repository root does not match --repo") manifest.setdefault("files", {}) return manifest @@ -203,7 +205,7 @@ def normalize_rel(root: Path, value: str, require_file: bool = True) -> tuple[st candidate = Path(value).expanduser() path = candidate.resolve() if candidate.is_absolute() else (root / candidate).resolve() try: - relative = path.relative_to(root) + relative = path.relative_to(root.resolve()) except ValueError as exc: raise ValueError(f"Path escapes repository root: {path}") from exc if CACHE_DIRNAME in relative.parts: @@ -244,6 +246,24 @@ def git_files(root: Path) -> list[str] | None: return None +def git_revision(root: Path) -> str: + try: + head = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--short", "HEAD"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, text=True, + ) + if head.returncode != 0: + return "none" + sha = head.stdout.strip() + dirty = subprocess.run( + ["git", "-C", str(root), "status", "--porcelain"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, text=True, + ) + return f"dirty:{sha}" if dirty.stdout.strip() else sha + except OSError: + return "none" + + def walk_files(root: Path, excluded: set[str]) -> Iterable[str]: for current, dirs, files in os.walk(root): dirs[:] = [name for name in dirs if name not in excluded] @@ -363,211 +383,6 @@ def refresh_one(root: Path, paths: dict[str, Path], relative: str, source: Path) return digest, record -def sanitize_string(value: str, secret_hint: bool = False) -> str: - digest = sha256_bytes(value.encode("utf-8", "surrogatepass"))[:12] - if secret_hint or SECRET_VALUE.search(value): - return f"" - if len(value) > 160: - return f"" - return value - - -class LiteralSanitizer(ast.NodeTransformer): - def visit_Constant(self, node: ast.Constant): - if isinstance(node.value, str): - return ast.copy_location(ast.Constant(sanitize_string(node.value)), node) - return node - - -def expr(node: ast.AST | None, max_length: int = 260, secret_hint: bool = False) -> str: - if node is None: - return "None" - if isinstance(node, ast.Constant) and isinstance(node.value, str): - value = sanitize_string(node.value, secret_hint) - rendered = repr(value) - else: - clone = LiteralSanitizer().visit(copy.deepcopy(node)) - ast.fix_missing_locations(clone) - try: - rendered = ast.unparse(clone) - except Exception: - rendered = f"<{node.__class__.__name__}>" - rendered = SPACE.sub(" ", rendered).strip() - if secret_hint and rendered and not rendered.startswith("'" - if len(rendered) > max_length: - digest = sha256_bytes(rendered.encode())[:12] - rendered = rendered[: max_length - 32] + f"…" - return rendered - - -def target_text(node: ast.AST) -> str: - try: - return SPACE.sub(" ", ast.unparse(node)).strip() - except Exception: - return f"<{node.__class__.__name__}>" - - -def call_name(node: ast.Call) -> str: - try: - return expr(node.func, 100) - except Exception: - return "" - - -def outline(statements: list[ast.stmt], depth: int = 0, limit: int = 140) -> list[str]: - result: list[str] = [] - - def emit(node: ast.AST, opcode: str, detail: str = "") -> None: - if len(result) >= limit: - return - prefix = " " * depth - line = getattr(node, "lineno", "?") - result.append(f"{prefix}{line}|{opcode}" + (f" {detail}" if detail else "")) - - for statement in statements: - if len(result) >= limit: - break - if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant) and isinstance(statement.value.value, str): - continue - if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - emit(statement, "NESTED", getattr(statement, "name", "?")) - elif isinstance(statement, ast.Assign): - names = ",".join(target_text(item) for item in statement.targets) - emit(statement, "SET", f"{names}={expr(statement.value, secret_hint=bool(SECRET_NAME.search(names)))}") - elif isinstance(statement, ast.AnnAssign): - name = target_text(statement.target) - emit(statement, "SET", f"{name}:{expr(statement.annotation)}={expr(statement.value, secret_hint=bool(SECRET_NAME.search(name)))}") - elif isinstance(statement, ast.AugAssign): - emit(statement, "MUT", f"{target_text(statement.target)} {statement.op.__class__.__name__}= {expr(statement.value)}") - elif isinstance(statement, ast.If): - emit(statement, "IF", expr(statement.test)) - result.extend(outline(statement.body, depth + 1, max(0, limit - len(result)))) - if statement.orelse and len(result) < limit: - emit(statement, "ELSE") - result.extend(outline(statement.orelse, depth + 1, max(0, limit - len(result)))) - elif isinstance(statement, (ast.For, ast.AsyncFor)): - emit(statement, "LOOP", f"{target_text(statement.target)} in {expr(statement.iter)}") - result.extend(outline(statement.body, depth + 1, max(0, limit - len(result)))) - elif isinstance(statement, ast.While): - emit(statement, "WHILE", expr(statement.test)) - result.extend(outline(statement.body, depth + 1, max(0, limit - len(result)))) - elif isinstance(statement, (ast.With, ast.AsyncWith)): - emit(statement, "WITH", ", ".join(expr(item.context_expr) for item in statement.items)) - result.extend(outline(statement.body, depth + 1, max(0, limit - len(result)))) - elif isinstance(statement, (ast.Try, getattr(ast, "TryStar", ast.Try))): - emit(statement, "TRY") - result.extend(outline(statement.body, depth + 1, max(0, limit - len(result)))) - for handler in statement.handlers: - emit(handler, "CATCH", expr(handler.type)) - result.extend(outline(handler.body, depth + 1, max(0, limit - len(result)))) - if statement.finalbody: - emit(statement, "FINALLY") - result.extend(outline(statement.finalbody, depth + 1, max(0, limit - len(result)))) - elif isinstance(statement, ast.Return): - emit(statement, "RET", expr(statement.value)) - elif isinstance(statement, ast.Raise): - emit(statement, "ERR", expr(statement.exc)) - elif isinstance(statement, ast.Assert): - emit(statement, "ASSERT", expr(statement.test)) - elif isinstance(statement, ast.Expr): - value = statement.value - if isinstance(value, ast.Await): - emit(statement, "AWAIT", expr(value.value)) - elif isinstance(value, ast.Call): - emit(statement, "CALL", expr(value)) - elif isinstance(value, (ast.Yield, ast.YieldFrom)): - emit(statement, "YIELD", expr(getattr(value, "value", None))) - elif isinstance(statement, ast.Match): - emit(statement, "MATCH", expr(statement.subject)) - for case in statement.cases: - emit(statement, "CASE", expr(case.pattern)) - result.extend(outline(case.body, depth + 1, max(0, limit - len(result)))) - elif isinstance(statement, ast.Delete): - emit(statement, "DEL", ",".join(target_text(item) for item in statement.targets)) - elif isinstance(statement, ast.Break): - emit(statement, "BREAK") - elif isinstance(statement, ast.Continue): - emit(statement, "CONTINUE") - return result[:limit] - - -def function_card(node: ast.FunctionDef | ast.AsyncFunctionDef, qualified: str) -> list[str]: - prefix = "ASYNC F" if isinstance(node, ast.AsyncFunctionDef) else "F" - signature = expr(node.args, 300) - returns = expr(node.returns, 140) if node.returns else "Any" - decorators = [expr(item, 100) for item in node.decorator_list] - calls: list[str] = [] - for child in ast.walk(node): - if isinstance(child, ast.Call): - name = call_name(child) - if name not in calls: - calls.append(name) - lines = [f"{prefix} {qualified}({signature})->{returns} @L{node.lineno}-{node.end_lineno or node.lineno}"] - if decorators: - lines.append(" DECORATORS " + ", ".join(decorators)) - doc = ast.get_docstring(node, clean=True) - if doc: - lines.append(" DOC " + sanitize_string(doc.splitlines()[0])[:180]) - if calls: - shown = calls[:40] - lines.append(" CALLS " + ", ".join(shown) + (f" …+{len(calls)-40}" if len(calls) > 40 else "")) - flow = outline(node.body) - lines.extend(" " + item for item in flow) - if len(flow) >= 140: - lines.append(" [...] flow outline capped; fault exact source") - return lines - - -def python_ir(text: str, relative: str, digest: str) -> str: - try: - tree = ast.parse(text) - except SyntaxError as exc: - return generic_ir(text, relative, digest, f"python parse failed at L{exc.lineno}: {exc.msg}") - lines = [ - CODEC_VERSION.upper(), - f"source_rel={relative}", - f"source_sha256={digest}", - f"generator={BASELINE_GENERATOR}", - "mode=python-ast-baseline", - "", - f"MODULE @L1-{max(1, len(text.splitlines()))}", - ] - for node in tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - lines.append(f"IMPORT @L{node.lineno} {expr(node)}") - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = ",".join(target_text(target) for target in targets) - value = node.value - annotation = f":{expr(node.annotation)}" if isinstance(node, ast.AnnAssign) else "" - lines.append( - f"C @L{node.lineno} {names}{annotation}={expr(value, secret_hint=bool(SECRET_NAME.search(names)))}" - ) - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - lines.append("") - lines.extend(function_card(node, node.name)) - elif isinstance(node, ast.ClassDef): - bases = ",".join(expr(base, 100) for base in node.bases) - lines.extend(["", f"CLASS {node.name}({bases}) @L{node.lineno}-{node.end_lineno or node.lineno}"]) - for member in node.body: - if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): - lines.append("") - lines.extend(function_card(member, f"{node.name}.{member.name}")) - lines.extend( - [ - "", - "DECLARED_OMISSIONS / REQUIRED PAGE-FAULTS", - " Comments, formatting, most docstring prose, and exact statement bodies are omitted.", - " Long strings are represented by length and digest; likely credentials are redacted.", - " Fault exact source before edits, exact literal claims, security, concurrency, math, regex, or exception matching.", - ] - ) - return "\n".join(lines).rstrip() + "\n" - - GENERIC_KEEP = re.compile( r"^\s*(?:import\b|from\b|export\b|package\b|use\b|class\b|interface\b|type\b|enum\b|" r"(?:async\s+)?(?:def|function|fn|func)\b|(?:public|private|protected|static|final|abstract)\b|" @@ -611,12 +426,47 @@ def generic_ir(text: str, relative: str, digest: str, parse_note: str = "generic return "\n".join(lines).rstrip() + "\n" -def baseline_for(source: Path, relative: str, digest: str) -> str: +def stub_ir(text: str, relative: str, digest: str) -> str: + lines = [ + CODEC_VERSION.upper(), + f"source_rel={relative}", + f"source_sha256={digest}", + f"generator={BASELINE_GENERATOR}", + "mode=source-cheaper", + f"QA source is ~{max(1, len(text) // 4)} tokens ({len(text.splitlines())} lines), below the IL floor; fault exact source.", + ] + return "\n".join(lines) + "\n" + + +def baseline_for(root: Path, source: Path, relative: str, digest: str, + config: dict[str, Any], tier: str | None = None) -> tuple[str, str]: + """Return (il_text, tier_used).""" text = source.read_text(encoding="utf-8", errors="replace") - return python_ir(text, relative, digest) if source.suffix.lower() == ".py" else generic_ir(text, relative, digest) + if max(1, len(text) // 4) < int(config.get("min_ir_tokens", 600)): + return stub_ir(text, relative, digest), "stub" + selected = tier or str(config.get("default_tier", "L2")) + note = "generic language fallback" + try: + import kern_compile + suffix = source.suffix.lower() + module = None + if suffix == ".py": + module = kern_compile.parse_python(text) + elif suffix in TSJS_SUFFIXES and kern_compile.tsjs_available(): + module = kern_compile.parse_tsjs(text, typescript=suffix in {".ts", ".tsx"}) + if module is not None: + if module.parse_error: + note = f"parse failed: {module.parse_error}" + else: + revision = git_revision(root) + return kern_compile.emit_il(module, relative, digest, revision, selected), selected + except Exception as exc: + note = f"deterministic compiler failed: {exc}" + return generic_ir(text, relative, digest, note), "generic" -def ensure_file(root: Path, paths: dict[str, Path], relative: str, source: Path) -> dict[str, Any]: +def ensure_file(root: Path, paths: dict[str, Path], relative: str, source: Path, + config: dict[str, Any], tier: str | None = None) -> dict[str, Any]: digest, record = refresh_one(root, paths, relative, source) artifacts = artifact_paths(paths, relative) ir_exists = artifacts["ir"].is_file() @@ -625,9 +475,10 @@ def ensure_file(root: Path, paths: dict[str, Path], relative: str, source: Path) and record.get("ir_source_sha256") == digest and ir_exists and record.get("ir_sha256") == sha256_file(artifacts["ir"]) + and (tier is None or record.get("ir_tier") == tier) ) if not usable: - ir = baseline_for(source, relative, digest) + ir, tier_used = baseline_for(root, source, relative, digest, config, tier) if sha256_file(source) != digest: raise RuntimeError("Source changed while baseline IR was generated; retry ensure") atomic_write(artifacts["ir"], ir.encode("utf-8")) @@ -642,6 +493,7 @@ def ensure_file(root: Path, paths: dict[str, Path], relative: str, source: Path) "ir_source_sha256": digest, "ir_sha256": sha256_file(artifacts["ir"]), "ir_generator": BASELINE_GENERATOR, + "ir_tier": tier_used, "image_status": "stale", "ir_rel": artifacts["ir"].relative_to(root).as_posix(), "images_rel": artifacts["images"].relative_to(root).as_posix(), @@ -665,8 +517,9 @@ def ensure_file(root: Path, paths: dict[str, Path], relative: str, source: Path) } -def prepare_file(root: Path, paths: dict[str, Path], relative: str, source: Path) -> dict[str, Any]: - ensured = ensure_file(root, paths, relative, source) +def prepare_file(root: Path, paths: dict[str, Path], relative: str, source: Path, + config: dict[str, Any]) -> dict[str, Any]: + ensured = ensure_file(root, paths, relative, source, config) digest = ensured["source_sha256"] job_id = uuid.uuid4().hex staging = paths["staging"] / Path(relative + f".{digest[:12]}.{job_id}.kern-il.txt") @@ -864,7 +717,7 @@ def sync_cache( for relative in pending: source = root / relative try: - result = ensure_file(root, paths, relative, source) + result = ensure_file(root, paths, relative, source, config) ensured.append( { "source_rel": relative, @@ -933,9 +786,13 @@ def parse_args() -> argparse.Namespace: sync = sub.add_parser("sync") sync.add_argument("--eager", action="store_true", help="Generate deterministic IR for every stale/missing file") sync.add_argument("--limit", type=int, help="Maximum files to ensure in eager mode") - for name in ("ensure", "prepare", "paths"): - command = sub.add_parser(name) - command.add_argument("file") + ensure = sub.add_parser("ensure") + ensure.add_argument("file") + ensure.add_argument("--tier", choices=("L1", "L2", "L3")) + prepare = sub.add_parser("prepare") + prepare.add_argument("file") + paths_cmd = sub.add_parser("paths") + paths_cmd.add_argument("file") commit = sub.add_parser("commit") commit.add_argument("file") commit.add_argument("--ir-file", required=True, type=Path) @@ -967,9 +824,9 @@ def main() -> int: else: relative, source = normalize_rel(root, args.file) if args.command == "ensure": - result = ensure_file(root, paths, relative, source) + result = ensure_file(root, paths, relative, source, config, tier=getattr(args, "tier", None)) elif args.command == "prepare": - result = prepare_file(root, paths, relative, source) + result = prepare_file(root, paths, relative, source, config) elif args.command == "paths": result = paths_for(root, paths, relative, source) elif args.command == "commit": diff --git a/tests/test_cache_integration.py b/tests/test_cache_integration.py new file mode 100644 index 0000000..591454b --- /dev/null +++ b/tests/test_cache_integration.py @@ -0,0 +1,71 @@ +import json +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_cache # noqa: E402 + +BIG_PY = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( + f'def fn_{i}(path, n):\n' + f' """Doc {i}."""\n' + f' data = path.read_bytes()\n' + f' if not data:\n' + f' raise ValueError(n)\n' + f' return json.loads(data)\n' + for i in range(30) +) + + +class TestCacheIntegration(unittest.TestCase): + def setUp(self): + self.root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + (self.root / "big.py").write_text(BIG_PY) + (self.root / "tiny.py").write_text("X = 1\n") + self.paths, self.config = kern_cache.initialize(self.root) + + def ensure(self, name, tier=None): + rel, src = kern_cache.normalize_rel(self.root, name) + return kern_cache.ensure_file(self.root, self.paths, rel, src, self.config, tier=tier) + + def test_codec_is_0_2(self): + self.assertEqual(kern_cache.CODEC_VERSION, "kern-il/0.2") + + def test_big_python_file_gets_deterministic_il(self): + result = self.ensure("big.py") + il = Path(result["ir"]).read_text() + self.assertTrue(il.startswith("KERN-IL/0.2")) + self.assertIn("tier=L2", il) + self.assertIn("F fn_0(", il) + self.assertIn("EFFECTS fs:read", il) + self.assertIn("RAISES ValueError", il) + + def test_tier_override(self): + result = self.ensure("big.py", tier="L1") + il = Path(result["ir"]).read_text() + self.assertIn("tier=L1", il) + self.assertNotIn(" IF", il) + manifest = json.loads((self.paths["manifest"]).read_text()) + self.assertEqual(manifest["files"]["big.py"]["ir_tier"], "L1") + + def test_tiny_file_gets_source_cheaper_stub(self): + result = self.ensure("tiny.py") + il = Path(result["ir"]).read_text() + self.assertIn("mode=source-cheaper", il) + self.assertNotIn("F ", il) + + def test_syntax_error_falls_back_to_generic(self): + (self.root / "broken.py").write_text("def broken(:\n pass\n" * 300) + il = Path(self.ensure("broken.py")["ir"]).read_text() + self.assertIn("mode=generic-line-baseline", il) + + def test_repo_revision_header_present(self): + il = Path(self.ensure("big.py")["ir"]).read_text() + self.assertIn("repo_revision=", il) + + +if __name__ == "__main__": + unittest.main() From 1e59f0af452b9dabdfe0e12357a9aa9e87144984 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:05:58 +0200 Subject: [PATCH 14/55] feat: verify verb traps stale symbol reads Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 35 ++++++++++++++++ tests/test_verify.py | 67 +++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 tests/test_verify.py diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 0c34391..e0df223 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -776,6 +776,34 @@ def fault_source(source: Path, relative: str, start: int | None, end: int | None ) +def verify_symbol(root: Path, paths: dict[str, Path], relative: str, source: Path, + symbol: str, expected_hash: str, expected_span: str | None = None) -> dict[str, Any]: + import kern_compile + text = source.read_text(encoding="utf-8", errors="replace") + suffix = source.suffix.lower() + if suffix == ".py": + module = kern_compile.parse_python(text) + elif suffix in TSJS_SUFFIXES and kern_compile.tsjs_available(): + module = kern_compile.parse_tsjs(text, typescript=suffix in {".ts", ".tsx"}) + else: + raise ValueError(f"verify does not support {suffix or 'this file type'}; use fault with --expect-sha") + if module.parse_error: + raise RuntimeError(f"current source does not parse ({module.parse_error}); fault exact source") + base = {"ok": True, "operation": "verify", "source_rel": relative, "symbol": symbol, + "source_sha256": sha256_bytes(text.encode("utf-8", "surrogatepass"))} + matches = [s for s in module.symbols if s.kind in {"function", "class"} and s.name == symbol] + if not matches: + return {**base, "result": "stale", "reason": "symbol-not-found"} + found = matches[0] + current_span = f"L{found.span[0]}-{found.span[1]}" + if found.slice8 != expected_hash: + return {**base, "result": "stale", "reason": "symbol-bytes-changed", + "current_hash": found.slice8, "current_span": current_span} + if expected_span and expected_span != current_span: + return {**base, "result": "moved", "current_span": current_span} + return {**base, "result": "ok", "current_span": current_span} + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo", default=".", help="Repository root (default: current directory)") @@ -805,6 +833,11 @@ def parse_args() -> argparse.Namespace: fault.add_argument("--start", type=int) fault.add_argument("--end", type=int) fault.add_argument("--expect-sha") + verify = sub.add_parser("verify") + verify.add_argument("file") + verify.add_argument("--symbol", required=True) + verify.add_argument("--hash", required=True) + verify.add_argument("--span") return parser.parse_args() @@ -836,6 +869,8 @@ def main() -> int: elif args.command == "fault": sys.stdout.write(fault_source(source, relative, args.start, args.end, args.expect_sha)) return 0 + elif args.command == "verify": + result = verify_symbol(root, paths, relative, source, args.symbol, args.hash, args.span) else: # pragma: no cover raise RuntimeError(f"Unknown command: {args.command}") json.dump(result, sys.stdout, indent=2, sort_keys=True) diff --git a/tests/test_verify.py b/tests/test_verify.py new file mode 100644 index 0000000..66cd07f --- /dev/null +++ b/tests/test_verify.py @@ -0,0 +1,67 @@ +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_cache # noqa: E402 +import kern_compile # noqa: E402 + +SRC = '''import json + + +def load_entry(path, expected_sha): + data = path.read_bytes() + if not data: + raise ValueError(path) + return json.loads(data) +''' + + +class TestVerify(unittest.TestCase): + def setUp(self): + self.root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + self.file = self.root / "mod.py" + self.file.write_text(SRC) + self.paths, self.config = kern_cache.initialize(self.root) + mod = kern_compile.parse_python(SRC) + self.sym = next(s for s in mod.symbols if s.name == "load_entry") + + def verify(self, expected_hash, span=None): + rel, src = kern_cache.normalize_rel(self.root, "mod.py") + return kern_cache.verify_symbol(self.root, self.paths, rel, src, + "load_entry", expected_hash, span) + + def test_ok(self): + r = self.verify(self.sym.slice8, f"L{self.sym.span[0]}-{self.sym.span[1]}") + self.assertEqual(r["result"], "ok") + + def test_moved_when_file_shifts(self): + self.file.write_text("# new comment line\n" + SRC) + r = self.verify(self.sym.slice8, f"L{self.sym.span[0]}-{self.sym.span[1]}") + self.assertEqual(r["result"], "moved") + self.assertIn("current_span", r) + + def test_stale_when_body_changes(self): + self.file.write_text(SRC.replace("json.loads(data)", "json.loads(data.strip())")) + r = self.verify(self.sym.slice8) + self.assertEqual(r["result"], "stale") + self.assertEqual(r["reason"], "symbol-bytes-changed") + + def test_stale_when_symbol_deleted(self): + self.file.write_text("import json\n") + r = self.verify(self.sym.slice8) + self.assertEqual(r["result"], "stale") + self.assertEqual(r["reason"], "symbol-not-found") + + def test_unsupported_suffix_raises(self): + (self.root / "x.rb").write_text("def x; end\n" * 200) + rel, src = kern_cache.normalize_rel(self.root, "x.rb") + with self.assertRaises(ValueError): + kern_cache.verify_symbol(self.root, self.paths, rel, src, "x", "deadbeef", None) + + +if __name__ == "__main__": + unittest.main() From 37cec9ac64b8b15ba89bb5c12b0824c99a2ef47a Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:13:51 +0200 Subject: [PATCH 15/55] fix: verify hashes raw bytes and resolves duplicate symbol names Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 33 ++++++++++++++++------- tests/test_verify.py | 44 +++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index e0df223..3c45614 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -779,7 +779,8 @@ def fault_source(source: Path, relative: str, start: int | None, end: int | None def verify_symbol(root: Path, paths: dict[str, Path], relative: str, source: Path, symbol: str, expected_hash: str, expected_span: str | None = None) -> dict[str, Any]: import kern_compile - text = source.read_text(encoding="utf-8", errors="replace") + data = source.read_bytes() + text = data.decode("utf-8", "replace") suffix = source.suffix.lower() if suffix == ".py": module = kern_compile.parse_python(text) @@ -790,18 +791,32 @@ def verify_symbol(root: Path, paths: dict[str, Path], relative: str, source: Pat if module.parse_error: raise RuntimeError(f"current source does not parse ({module.parse_error}); fault exact source") base = {"ok": True, "operation": "verify", "source_rel": relative, "symbol": symbol, - "source_sha256": sha256_bytes(text.encode("utf-8", "surrogatepass"))} + "source_sha256": sha256_bytes(data)} matches = [s for s in module.symbols if s.kind in {"function", "class"} and s.name == symbol] if not matches: return {**base, "result": "stale", "reason": "symbol-not-found"} - found = matches[0] - current_span = f"L{found.span[0]}-{found.span[1]}" - if found.slice8 != expected_hash: - return {**base, "result": "stale", "reason": "symbol-bytes-changed", - "current_hash": found.slice8, "current_span": current_span} - if expected_span and expected_span != current_span: + + def span_of(sym) -> str: + return f"L{sym.span[0]}-{sym.span[1]}" + + hash_hit = next((m for m in matches if m.slice8 == expected_hash), None) + if hash_hit is not None: + current_span = span_of(hash_hit) + if expected_span is None or expected_span == current_span: + return {**base, "result": "ok", "current_span": current_span} return {**base, "result": "moved", "current_span": current_span} - return {**base, "result": "ok", "current_span": current_span} + + span_hit = None + if expected_span: + span_hit = next((m for m in matches if span_of(m) == expected_span), None) + if span_hit is not None: + return {**base, "result": "stale", "reason": "symbol-bytes-changed", + "current_hash": span_hit.slice8, "current_span": span_of(span_hit)} + + found = matches[0] + return {**base, "result": "stale", "reason": "symbol-bytes-changed", + "current_hash": found.slice8, "current_span": span_of(found), + "candidates": [{"span": span_of(m), "hash": m.slice8} for m in matches]} def parse_args() -> argparse.Namespace: diff --git a/tests/test_verify.py b/tests/test_verify.py index 66cd07f..c4a7e36 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -62,6 +62,50 @@ def test_unsupported_suffix_raises(self): with self.assertRaises(ValueError): kern_cache.verify_symbol(self.root, self.paths, rel, src, "x", "deadbeef", None) + def test_duplicate_names_hash_selects_correct_candidate(self): + import kern_compile + src = ( + "class Widget:\n" + " @property\n" + " def x(self):\n" + " return self._x\n" + " @x.setter\n" + " def x(self, value):\n" + " self._x = value\n" + ) + self.file.write_text(src) + mod = kern_compile.parse_python(src) + setter = [s for s in mod.symbols if s.name == "Widget.x"][1] + rel, srcp = kern_cache.normalize_rel(self.root, "mod.py") + r = kern_cache.verify_symbol(self.root, self.paths, rel, srcp, "Widget.x", + setter.slice8, f"L{setter.span[0]}-{setter.span[1]}") + self.assertEqual(r["result"], "ok") + self.assertEqual(r["current_span"], f"L{setter.span[0]}-{setter.span[1]}") + + def test_duplicate_names_stale_lists_candidates(self): + src = ( + "class Widget:\n" + " @property\n" + " def x(self):\n" + " return self._x\n" + " @x.setter\n" + " def x(self, value):\n" + " self._x = value\n" + ) + self.file.write_text(src) + rel, srcp = kern_cache.normalize_rel(self.root, "mod.py") + r = kern_cache.verify_symbol(self.root, self.paths, rel, srcp, "Widget.x", + "00000000", None) + self.assertEqual(r["result"], "stale") + self.assertEqual(len(r["candidates"]), 2) + + def test_source_sha256_matches_raw_bytes(self): + data = self.file.read_bytes() + rel, srcp = kern_cache.normalize_rel(self.root, "mod.py") + r = kern_cache.verify_symbol(self.root, self.paths, rel, srcp, + "load_entry", "00000000", None) + self.assertEqual(r["source_sha256"], kern_cache.sha256_bytes(data)) + if __name__ == "__main__": unittest.main() From de85ea2ae9f30489fcfe47a63d0023967b41f315 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:19:16 +0200 Subject: [PATCH 16/55] feat: per-tier token benchmark with fidelity check Co-Authored-By: Claude Fable 5 --- benchmarks/results/python-det-v2.json | 26 ++++++++++ benchmarks/token_bench.py | 68 +++++++++++++++++++++++++++ tests/test_token_bench.py | 56 ++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 benchmarks/results/python-det-v2.json create mode 100644 benchmarks/token_bench.py create mode 100644 tests/test_token_bench.py diff --git a/benchmarks/results/python-det-v2.json b/benchmarks/results/python-det-v2.json new file mode 100644 index 0000000..1bad7d5 --- /dev/null +++ b/benchmarks/results/python-det-v2.json @@ -0,0 +1,26 @@ +{ + "estimator": "chars/4", + "files": [ + { + "bucket": "medium(2k-10k)", + "fidelity_missing": [], + "file": "skills/kern/scripts/kern_cache.py", + "source_tokens": 8789, + "tiers": { + "L1": { + "ratio": 2.64, + "tokens": 3331 + }, + "L2": { + "ratio": 1.9, + "tokens": 4619 + }, + "L3": { + "ratio": 0.97, + "tokens": 9083 + } + } + } + ], + "schema": "kern-bench/0.2" +} diff --git a/benchmarks/token_bench.py b/benchmarks/token_bench.py new file mode 100644 index 0000000..0fa6ee1 --- /dev/null +++ b/benchmarks/token_bench.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Token benchmark: source vs deterministic KERN-IL per tier, bucketed by size.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + + +def estimate_tokens(text: str) -> int: + return max(1, len(text) // 4) + + +def bucket(tokens: int) -> str: + if tokens < 2_000: + return "small(<2k)" + if tokens < 10_000: + return "medium(2k-10k)" + return "large(>10k)" + + +def fidelity_missing(module, il: str) -> list[str]: + missing = [] + for symbol in module.symbols: + if symbol.kind == "function" and symbol.name.split(".")[-1] not in il: + missing.append(symbol.name) + return missing + + +def bench_file(path: Path) -> dict: + text = path.read_text(encoding="utf-8", errors="replace") + module = kern_compile.parse_python(text) + if module.parse_error: + return {"file": str(path), "error": module.parse_error} + source_tokens = estimate_tokens(text) + row = {"file": str(path), "source_tokens": source_tokens, + "bucket": bucket(source_tokens), "tiers": {}, "fidelity_missing": []} + for tier in ("L1", "L2", "L3"): + il = kern_compile.emit_il(module, path.name, "0" * 64, "none", tier) + il_tokens = estimate_tokens(il) + row["tiers"][tier] = {"tokens": il_tokens, "ratio": round(source_tokens / il_tokens, 2)} + if tier == "L2": + row["fidelity_missing"] = fidelity_missing(module, il) + return row + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("files", nargs="+", type=Path) + parser.add_argument("--out", type=Path, help="Write JSON report here as well as stdout") + args = parser.parse_args() + report = {"schema": "kern-bench/0.2", "estimator": "chars/4", + "files": [bench_file(f) for f in args.files]} + payload = json.dumps(report, indent=2, sort_keys=True) + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(payload + "\n") + print(payload) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_token_bench.py b/tests/test_token_bench.py new file mode 100644 index 0000000..7199e1d --- /dev/null +++ b/tests/test_token_bench.py @@ -0,0 +1,56 @@ +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "skills" / "kern" / "scripts")) + +spec = importlib.util.spec_from_file_location("token_bench", REPO / "benchmarks" / "token_bench.py") +token_bench = importlib.util.module_from_spec(spec) +spec.loader.exec_module(token_bench) + +BIG = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( + f'def fn_{i}(path):\n' + f' """Doc {i} with a somewhat longer explanatory sentence to add bulk."""\n' + f' # a comment line adding source-only weight\n' + f' data = path.read_bytes()\n' + f' if not data:\n' + f' raise ValueError(path)\n' + f' return json.loads(data)\n' + for i in range(40) +) + + +class TestTokenBench(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.f = self.tmp / "big.py" + self.f.write_text(BIG) + + def test_bench_file_shape(self): + row = token_bench.bench_file(self.f) + self.assertIn("source_tokens", row) + self.assertEqual(set(row["tiers"]), {"L1", "L2", "L3"}) + for tier in row["tiers"].values(): + self.assertGreater(tier["ratio"], 1.0) + + def test_tier_ordering(self): + row = token_bench.bench_file(self.f) + self.assertGreater(row["tiers"]["L1"]["ratio"], row["tiers"]["L2"]["ratio"]) + self.assertGreater(row["tiers"]["L2"]["ratio"], row["tiers"]["L3"]["ratio"]) + + def test_fidelity_no_missing_functions(self): + row = token_bench.bench_file(self.f) + self.assertEqual(row["fidelity_missing"], []) + + def test_parse_error_reported_not_raised(self): + bad = self.tmp / "bad.py" + bad.write_text("def broken(:\n") + row = token_bench.bench_file(bad) + self.assertIn("error", row) + + +if __name__ == "__main__": + unittest.main() From 521c68a3f2f64218fc94bed788a2b8e345e76fe5 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:21:22 +0200 Subject: [PATCH 17/55] fix: suppress CALLS line at L3 to remove flow duplication Co-Authored-By: Claude Fable 5 --- benchmarks/results/python-det-v2.json | 4 ++-- skills/kern/scripts/kern_compile.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/results/python-det-v2.json b/benchmarks/results/python-det-v2.json index 1bad7d5..99741b1 100644 --- a/benchmarks/results/python-det-v2.json +++ b/benchmarks/results/python-det-v2.json @@ -16,8 +16,8 @@ "tokens": 4619 }, "L3": { - "ratio": 0.97, - "tokens": 9083 + "ratio": 1.13, + "tokens": 7745 } } } diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 3d233bc..512202f 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -395,7 +395,7 @@ def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: f"@L{s.span[0]}-{s.span[1]} ^{s.slice8} ~{tier}"] if s.decorators: lines.append(" DECORATORS " + ", ".join(s.decorators)) - if s.calls: + if s.calls and level <= 2: shown = s.calls[:25] extra = f" …+{len(s.calls) - 25}" if len(s.calls) > 25 else "" lines.append(" CALLS " + ", ".join(shown) + extra) From bf5d2b97f285bc67f65d52e97b296b156b79b48f Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:29:12 +0200 Subject: [PATCH 18/55] fix: L3 keeps compound-expression calls; anchored fidelity check Co-Authored-By: Claude Fable 5 --- benchmarks/token_bench.py | 7 ++++++- skills/kern/scripts/kern_compile.py | 16 ++++++++++++---- tests/test_emitter.py | 19 +++++++++++++++++++ tests/test_token_bench.py | 10 ++++++++++ 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/benchmarks/token_bench.py b/benchmarks/token_bench.py index 0fa6ee1..894ac54 100644 --- a/benchmarks/token_bench.py +++ b/benchmarks/token_bench.py @@ -5,6 +5,7 @@ import argparse import json +import re import sys from pathlib import Path @@ -27,7 +28,11 @@ def bucket(tokens: int) -> str: def fidelity_missing(module, il: str) -> list[str]: missing = [] for symbol in module.symbols: - if symbol.kind == "function" and symbol.name.split(".")[-1] not in il: + if symbol.kind != "function": + continue + tail = symbol.name.split(".")[-1] + pattern = rf"^(?:ASYNC )?F .*{re.escape(tail)}\(" + if re.search(pattern, il, re.MULTILINE) is None: missing.append(symbol.name) return missing diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 512202f..376f568 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -395,10 +395,18 @@ def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: f"@L{s.span[0]}-{s.span[1]} ^{s.slice8} ~{tier}"] if s.decorators: lines.append(" DECORATORS " + ", ".join(s.decorators)) - if s.calls and level <= 2: - shown = s.calls[:25] - extra = f" …+{len(s.calls) - 25}" if len(s.calls) > 25 else "" - lines.append(" CALLS " + ", ".join(shown) + extra) + if level <= 2: + if s.calls: + shown = s.calls[:25] + extra = f" …+{len(s.calls) - 25}" if len(s.calls) > 25 else "" + lines.append(" CALLS " + ", ".join(shown) + extra) + else: + covered = "\n".join(op.detail for op in s.flow) + leftover = [c for c in s.calls if c not in covered] + if leftover: + shown = leftover[:25] + extra = f" …+{len(leftover) - 25}" if len(leftover) > 25 else "" + lines.append(" CALLS " + ", ".join(shown) + extra) effects = _render_provenanced(s.effects, s.unknown_calls) if effects: lines.append(" EFFECTS " + effects) diff --git a/tests/test_emitter.py b/tests/test_emitter.py index 280c3ab..3861ec5 100644 --- a/tests/test_emitter.py +++ b/tests/test_emitter.py @@ -101,6 +101,25 @@ def test_scattered_imports_do_not_balloon_span(self): self.assertIn("os @L1-1", import_lines[0]) self.assertIn("sys @L8-8", import_lines[1]) + def test_l3_keeps_compound_expression_calls(self): + src = ( + "def combine(a, b):\n" + " total = compute_one(a) + compute_two(b)\n" + " return total\n" + ) + mod = kern_compile.parse_python(src) + il3 = kern_compile.emit_il(mod, "src/x.py", "a" * 64, "none", "L3") + self.assertIn("compute_one", il3) + self.assertIn("compute_two", il3) + il3_dup = kern_compile.emit_il(mod, "src/x.py", "a" * 64, "none", "L3") + self.assertEqual(il3, il3_dup) + + def test_l3_omits_calls_already_in_flow(self): + mod = kern_compile.parse_python(SAMPLE) + il3 = kern_compile.emit_il(mod, "src/x.py", "a" * 64, "none", "L3") + f_block = il3.split("F load_entry")[1].split("\n\n")[0] + self.assertNotIn("CALLS path.read_bytes", f_block) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_token_bench.py b/tests/test_token_bench.py index 7199e1d..ad48be0 100644 --- a/tests/test_token_bench.py +++ b/tests/test_token_bench.py @@ -51,6 +51,16 @@ def test_parse_error_reported_not_raised(self): row = token_bench.bench_file(bad) self.assertIn("error", row) + def test_fidelity_not_fooled_by_substring(self): + f = self.tmp / "sub.py" + f.write_text( + '"""Doc."""\n\n' + "\n\n".join( + f'def helper_{i}(x):\n return format(x)\n' for i in range(40) + ) + ) + row = token_bench.bench_file(f) + self.assertEqual(row["fidelity_missing"], []) + if __name__ == "__main__": unittest.main() From 42da38d2e9bcce8ba0adb30e8f6ca935d62dfafa Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:30:11 +0200 Subject: [PATCH 19/55] chore: regenerate benchmark results after L3 fix Co-Authored-By: Claude Fable 5 --- benchmarks/results/python-det-v2.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/results/python-det-v2.json b/benchmarks/results/python-det-v2.json index 99741b1..fa90e93 100644 --- a/benchmarks/results/python-det-v2.json +++ b/benchmarks/results/python-det-v2.json @@ -16,8 +16,8 @@ "tokens": 4619 }, "L3": { - "ratio": 1.13, - "tokens": 7745 + "ratio": 1.1, + "tokens": 7988 } } } From 8aaaff80aec241e92117476320200483fde91d16 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:35:10 +0200 Subject: [PATCH 20/55] fix: identifier-boundary matching for L3 leftover calls and fidelity check Co-Authored-By: Claude Fable 5 --- benchmarks/results/python-det-v2.json | 2 +- benchmarks/token_bench.py | 2 +- skills/kern/scripts/kern_compile.py | 5 ++++- tests/test_emitter.py | 12 ++++++++++++ tests/test_token_bench.py | 19 +++++++++++++++++++ 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/benchmarks/results/python-det-v2.json b/benchmarks/results/python-det-v2.json index fa90e93..683725b 100644 --- a/benchmarks/results/python-det-v2.json +++ b/benchmarks/results/python-det-v2.json @@ -17,7 +17,7 @@ }, "L3": { "ratio": 1.1, - "tokens": 7988 + "tokens": 7989 } } } diff --git a/benchmarks/token_bench.py b/benchmarks/token_bench.py index 894ac54..5821368 100644 --- a/benchmarks/token_bench.py +++ b/benchmarks/token_bench.py @@ -31,7 +31,7 @@ def fidelity_missing(module, il: str) -> list[str]: if symbol.kind != "function": continue tail = symbol.name.split(".")[-1] - pattern = rf"^(?:ASYNC )?F .*{re.escape(tail)}\(" + pattern = rf"^(?:ASYNC )?F (?:[\w.]+\.)?{re.escape(tail)}\(" if re.search(pattern, il, re.MULTILINE) is None: missing.append(symbol.name) return missing diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 376f568..d7eefdb 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -402,7 +402,10 @@ def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: lines.append(" CALLS " + ", ".join(shown) + extra) else: covered = "\n".join(op.detail for op in s.flow) - leftover = [c for c in s.calls if c not in covered] + leftover = [ + c for c in s.calls + if not re.search(rf"(? 25 else "" diff --git a/tests/test_emitter.py b/tests/test_emitter.py index 3861ec5..bb4fcc9 100644 --- a/tests/test_emitter.py +++ b/tests/test_emitter.py @@ -120,6 +120,18 @@ def test_l3_omits_calls_already_in_flow(self): f_block = il3.split("F load_entry")[1].split("\n\n")[0] self.assertNotIn("CALLS path.read_bytes", f_block) + def test_l3_short_call_name_not_falsely_covered(self): + src = ( + "def f(y):\n" + " pos = at(y) + 1\n" + " return format(y)\n" + ) + mod = kern_compile.parse_python(src) + il3 = kern_compile.emit_il(mod, "src/x.py", "a" * 64, "none", "L3") + f_block = il3.split("F f(")[1] + self.assertIn("CALLS", f_block) + self.assertIn("at", [c.strip() for c in f_block.split("CALLS")[1].splitlines()[0].split(",")]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_token_bench.py b/tests/test_token_bench.py index ad48be0..b7a8a2e 100644 --- a/tests/test_token_bench.py +++ b/tests/test_token_bench.py @@ -61,6 +61,25 @@ def test_fidelity_not_fooled_by_substring(self): row = token_bench.bench_file(f) self.assertEqual(row["fidelity_missing"], []) + def test_fidelity_flags_suffix_collision(self): + f = self.tmp / "suffix.py" + f.write_text( + '"""Doc."""\n\n' + "def foobar(x):\n return x\n\n\n" + "def bar(x):\n return x\n" + ) + text = f.read_text() + module = token_bench.kern_compile.parse_python(text) + il = token_bench.kern_compile.emit_il(module, "x.py", "0" * 64, "none", "L2") + # Drop bar's own F-line, leaving only "F foobar(" in the IL. The tail + # "bar" is a strict suffix of "foobar" but not the full final name + # segment, so it must not count as present. + il_without_bar = "\n".join( + l for l in il.splitlines() if not l.startswith("F bar(") + ) + missing = token_bench.fidelity_missing(module, il_without_bar) + self.assertIn("bar", missing) + if __name__ == "__main__": unittest.main() From 6c373e0079fca0bee63103aaf7d033898137ac28 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:40:32 +0200 Subject: [PATCH 21/55] docs: kern-il/0.2 tiers, verify contract, dev commands Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 8 ++++++++ README.md | 7 +++++-- docs/architecture.md | 3 ++- package.json | 2 +- skills/kern/SKILL.md | 15 +++++++++++++++ 5 files changed, 31 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a06cb40..f485ae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to KERN are documented here. +## 0.2.0 — 2026-07-11 + +- Implement codec `kern-il/0.2` with deterministic tiered compiler (L1/L2/L3). +- Add computed side effects and exception propagation through the IL. +- Introduce `verify` CLI verb for source-map validation before edits. +- Implement size floor (`min_ir_tokens=600`) with `mode=source-cheaper` stubs. +- Add token benchmark harness for compression measurement and modeling. + ## 0.1.1 — 2026-07-11 - Enforce codec-version invalidation and reject stale rendered pages. diff --git a/README.md b/README.md index 1096c75..1e9c96a 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,9 @@ KERN complements parsing, retrieval, tests, and version control. It does not rep ## Language and runtime coverage -Python receives a deterministic AST-based baseline. JavaScript, TypeScript, Go, Rust, Java, Kotlin, Swift, C, C++, C#, and other recognized source formats receive a deterministic generic baseline before optional compiler-model enrichment. Unsupported text formats fall back to the generic representation and exact-source path. +Python and (when tree-sitter is installed) JavaScript/TypeScript receive a deterministic +AST-based compiler with tiered detail, computed side effects, and exception propagation. +Other recognized source formats receive a deterministic generic baseline. | Environment | Distribution | Compiler selection | |---|---|---| @@ -157,7 +159,8 @@ See [model routing](skills/kern/references/model-routing.md) for the host-specif ```bash npm ci npm run build -python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/render_ir.py +python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/render_ir.py skills/kern/scripts/kern_compile.py +python3 -m unittest discover -s tests python3 skills/kern/scripts/kern_cache.py --repo . scan ``` diff --git a/docs/architecture.md b/docs/architecture.md index e0528ad..356625d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,11 +10,12 @@ KERN applies the compiler/virtual-memory pattern to coding-agent context. 4. Only the semantic working set is enriched and loaded. 5. Exact current source must be faulted before an edit. 6. A source change invalidates its prior IL and rendered pages. +7. Every IL symbol carries a slice hash; a passing verify or fresh fault is required before that symbol is edited. ## Lifecycle ```text -scan → hash → invalidate → baseline → enrich → render → page in → exact fault → write → invalidate +scan → hash → invalidate → compile(tiered) → render → page in → verify/fault → write → invalidate ``` The repository-wide scan is cheap. Model work and image rendering are lazy. This is analogous to keeping a page table resident while paging detailed code representations into the active context only when a task touches them. diff --git a/package.json b/package.json index 248546e..cae760d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "kern-il-context-compiler", - "version": "0.1.1", + "version": "0.2.0", "private": true, "scripts": { "dev": "next dev", diff --git a/skills/kern/SKILL.md b/skills/kern/SKILL.md index 0f9428f..e4969c3 100644 --- a/skills/kern/SKILL.md +++ b/skills/kern/SKILL.md @@ -37,6 +37,11 @@ python3 /scripts/kern_cache.py --repo ensure path/to/file `ensure` immediately creates a deterministic baseline IL when the cache is absent or stale. It returns cache paths, the current source hash, and whether model enrichment is needed. +`ensure` accepts `--tier L1|L2|L3` (default from config `default_tier`, `L2`). +L1 = signatures + calls + effects + raises; L2 = + control-flow skeleton; +L3 = + expressions and dataflow. Files below the `min_ir_tokens` floor get a +`mode=source-cheaper` stub — read exact source instead. + When enrichment is needed and a fast compiler subagent is available: 1. Prepare a unique job and retain its JSON output: @@ -72,6 +77,16 @@ Prefer the `dense` image profile: 10 px, four columns, lossless WebP. Do not use ## Fault exact source before edits +Before editing any symbol read from IL or an image, verify its source-map handle: + + python3 /scripts/kern_cache.py --repo verify path/to/file \ + --symbol --hash [--span L-L] + +`ok` — proceed. `moved` — same bytes at a new span; use the returned span. +`stale` — the symbol changed; the IL page is invalid, fault exact source. +Lines tagged `!FAULT(reason)` (regex, math, concurrency, elided-literal) may not +support a claim or an edit without an exact-source fault, regardless of verify. + ```bash python3 /scripts/kern_cache.py --repo fault path/to/file \ --start --end --expect-sha From 09f353a6c5ce253e1c73a56e58c7b6c2bde40c69 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:44:54 +0200 Subject: [PATCH 22/55] docs: complete FAULT reason list and verify caveats Co-Authored-By: Claude Fable 5 --- skills/kern/SKILL.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/skills/kern/SKILL.md b/skills/kern/SKILL.md index e4969c3..fd83b1c 100644 --- a/skills/kern/SKILL.md +++ b/skills/kern/SKILL.md @@ -79,13 +79,16 @@ Prefer the `dense` image profile: 10 px, four columns, lossless WebP. Do not use Before editing any symbol read from IL or an image, verify its source-map handle: - python3 /scripts/kern_cache.py --repo verify path/to/file \ - --symbol --hash [--span L-L] +```bash +python3 /scripts/kern_cache.py --repo verify path/to/file \ + --symbol --hash [--span L-L] +``` `ok` — proceed. `moved` — same bytes at a new span; use the returned span. `stale` — the symbol changed; the IL page is invalid, fault exact source. -Lines tagged `!FAULT(reason)` (regex, math, concurrency, elided-literal) may not +Lines tagged `!FAULT(reason)` (regex, crypto, math, concurrency, elided-literal) may not support a claim or an edit without an exact-source fault, regardless of verify. +verify supports Python always and JavaScript/TypeScript when tree-sitter is installed; for other formats use fault with --expect-sha. ```bash python3 /scripts/kern_cache.py --repo fault path/to/file \ From 20632b028516d1608d8cae7507b599dfc01058ab Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:47:43 +0200 Subject: [PATCH 23/55] feat: enrichment is append-only INTENT lines over deterministic IL Co-Authored-By: Claude Fable 5 --- skills/kern/references/compiler-worker.md | 53 +++++---------------- skills/kern/scripts/kern_cache.py | 14 ++++++ tests/test_enrichment.py | 57 +++++++++++++++++++++++ 3 files changed, 82 insertions(+), 42 deletions(-) create mode 100644 tests/test_enrichment.py diff --git a/skills/kern/references/compiler-worker.md b/skills/kern/references/compiler-worker.md index 98ca02f..551a587 100644 --- a/skills/kern/references/compiler-worker.md +++ b/skills/kern/references/compiler-worker.md @@ -1,45 +1,14 @@ -# KERN compiler worker contract +# Compiler worker contract (enrichment) -Act as the isolated semantic compiler for `KERN-IL/0.1`. +The deterministic IL is authoritative and already committed. Your job is ONLY to +append intent summaries. Output the baseline IL verbatim, then: -Inputs from the coordinator: + ENRICHMENT model= + INTENT : -- Exact source path and repository-relative path. -- Expected SHA-256 of the exact source bytes. -- Deterministic baseline IL path. -- Unique staging output path. - -Read only the assigned source and baseline unless one referenced type is indispensable. Never edit repository source. Write only the staging IL. - -## Required output - -Start with: - -```text -KERN-IL/0.1 -source_rel= -source_sha256= -generator= -``` - -Emit compact semantic cards with exact source ranges. Preserve: - -- Imports, exports, signatures, types, decorators, and annotations. -- Branch predicates, evaluation order, loops, returns, raises, cleanup, and exception paths. -- Calls, data movement, mutation, I/O, side effects, concurrency, and persistent-state changes. -- Exact short literals, thresholds, retry sets, model/config identifiers, and behavior-changing operators. -- Contradictions between comments and executed behavior. -- Defects or ambiguity as `QA`; never silently correct behavior. - -Use compact labels such as `MODULE`, `CLASS`, `F`, `C`, `IF`, `LOOP`, `CALL`, `RET`, `ERR`, `CATCH`, `SIDE`, `INV`, and `QA`. Prefer compact English over unfamiliar pseudo-opcodes. - -End with `DECLARED_OMISSIONS / REQUIRED PAGE-FAULTS`. State exactly what was dropped and when exact source is mandatory. - -## Constraints - -- Target 5–8× fewer text tokens without inventing semantics. -- Redact credentials, tokens, passwords, private keys, and high-entropy values. Record only type, length, and a short one-way digest when useful. -- Do not include benchmark answer keys or fabricate test questions. -- Do not claim the IL is authoritative. -- Do not overwrite another job's staging file. -- Return the staging path, source hash, IL character count, and declared losses. +Rules: +- Never modify, reorder, or omit any deterministic line. The commit will be + rejected if the baseline is not a byte-exact prefix of your output. +- Only `INTENT` lines may follow the `ENRICHMENT` header. +- One INTENT line per symbol, at most; skip symbols whose purpose is obvious. +- Never include secrets, credentials, or long literals. diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 3c45614..f631d13 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -574,6 +574,20 @@ def commit_file( raise ValueError("Staging IR does not declare the expected source_rel") if not headers.get("generator"): raise ValueError("Staging IR does not declare a generator") + baseline_path = artifact_paths(paths, relative)["ir"] + if not baseline_path.is_file(): + raise ValueError("No deterministic baseline IL exists; run ensure before commit") + baseline_text = baseline_path.read_text(encoding="utf-8") + if not text.startswith(baseline_text.rstrip("\n")): + raise ValueError("Enrichment must preserve the deterministic IL verbatim as a prefix") + appended = text[len(baseline_text.rstrip("\n")):].strip("\n") + if appended: + appended_lines = appended.splitlines() + if not appended_lines[0].startswith("ENRICHMENT model="): + raise ValueError("Appended section must start with 'ENRICHMENT model='") + for line in appended_lines[1:]: + if line.strip() and not line.startswith("INTENT "): + raise ValueError(f"Enrichment may only append INTENT lines, found: {line[:60]!r}") artifacts = artifact_paths(paths, relative) atomic_write(artifacts["ir"], payload) if sha256_file(source) != expected_sha: diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py new file mode 100644 index 0000000..5b8a531 --- /dev/null +++ b/tests/test_enrichment.py @@ -0,0 +1,57 @@ +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_cache # noqa: E402 + +BIG_PY = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( + f'def fn_{i}(path):\n' + f' data = path.read_bytes()\n' + f' return json.loads(data)\n' + for i in range(40) +) + + +class TestEnrichmentAppendOnly(unittest.TestCase): + def setUp(self): + self.root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + (self.root / "mod.py").write_text(BIG_PY) + self.paths, self.config = kern_cache.initialize(self.root) + rel, src = kern_cache.normalize_rel(self.root, "mod.py") + self.rel, self.src = rel, src + ensured = kern_cache.ensure_file(self.root, self.paths, rel, src, self.config) + self.digest = ensured["source_sha256"] + self.baseline = Path(ensured["ir"]).read_text() + + def commit(self, staging_text): + staging = self.root / "staging.kern-il.txt" + staging.write_text(staging_text) + return kern_cache.commit_file(self.root, self.paths, self.rel, self.src, + staging, self.digest) + + def test_valid_append_accepted(self): + staged = self.baseline + "\nENRICHMENT model=test-model\nINTENT fn_0: reads and parses a JSON file\n" + result = self.commit(staged) + self.assertEqual(result["status"], "ready") + + def test_replacement_rejected(self): + rogue = self.baseline.replace("F fn_0(", "F totally_different(") + with self.assertRaises(ValueError): + self.commit(rogue + "\nENRICHMENT model=test-model\nINTENT fn_0: x\n") + + def test_missing_enrichment_header_rejected(self): + with self.assertRaises(ValueError): + self.commit(self.baseline + "\nINTENT fn_0: no header line\n") + + def test_non_intent_lines_rejected(self): + staged = self.baseline + "\nENRICHMENT model=test-model\nF injected_fact() -> Any @L1-1 ^deadbeef ~L2\n" + with self.assertRaises(ValueError): + self.commit(staged) + + +if __name__ == "__main__": + unittest.main() From 8233d6da57e2aefb566309f4d4ee5c10db47b74c Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:53:35 +0200 Subject: [PATCH 24/55] fix: enrichment splice bypass and lock validate-write window Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 37 +++++++++++++++++-------------- tests/test_enrichment.py | 9 ++++++++ 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index f631d13..3cb0c6e 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -574,25 +574,28 @@ def commit_file( raise ValueError("Staging IR does not declare the expected source_rel") if not headers.get("generator"): raise ValueError("Staging IR does not declare a generator") - baseline_path = artifact_paths(paths, relative)["ir"] - if not baseline_path.is_file(): - raise ValueError("No deterministic baseline IL exists; run ensure before commit") - baseline_text = baseline_path.read_text(encoding="utf-8") - if not text.startswith(baseline_text.rstrip("\n")): - raise ValueError("Enrichment must preserve the deterministic IL verbatim as a prefix") - appended = text[len(baseline_text.rstrip("\n")):].strip("\n") - if appended: - appended_lines = appended.splitlines() - if not appended_lines[0].startswith("ENRICHMENT model="): - raise ValueError("Appended section must start with 'ENRICHMENT model='") - for line in appended_lines[1:]: - if line.strip() and not line.startswith("INTENT "): - raise ValueError(f"Enrichment may only append INTENT lines, found: {line[:60]!r}") artifacts = artifact_paths(paths, relative) - atomic_write(artifacts["ir"], payload) - if sha256_file(source) != expected_sha: - raise RuntimeError("Source changed during KERN IL commit; rerun prepare") with CacheLock(paths["lock"]): + baseline_path = artifacts["ir"] + if not baseline_path.is_file(): + raise ValueError("No deterministic baseline IL exists; run ensure before commit") + baseline_text = baseline_path.read_text(encoding="utf-8") + if text == baseline_text or text == baseline_text.rstrip("\n"): + appended = "" + elif text.startswith(baseline_text): + appended = text[len(baseline_text):].strip("\n") + else: + raise ValueError("Enrichment must preserve the deterministic IL verbatim as a prefix") + if appended: + appended_lines = appended.splitlines() + if not appended_lines[0].startswith("ENRICHMENT model="): + raise ValueError("Appended section must start with 'ENRICHMENT model='") + for line in appended_lines[1:]: + if line.strip() and not line.startswith("INTENT "): + raise ValueError(f"Enrichment may only append INTENT lines, found: {line[:60]!r}") + atomic_write(artifacts["ir"], payload) + if sha256_file(source) != expected_sha: + raise RuntimeError("Source changed during KERN IL commit; rerun prepare") manifest = load_manifest(paths["manifest"], root) record = manifest["files"].get(relative, {}) if record.get("source_sha256") != expected_sha: diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py index 5b8a531..f1d48db 100644 --- a/tests/test_enrichment.py +++ b/tests/test_enrichment.py @@ -52,6 +52,15 @@ def test_non_intent_lines_rejected(self): with self.assertRaises(ValueError): self.commit(staged) + def test_newline_splice_rejected(self): + spliced = self.baseline.rstrip("\n") + "ENRICHMENT model=evil\nINTENT fn_0: x\n" + with self.assertRaises(ValueError): + self.commit(spliced) + + def test_identical_baseline_accepted_as_noop(self): + result = self.commit(self.baseline) + self.assertEqual(result["status"], "ready") + if __name__ == "__main__": unittest.main() From 2689b7186f85359029dcf372fcdd9335d7e371c9 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 16:59:10 +0200 Subject: [PATCH 25/55] fix: close two-step enrichment splice; normalize IR trailing newline Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 3 ++- tests/test_enrichment.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 3cb0c6e..bb4673b 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -580,7 +580,7 @@ def commit_file( if not baseline_path.is_file(): raise ValueError("No deterministic baseline IL exists; run ensure before commit") baseline_text = baseline_path.read_text(encoding="utf-8") - if text == baseline_text or text == baseline_text.rstrip("\n"): + if text == baseline_text: appended = "" elif text.startswith(baseline_text): appended = text[len(baseline_text):].strip("\n") @@ -593,6 +593,7 @@ def commit_file( for line in appended_lines[1:]: if line.strip() and not line.startswith("INTENT "): raise ValueError(f"Enrichment may only append INTENT lines, found: {line[:60]!r}") + payload = payload.rstrip(b"\n") + b"\n" atomic_write(artifacts["ir"], payload) if sha256_file(source) != expected_sha: raise RuntimeError("Source changed during KERN IL commit; rerun prepare") diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py index f1d48db..f2d96f4 100644 --- a/tests/test_enrichment.py +++ b/tests/test_enrichment.py @@ -61,6 +61,19 @@ def test_identical_baseline_accepted_as_noop(self): result = self.commit(self.baseline) self.assertEqual(result["status"], "ready") + def test_two_step_splice_rejected(self): + with self.assertRaises(ValueError): + self.commit(self.baseline.rstrip("\n")) + + def test_committed_ir_always_ends_with_single_newline(self): + staged = self.baseline + "\nENRICHMENT model=test-model\nINTENT fn_0: reads json\n\n\n" + self.commit(staged) + rel, _ = kern_cache.normalize_rel(self.root, "mod.py") + ir_path = kern_cache.artifact_paths(self.paths, rel)["ir"] + data = ir_path.read_bytes() + self.assertTrue(data.endswith(b"\n")) + self.assertFalse(data.endswith(b"\n\n")) + if __name__ == "__main__": unittest.main() From 9166ee564e8d29e713a64a12c4426a270c9f36b5 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 17:18:00 +0200 Subject: [PATCH 26/55] fix: newline-only line math, secret default redaction, annotation consts, INTENT redaction Co-Authored-By: Claude Fable 5 --- benchmarks/results/python-det-v2.json | 14 +++--- skills/kern/scripts/kern_cache.py | 11 ++++- skills/kern/scripts/kern_compile.py | 68 ++++++++++++++++++++++----- tests/test_emitter.py | 16 +++++++ tests/test_enrichment.py | 5 ++ tests/test_kern_compile.py | 18 ++++++- tests/test_verify.py | 18 +++++++ 7 files changed, 128 insertions(+), 22 deletions(-) diff --git a/benchmarks/results/python-det-v2.json b/benchmarks/results/python-det-v2.json index 683725b..7fefd56 100644 --- a/benchmarks/results/python-det-v2.json +++ b/benchmarks/results/python-det-v2.json @@ -5,19 +5,19 @@ "bucket": "medium(2k-10k)", "fidelity_missing": [], "file": "skills/kern/scripts/kern_cache.py", - "source_tokens": 8789, + "source_tokens": 9150, "tiers": { "L1": { - "ratio": 2.64, - "tokens": 3331 + "ratio": 2.74, + "tokens": 3336 }, "L2": { - "ratio": 1.9, - "tokens": 4619 + "ratio": 1.95, + "tokens": 4686 }, "L3": { - "ratio": 1.1, - "tokens": 7989 + "ratio": 1.11, + "tokens": 8249 } } } diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index bb4673b..714a9a4 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -593,6 +593,8 @@ def commit_file( for line in appended_lines[1:]: if line.strip() and not line.startswith("INTENT "): raise ValueError(f"Enrichment may only append INTENT lines, found: {line[:60]!r}") + if SECRET_VALUE.search(line): + raise ValueError("Enrichment line contains a likely credential") payload = payload.rstrip(b"\n") + b"\n" atomic_write(artifacts["ir"], payload) if sha256_file(source) != expected_sha: @@ -778,12 +780,17 @@ def fault_source(source: Path, relative: str, start: int | None, end: int | None if expected and expected != digest: raise RuntimeError(f"Source hash mismatch: expected {expected}, current {digest}") text = data.decode("utf-8", "strict") - lines = text.splitlines(keepends=True) + # Split on "\n" only, matching ast/tree-sitter line numbering; str.splitlines() + # also breaks on \v, \f, \x1c-\x1e, \x85, U+2028, U+2029, etc., which would + # silently return the wrong bytes for a requested line range. + lines = text.split("\n") + if lines and lines[-1] == "": + lines = lines[:-1] first = start or 1 last = end or len(lines) if first < 1 or last < first or last > max(1, len(lines)): raise ValueError(f"Invalid line range {first}-{last}; file has {len(lines)} lines") - body = "".join(lines[first - 1 : last]) + body = "\n".join(lines[first - 1 : last]) if body and not body.endswith("\n"): body += "\n" return ( diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index d7eefdb..c79a3e8 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -27,8 +27,13 @@ def sha256_hex(data: bytes) -> str: def slice_sha8(source_text: str, start: int, end: int) -> str: - lines = source_text.splitlines(keepends=True) - return sha256_hex("".join(lines[start - 1:end]).encode("utf-8", "surrogatepass"))[:8] + # Split on "\n" only: ast/tree-sitter line numbers count "\n" exclusively, + # while str.splitlines() also breaks on \v, \f, \x1c-\x1e, \x85, U+2028, + # U+2029, etc. A source string containing one of those characters would + # silently shift the hashed window relative to the true line numbers. + lines = source_text.split("\n") + segment = "\n".join(lines[start - 1:end]) + return sha256_hex((segment + "\n").encode("utf-8", "surrogatepass"))[:8] def sanitize_string(value: str, secret_hint: bool = False) -> str: @@ -69,6 +74,36 @@ def expr_text(node: ast.AST | None, max_length: int = 200, secret_hint: bool = F return rendered +def _redact_secret_defaults(args_node: ast.arguments) -> ast.arguments: + """Deep-copy a function's parameter list and replace the default value of + any secret-named parameter (e.g. password="hunter2") with a redacted + placeholder, so literal secrets never appear in an emitted signature.""" + clone = copy.deepcopy(args_node) + + def redacted(default: ast.AST) -> ast.AST: + if isinstance(default, ast.Constant) and isinstance(default.value, str): + value = sanitize_string(default.value, secret_hint=True) + else: + value = "" + return ast.copy_location(ast.Constant(value), default) + + positional = clone.posonlyargs + clone.args + offset = len(positional) - len(clone.defaults) + for i, default in enumerate(clone.defaults): + if default is None: + continue + arg = positional[offset + i] + if arg is not None and SECRET_NAME.search(arg.arg or ""): + clone.defaults[i] = redacted(default) + for i, default in enumerate(clone.kw_defaults): + if default is None: + continue + arg = clone.kwonlyargs[i] + if arg is not None and SECRET_NAME.search(arg.arg or ""): + clone.kw_defaults[i] = redacted(default) + return clone + + def _target(node: ast.AST) -> str: try: return SPACE.sub(" ", ast.unparse(node)).strip() @@ -330,7 +365,7 @@ def _function_symbol(node, qualified: str, text: str) -> Symbol: kind="function", name=qualified, span=(start, end), - signature=expr_text(node.args, 200), + signature=expr_text(_redact_secret_defaults(node.args), 200), returns=expr_text(node.returns, 60) if node.returns else "", decorators=[expr_text(d, 60) for d in node.decorator_list], slice8=slice_sha8(text, start, end), @@ -355,10 +390,17 @@ def parse_python(text: str) -> ModuleIR: targets = node.targets if isinstance(node, ast.Assign) else [node.target] names = ",".join(_target(t) for t in targets) hint = bool(SECRET_NAME.search(names)) - symbol = Symbol(kind="const", name=names, - detail=expr_text(node.value, 100, hint), + if isinstance(node, ast.AnnAssign) and node.value is None: + # Annotation-only declaration (e.g. `count: int`) has no value; + # emit the annotation, not a fabricated "=None". + detail = ": " + expr_text(node.annotation, 60) + risk_value = None + else: + detail = "=" + expr_text(node.value, 100, hint) + risk_value = node.value + symbol = Symbol(kind="const", name=names, detail=detail, span=(node.lineno, node.end_lineno or node.lineno)) - symbol.risk = expr_risk(node.value) + symbol.risk = expr_risk(risk_value) symbols.append(symbol) elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): symbols.append(_function_symbol(node, node.name, text)) @@ -583,10 +625,14 @@ def top(n, class_prefix=""): if value is not None and value.type in ("arrow_function", "function_expression"): symbols.append(function_symbol(value, ntext(name, 60))) elif name is not None: - detail = ntext(value, 100) if value is not None else "" - hint = bool(SECRET_NAME.search(ntext(name, 60))) - if hint: - detail = sanitize_string(detail, secret_hint=True) + if value is not None: + rendered = ntext(value, 100) + hint = bool(SECRET_NAME.search(ntext(name, 60))) + if hint: + rendered = sanitize_string(rendered, secret_hint=True) + detail = "=" + rendered + else: + detail = "" symbols.append(Symbol(kind="const", name=ntext(name, 60), detail=detail, span=span(c))) elif t in _TS_FUNC_NODES: name = field_node(c, "name") @@ -669,7 +715,7 @@ def emit_il(module: ModuleIR, source_rel: str, source_sha256: str, if s.risk: tag = f" !FAULT({s.risk})" faults.append(f"{s.risk}(L{s.span[0]})") - out.append(f"C {s.name}={s.detail} @L{s.span[0]}{tag}") + out.append(f"C {s.name}{s.detail} @L{s.span[0]}{tag}") for s in module.symbols: if s.kind == "class": out.extend(["", f"CLASS {s.name}({s.bases}) @L{s.span[0]}-{s.span[1]} ^{s.slice8}"]) diff --git a/tests/test_emitter.py b/tests/test_emitter.py index bb4fcc9..5259810 100644 --- a/tests/test_emitter.py +++ b/tests/test_emitter.py @@ -132,6 +132,22 @@ def test_l3_short_call_name_not_falsely_covered(self): self.assertIn("CALLS", f_block) self.assertIn("at", [c.strip() for c in f_block.split("CALLS")[1].splitlines()[0].split(",")]) + def test_secret_parameter_defaults_redacted(self): + src = 'def connect(host, password="hunter2", api_key="abc123shortkey"):\n return host\n' + mod = kern_compile.parse_python(src) + for tier in ("L1", "L2", "L3"): + il = kern_compile.emit_il(mod, "src/x.py", "a" * 64, "none", tier) + self.assertNotIn("hunter2", il) + self.assertNotIn("abc123shortkey", il) + self.assertIn("password", il) + + def test_annotation_only_const_has_no_fabricated_value(self): + src = "count: int\nNAME = 'x'\n\n\ndef f():\n return count\n" + mod = kern_compile.parse_python(src) + il = kern_compile.emit_il(mod, "src/x.py", "a" * 64, "none", "L2") + self.assertNotIn("count=None", il) + self.assertIn("count", il) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py index f2d96f4..aa1e14f 100644 --- a/tests/test_enrichment.py +++ b/tests/test_enrichment.py @@ -74,6 +74,11 @@ def test_committed_ir_always_ends_with_single_newline(self): self.assertTrue(data.endswith(b"\n")) self.assertFalse(data.endswith(b"\n\n")) + def test_intent_line_with_secret_rejected(self): + staged = self.baseline + "\nENRICHMENT model=test-model\nINTENT fn_0: uses key ghp_abcdefghijklmnop1234\n" + with self.assertRaises(ValueError): + self.commit(staged) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_kern_compile.py b/tests/test_kern_compile.py index 78cd4ac..32cb650 100644 --- a/tests/test_kern_compile.py +++ b/tests/test_kern_compile.py @@ -61,8 +61,9 @@ def test_function_symbol(self): def test_slice_hash_matches_exact_source_lines(self): f = self.sym("load_entry") start, end = f.span - lines = SAMPLE.splitlines(keepends=True) - expected = hashlib.sha256("".join(lines[start - 1:end]).encode()).hexdigest()[:8] + lines = SAMPLE.split("\n") + segment = "\n".join(lines[start - 1:end]) + "\n" + expected = hashlib.sha256(segment.encode()).hexdigest()[:8] self.assertEqual(f.slice8, expected) self.assertEqual(SAMPLE.splitlines()[start - 1].strip(), "def load_entry(path: Path, expected_sha: str) -> dict:") @@ -104,6 +105,19 @@ def test_decorated_class_span_includes_decorator(self): self.assertEqual(c.span[0], 3) self.assertEqual(c.decorators, ["functools.total_ordering"]) + def test_slice_hash_immune_to_unicode_line_separators(self): + # U+2028 LINE SEPARATOR inside a string literal is a single physical + # line to ast/tree-sitter (which only count "\n"), but str.splitlines() + # treats it as a line break too, shifting the hashed window so an + # edit to the function body below can go undetected. + src = 'X = "a
b"\n\n\ndef f():\n return 1\n' + mod = kern_compile.parse_python(src) + f1 = next(s for s in mod.symbols if s.name == "f") + changed = src.replace("return 1", "return 999") + mod2 = kern_compile.parse_python(changed) + f2 = next(s for s in mod2.symbols if s.name == "f") + self.assertNotEqual(f1.slice8, f2.slice8) + FLOW_SAMPLE = ''' import re diff --git a/tests/test_verify.py b/tests/test_verify.py index c4a7e36..705b62d 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -106,6 +106,24 @@ def test_source_sha256_matches_raw_bytes(self): "load_entry", "00000000", None) self.assertEqual(r["source_sha256"], kern_cache.sha256_bytes(data)) + def test_unicode_separator_cannot_fake_ok(self): + import kern_compile + src = 'X = "a
b"\n\n\ndef load_entry(path):\n return path.read_bytes()\n' + self.file.write_text(src) + mod = kern_compile.parse_python(src) + sym = next(s for s in mod.symbols if s.name == "load_entry") + self.file.write_text(src.replace("read_bytes()", "read_text()")) + rel, srcp = kern_cache.normalize_rel(self.root, "mod.py") + r = kern_cache.verify_symbol(self.root, self.paths, rel, srcp, "load_entry", sym.slice8, None) + self.assertEqual(r["result"], "stale") + + def test_fault_source_immune_to_unicode_line_separators(self): + src = 'X = "a
b"\n\n\ndef load_entry(path):\n return path.read_bytes()\n' + self.file.write_text(src) + body = kern_cache.fault_source(self.file, "mod.py", 4, 5, None) + self.assertIn("def load_entry(path):", body) + self.assertIn("return path.read_bytes()", body) + if __name__ == "__main__": unittest.main() From 9bf4b145654bf15c73b9bd0d792dc468275b7362 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 17:27:33 +0200 Subject: [PATCH 27/55] fix: fault byte fidelity, enrichment header gate, generic IR line math Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 18 ++++++++++++---- tests/test_cache_integration.py | 19 +++++++++++++++++ tests/test_enrichment.py | 5 +++++ tests/test_fault.py | 35 +++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 tests/test_fault.py diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 714a9a4..2b48d08 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -404,7 +404,10 @@ def redact_line(line: str) -> str: def generic_ir(text: str, relative: str, digest: str, parse_note: str = "generic language fallback") -> str: kept = [] - for number, line in enumerate(text.splitlines(), 1): + # Match fault_source's \n-only line numbering (see comment there): text.splitlines() + # also splits on \v, \f, \x1c-\x1e, \x85, U+2028, U+2029, etc., which would make the + # "N|" refs point at the wrong source line for files containing those characters. + for number, line in enumerate(text.split("\n"), 1): if GENERIC_KEEP.search(line): kept.append(f"{number}|{redact_line(line.strip())}") if len(kept) >= 1200: @@ -590,11 +593,12 @@ def commit_file( appended_lines = appended.splitlines() if not appended_lines[0].startswith("ENRICHMENT model="): raise ValueError("Appended section must start with 'ENRICHMENT model='") + for line in appended_lines: + if SECRET_VALUE.search(line): + raise ValueError("Enrichment line contains a likely credential") for line in appended_lines[1:]: if line.strip() and not line.startswith("INTENT "): raise ValueError(f"Enrichment may only append INTENT lines, found: {line[:60]!r}") - if SECRET_VALUE.search(line): - raise ValueError("Enrichment line contains a likely credential") payload = payload.rstrip(b"\n") + b"\n" atomic_write(artifacts["ir"], payload) if sha256_file(source) != expected_sha: @@ -791,7 +795,13 @@ def fault_source(source: Path, relative: str, start: int | None, end: int | None if first < 1 or last < first or last > max(1, len(lines)): raise ValueError(f"Invalid line range {first}-{last}; file has {len(lines)} lines") body = "\n".join(lines[first - 1 : last]) - if body and not body.endswith("\n"): + # "\n".join only inserts separators *between* the selected lines, so the + # final selected line still needs its own terminator appended to match + # the source bytes exactly -- unless that line is truly the file's last + # line and the file itself has no trailing newline, in which case there + # is no "\n" to reproduce. + ends_at_file_end = last >= len(lines) + if not (ends_at_file_end and not text.endswith("\n")): body += "\n" return ( "--- KERN EXACT SOURCE FAULT ---\n" diff --git a/tests/test_cache_integration.py b/tests/test_cache_integration.py index 591454b..1097eff 100644 --- a/tests/test_cache_integration.py +++ b/tests/test_cache_integration.py @@ -66,6 +66,25 @@ def test_repo_revision_header_present(self): il = Path(self.ensure("big.py")["ir"]).read_text() self.assertIn("repo_revision=", il) + def test_generic_ir_line_numbers_use_newline_math(self): + # A form feed (\x0c) is a line boundary for str.splitlines() but not for + # a plain "\n" split; generic_ir's line numbers must match the latter + # (the same \n-only math fault_source uses) or "N|" refs point at the + # wrong source line whenever a file contains such control characters. + content = ( + "#!/bin/sh\n# \x0c\n" + + ("echo filler\n" * 400) + + "if true; then\n echo a\nfi\n" + ) + (self.root / "notpython.sh").write_text(content) + il = Path(self.ensure("notpython.sh")["ir"]).read_text() + self.assertIn("mode=generic-line-baseline", il) + line_refs = [l for l in il.splitlines() if l and l[0].isdigit()] + ref = next(l for l in line_refs if "if true" in l) + n = int(ref.split("|")[0]) + raw_lines = (self.root / "notpython.sh").read_text().split("\n") + self.assertIn("if true", raw_lines[n - 1]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py index aa1e14f..c4a5ac5 100644 --- a/tests/test_enrichment.py +++ b/tests/test_enrichment.py @@ -79,6 +79,11 @@ def test_intent_line_with_secret_rejected(self): with self.assertRaises(ValueError): self.commit(staged) + def test_enrichment_header_with_secret_rejected(self): + staged = self.baseline + "\nENRICHMENT model=ghp_abcdefghijklmnop1234\nINTENT fn_0: fine\n" + with self.assertRaises(ValueError): + self.commit(staged) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_fault.py b/tests/test_fault.py new file mode 100644 index 0000000..8294e90 --- /dev/null +++ b/tests/test_fault.py @@ -0,0 +1,35 @@ +import sys +import unittest +from pathlib import Path +import shutil +import tempfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_cache # noqa: E402 + + +class TestFaultSource(unittest.TestCase): + def fault_body(self, content, start, end): + tmp = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) + f = tmp / "s.py" + f.write_bytes(content) + out = kern_cache.fault_source(f, "s.py", start, end, None) + return out.split("--- SOURCE ---\n", 1)[1] + + def test_range_ending_on_blank_line_is_byte_exact(self): + self.assertEqual(self.fault_body(b"a = 1\n\n\nb = 2\n", 1, 3), "a = 1\n\n\n") + + def test_single_blank_line_file(self): + self.assertEqual(self.fault_body(b"\n", 1, 1), "\n") + + def test_full_file_roundtrip(self): + content = b"x = 1\ny = 2\n\nz = 3\n" + self.assertEqual(self.fault_body(content, 1, 4), content.decode()) + + def test_crlf_preserved(self): + self.assertEqual(self.fault_body(b"a = 1\r\nb = 2\r\n", 1, 1), "a = 1\r\n") + + +if __name__ == "__main__": + unittest.main() From 5421b69f2465b42bc13ff181bc799ccb35cb0662 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 18:19:38 +0200 Subject: [PATCH 28/55] chore: gitignore superpowers plans and sdd scratch Co-Authored-By: Claude Fable 5 --- .gitignore | 2 + .../2026-07-11-deterministic-il-compiler.md | 2027 ----------------- 2 files changed, 2 insertions(+), 2027 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-11-deterministic-il-compiler.md diff --git a/.gitignore b/.gitignore index 592b2fa..85bf5e0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ out/ .wrangler/ .DS_Store npm-debug.log* +docs/superpowers/plans/ +.superpowers/ diff --git a/docs/superpowers/plans/2026-07-11-deterministic-il-compiler.md b/docs/superpowers/plans/2026-07-11-deterministic-il-compiler.md deleted file mode 100644 index 4f6cd9b..0000000 --- a/docs/superpowers/plans/2026-07-11-deterministic-il-compiler.md +++ /dev/null @@ -1,2027 +0,0 @@ -# KERN-IL/0.2 Deterministic Compiler Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace model-generated KERN-IL with a deterministic compiler (`kern_compile.py`) that lowers Python and TS/JS source into tiered KERN-IL/0.2 pages with effects, exceptions, per-symbol source-map hashes, and a `verify` CLI verb that traps stale reads. - -**Architecture:** A new module `skills/kern/scripts/kern_compile.py` holds language frontends (Python stdlib `ast`; TS/JS via optional tree-sitter) that produce a common symbol model, an effect/raise propagation engine, and a shared tiered emitter (L1/L2/L3). `kern_cache.py` keeps all cache/manifest machinery and calls the compiler from `baseline_for()`; it gains a codec bump to `kern-il/0.2`, a size floor, a `--tier` flag, and a `verify` verb. A benchmark script measures compression per tier and file-size bucket. - -**Tech Stack:** Python 3.10+ stdlib only, except optional `tree-sitter`, `tree-sitter-javascript`, `tree-sitter-typescript` (graceful fallback when absent). Tests use stdlib `unittest` (repo has no pytest). - -## Global Constraints - -- Spec: `docs/superpowers/specs/2026-07-11-deterministic-il-compiler-design.md`. Walkthrough with format examples: `docs/deterministic-compiler.md`. -- Determinism is a hard invariant: same source bytes → byte-identical IL. Never put timestamps, absolute paths, or environment data in IL output. -- Codec string is exactly `kern-il/0.2`; generator string is exactly `kern-det/0.2`; IL first line is exactly `KERN-IL/0.2`. -- Tier names are exactly `L1`, `L2`, `L3`. Default tier `L2`. Size floor config key `min_ir_tokens`, default `600` (chars/4 estimate). -- `kern_compile.py` must not import `kern_cache.py` (the reverse dependency only, and only as a local import inside functions). -- Secret redaction rules are copied from the existing `kern_cache.py` (`SECRET_NAME`, `SECRET_VALUE`); a likely credential must never appear in IL. -- All scripts must keep passing `python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/render_ir.py skills/kern/scripts/kern_compile.py`. -- Run all tests with `python3 -m unittest discover -s tests -v` from repo root. -- Work on branch `design/deterministic-il-compiler` (already exists, has the spec). - -## File Structure - -- `skills/kern/scripts/kern_compile.py` — NEW. Symbol model, Python frontend, TS/JS frontend, effect engine, tiered emitter. Self-contained. -- `skills/kern/scripts/kern_cache.py` — MODIFY. Codec bump, `baseline_for` dispatch, size-floor stub, `git_revision`, `--tier`, `verify` verb; delete the old AST baseline (`python_ir`, `function_card`, `outline`, `expr`, `LiteralSanitizer`, `sanitize_string`, `call_name`, `target_text`). Keep `generic_ir` + `redact_line` + `SECRET_*`. -- `benchmarks/token_bench.py` — NEW. Per-tier token benchmark + fidelity check. -- `tests/__init__.py`, `tests/test_kern_compile.py`, `tests/test_effects.py`, `tests/test_emitter.py`, `tests/test_tsjs.py`, `tests/test_cache_integration.py`, `tests/test_verify.py`, `tests/test_token_bench.py` — NEW. -- `skills/kern/SKILL.md`, `docs/architecture.md`, `CHANGELOG.md`, `README.md` — MODIFY (docs, Task 9). - ---- - -### Task 1: Symbol model + Python frontend (signatures, spans, slice hashes) - -**Files:** -- Create: `skills/kern/scripts/kern_compile.py` -- Create: `tests/__init__.py` (empty) -- Test: `tests/test_kern_compile.py` - -**Interfaces:** -- Consumes: nothing (first task). -- Produces: `kern_compile.parse_python(text: str) -> ModuleIR`; dataclasses `FlowOp(op, detail, binds, depth, line, risk)`, `Symbol(kind, name, span, signature, returns, decorators, slice8, calls, raises, flow, is_async, bases, detail, risk, effects, raises_all, unknown_calls)`, `ModuleIR(lang, frontend, symbols, omit, parse_error)`; helpers `slice_sha8(text, start, end) -> str` (8 hex), `expr_text(node, max_length, secret_hint) -> str`, `sanitize_string(value, secret_hint) -> str`; constants `CODEC_VERSION = "kern-il/0.2"`, `GENERATOR = "kern-det/0.2"`. Later tasks fill `flow` (Task 2) and `effects`/`raises_all` (Task 3). - -- [ ] **Step 1: Write the failing test** - -Create `tests/__init__.py` (empty file) and `tests/test_kern_compile.py`: - -```python -import hashlib -import sys -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) -import kern_compile # noqa: E402 - -SAMPLE = '''"""Module docstring.""" - -import json -from hashlib import sha256 -from pathlib import Path - -MANIFEST_NAME = "manifest.json" -API_KEY = "s2_abcdefghijklmnop1234" - - -class StaleSource(Exception): - """Raised on hash mismatch.""" - - -def load_entry(path: Path, expected_sha: str) -> dict: - """Read, verify, parse.""" - data = path.read_bytes() - current_sha = sha256(data).hexdigest() - if current_sha != expected_sha: - raise StaleSource(path) - return json.loads(data) - - -class Loader: - async def fetch(self, url): - return await self.client.get(url) -''' - - -class TestPythonFrontend(unittest.TestCase): - def setUp(self): - self.mod = kern_compile.parse_python(SAMPLE) - - def sym(self, name): - return next(s for s in self.mod.symbols if s.name == name) - - def test_module_metadata(self): - self.assertEqual(self.mod.lang, "python") - self.assertEqual(self.mod.frontend, "pyast") - self.assertEqual(self.mod.parse_error, "") - - def test_function_symbol(self): - f = self.sym("load_entry") - self.assertEqual(f.kind, "function") - self.assertIn("path: Path", f.signature) - self.assertIn("expected_sha: str", f.signature) - self.assertEqual(f.returns, "dict") - self.assertFalse(f.is_async) - self.assertIn("path.read_bytes", f.calls) - self.assertIn("json.loads", f.calls) - self.assertEqual(f.raises, ["StaleSource"]) - - def test_slice_hash_matches_exact_source_lines(self): - f = self.sym("load_entry") - start, end = f.span - lines = SAMPLE.splitlines(keepends=True) - expected = hashlib.sha256("".join(lines[start - 1:end]).encode()).hexdigest()[:8] - self.assertEqual(f.slice8, expected) - self.assertEqual(SAMPLE.splitlines()[start - 1].strip(), "def load_entry(path: Path, expected_sha: str) -> dict:") - - def test_method_is_qualified_and_async(self): - m = self.sym("Loader.fetch") - self.assertTrue(m.is_async) - - def test_class_symbol(self): - c = self.sym("StaleSource") - self.assertEqual(c.kind, "class") - self.assertEqual(c.bases, "Exception") - self.assertEqual(len(c.slice8), 8) - - def test_secret_const_redacted(self): - consts = [s for s in self.mod.symbols if s.kind == "const"] - api = next(s for s in consts if s.name == "API_KEY") - self.assertNotIn("s2_abcdefghijklmnop1234", api.detail) - self.assertIn("REDACTED", api.detail) - - def test_imports_collected(self): - imports = [s for s in self.mod.symbols if s.kind == "import"] - details = " ".join(s.detail for s in imports) - self.assertIn("json", details) - self.assertIn("sha256", details) - - def test_omit_counts(self): - self.assertGreaterEqual(self.mod.omit["docstrings"], 3) - self.assertGreaterEqual(self.mod.omit["blank"], 5) - - def test_parse_error_reported(self): - bad = kern_compile.parse_python("def broken(:\n") - self.assertNotEqual(bad.parse_error, "") - self.assertEqual(bad.symbols, []) - - -if __name__ == "__main__": - unittest.main() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python3 -m unittest tests.test_kern_compile -v` -Expected: FAIL — `ModuleNotFoundError: No module named 'kern_compile'` - -- [ ] **Step 3: Write minimal implementation** - -Create `skills/kern/scripts/kern_compile.py`: - -```python -#!/usr/bin/env python3 -"""Deterministic KERN-IL/0.2 compiler: language frontends, effect engine, tiered emitter.""" - -from __future__ import annotations - -import ast -import copy -import hashlib -import re -from dataclasses import dataclass, field - -CODEC_VERSION = "kern-il/0.2" -GENERATOR = "kern-det/0.2" - -SECRET_NAME = re.compile( - r"(?i)(api[_-]?key|access[_-]?key|auth|bearer|credential|passwd|password|private[_-]?key|secret|token)" -) -SECRET_VALUE = re.compile( - r"(?i)(?:sk|rk|pk|s2)[_-][A-Za-z0-9_-]{12,}|-----BEGIN [A-Z ]*PRIVATE KEY-----|" - r"(?:aws|ghp|github_pat)_[A-Za-z0-9_-]{12,}" -) -SPACE = re.compile(r"\s+") - - -def sha256_hex(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def slice_sha8(source_text: str, start: int, end: int) -> str: - lines = source_text.splitlines(keepends=True) - return sha256_hex("".join(lines[start - 1:end]).encode("utf-8", "surrogatepass"))[:8] - - -def sanitize_string(value: str, secret_hint: bool = False) -> str: - digest = sha256_hex(value.encode("utf-8", "surrogatepass"))[:12] - if secret_hint or SECRET_VALUE.search(value): - return f"" - if len(value) > 160: - return f"" - return value - - -class _LiteralSanitizer(ast.NodeTransformer): - def visit_Constant(self, node: ast.Constant): - if isinstance(node.value, str): - return ast.copy_location(ast.Constant(sanitize_string(node.value)), node) - return node - - -def expr_text(node: ast.AST | None, max_length: int = 200, secret_hint: bool = False) -> str: - if node is None: - return "None" - if isinstance(node, ast.Constant) and isinstance(node.value, str): - rendered = repr(sanitize_string(node.value, secret_hint)) - else: - clone = _LiteralSanitizer().visit(copy.deepcopy(node)) - ast.fix_missing_locations(clone) - try: - rendered = ast.unparse(clone) - except Exception: - rendered = f"<{node.__class__.__name__}>" - rendered = SPACE.sub(" ", rendered).strip() - if secret_hint and not rendered.startswith("'" - if len(rendered) > max_length: - digest = sha256_hex(rendered.encode())[:12] - rendered = rendered[: max_length - 24] + f"…" - return rendered - - -def _target(node: ast.AST) -> str: - try: - return SPACE.sub(" ", ast.unparse(node)).strip() - except Exception: - return f"<{node.__class__.__name__}>" - - -@dataclass -class FlowOp: - op: str - detail: str = "" - binds: str = "" - depth: int = 0 - line: int = 0 - risk: str = "" - - -@dataclass -class Symbol: - kind: str # function | class | const | import - name: str - span: tuple = (0, 0) - signature: str = "" - returns: str = "" - decorators: list = field(default_factory=list) - slice8: str = "" - calls: list = field(default_factory=list) - raises: list = field(default_factory=list) - flow: list = field(default_factory=list) - is_async: bool = False - bases: str = "" - detail: str = "" - risk: str = "" - effects: dict = field(default_factory=dict) # effect -> list of via names ([] = direct) - raises_all: dict = field(default_factory=dict) # exception -> list of via names - unknown_calls: int = 0 - - -@dataclass -class ModuleIR: - lang: str - frontend: str - symbols: list - omit: dict - parse_error: str = "" - - -def _import_detail(node: ast.Import | ast.ImportFrom) -> str: - if isinstance(node, ast.Import): - return ", ".join(a.name for a in node.names) - names = ",".join(a.name for a in node.names) - return f"{node.module or '.'}:{names}" - - -def _omit_counts(text: str, tree: ast.Module) -> dict: - lines = text.splitlines() - comments = sum(1 for l in lines if l.strip().startswith("#")) - blank = sum(1 for l in lines if not l.strip()) - docstrings = 0 - for node in ast.walk(tree): - if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): - if ast.get_docstring(node) is not None: - docstrings += 1 - assigns = sum(1 for n in ast.walk(tree) if isinstance(n, (ast.Assign, ast.AnnAssign, ast.AugAssign))) - return {"docstrings": docstrings, "comments": comments, "blank": blank, "assignments": assigns} - - -def _function_symbol(node, qualified: str, text: str) -> Symbol: - calls: list[str] = [] - raises: list[str] = [] - for ch in ast.walk(node): - if isinstance(ch, ast.Call): - try: - name = expr_text(ch.func, 80) - except Exception: - name = "" - if name not in calls: - calls.append(name) - elif isinstance(ch, ast.Raise) and ch.exc is not None: - name = expr_text(ch.exc, 60).split("(")[0] - if name not in raises: - raises.append(name) - start = min([d.lineno for d in node.decorator_list] + [node.lineno]) - end = node.end_lineno or node.lineno - return Symbol( - kind="function", - name=qualified, - span=(start, end), - signature=expr_text(node.args, 200), - returns=expr_text(node.returns, 60) if node.returns else "", - decorators=[expr_text(d, 60) for d in node.decorator_list], - slice8=slice_sha8(text, start, end), - calls=calls, - raises=raises, - flow=[], # filled by flow_ops in Task 2 - is_async=isinstance(node, ast.AsyncFunctionDef), - ) - - -def parse_python(text: str) -> ModuleIR: - try: - tree = ast.parse(text) - except SyntaxError as exc: - return ModuleIR("python", "pyast", [], {}, parse_error=f"L{exc.lineno}: {exc.msg}") - symbols: list[Symbol] = [] - for node in tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - symbols.append(Symbol(kind="import", name="", detail=_import_detail(node), - span=(node.lineno, node.end_lineno or node.lineno))) - elif isinstance(node, (ast.Assign, ast.AnnAssign)): - targets = node.targets if isinstance(node, ast.Assign) else [node.target] - names = ",".join(_target(t) for t in targets) - hint = bool(SECRET_NAME.search(names)) - symbols.append(Symbol(kind="const", name=names, - detail=expr_text(node.value, 100, hint), - span=(node.lineno, node.end_lineno or node.lineno))) - elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - symbols.append(_function_symbol(node, node.name, text)) - elif isinstance(node, ast.ClassDef): - end = node.end_lineno or node.lineno - symbols.append(Symbol(kind="class", name=node.name, - bases=",".join(expr_text(b, 60) for b in node.bases), - span=(node.lineno, end), slice8=slice_sha8(text, node.lineno, end))) - for member in node.body: - if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): - symbols.append(_function_symbol(member, f"{node.name}.{member.name}", text)) - return ModuleIR("python", "pyast", symbols, _omit_counts(text, tree)) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `python3 -m unittest tests.test_kern_compile -v` -Expected: all tests PASS. - -- [ ] **Step 5: Commit** - -```bash -git add skills/kern/scripts/kern_compile.py tests/__init__.py tests/test_kern_compile.py -git commit -m "feat: kern_compile symbol model and Python frontend" -``` - ---- - -### Task 2: Control-flow extraction with risk tags - -**Files:** -- Modify: `skills/kern/scripts/kern_compile.py` (add `flow_ops`, `expr_risk`; wire into `_function_symbol`) -- Test: `tests/test_kern_compile.py` (append test class) - -**Interfaces:** -- Consumes: Task 1 dataclasses. -- Produces: `flow_ops(statements: list[ast.stmt], depth=0, budget=200) -> list[FlowOp]`; `expr_risk(node: ast.AST) -> str` returning one of `"regex" | "crypto" | "concurrency" | "math" | ""`. `_function_symbol` now fills `Symbol.flow`. Ops emitted: `CALL` (bare calls and assign-from-call, with `binds`), `IF/ELSE/LOOP/WHILE/WITH/TRY/CATCH/FINALLY/RET/RAISE/AWAIT/YIELD/MATCH/CASE/NESTED/BREAK/CONTINUE`. Plain assignments are intentionally NOT emitted (they are counted in `omit["assignments"]`). - -- [ ] **Step 1: Write the failing test** - -Append to `tests/test_kern_compile.py`: - -```python -FLOW_SAMPLE = ''' -import re -import threading - -PATTERN = re.compile(r"^x+$") - - -def process(path, items): - data = path.read_bytes() - total = 0 - for item in items: - if item.bad: - raise ValueError(item) - total += 1 - try: - result = transform(data) - except KeyError as exc: - log(exc) - finally: - cleanup() - with threading.Lock(): - shared.append(total) - return result - - -def transform(data): - return re.sub(r"a+", "b", data.decode()) -''' - - -class TestFlowOps(unittest.TestCase): - def setUp(self): - self.mod = kern_compile.parse_python(FLOW_SAMPLE) - self.proc = next(s for s in self.mod.symbols if s.name == "process") - - def ops(self): - return [(o.op, o.depth) for o in self.proc.flow] - - def test_call_with_binds(self): - first = self.proc.flow[0] - self.assertEqual(first.op, "CALL") - self.assertEqual(first.binds, "data") - self.assertIn("path.read_bytes", first.detail) - self.assertGreater(first.line, 0) - - def test_plain_assignment_not_emitted(self): - details = " ".join(o.detail for o in self.proc.flow) - self.assertNotIn("total = 0", details) - - def test_structure(self): - ops = self.ops() - self.assertIn(("LOOP", 0), ops) - self.assertIn(("IF", 1), ops) - self.assertIn(("RAISE", 2), ops) - self.assertIn(("TRY", 0), ops) - self.assertIn(("CATCH", 0), ops) - self.assertIn(("FINALLY", 0), ops) - self.assertIn(("WITH", 0), ops) - self.assertIn(("RET", 0), ops) - - def test_regex_risk_tagged(self): - trans = next(s for s in self.mod.symbols if s.name == "transform") - ret = next(o for o in trans.flow if o.op == "RET") - self.assertEqual(ret.risk, "regex") - - def test_concurrency_risk_tagged(self): - withs = [o for o in self.proc.flow if o.op == "WITH"] - self.assertEqual(withs[0].risk, "concurrency") -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python3 -m unittest tests.test_kern_compile.TestFlowOps -v` -Expected: FAIL — flow list empty (`IndexError` / assertion failures). - -- [ ] **Step 3: Write implementation** - -Add to `kern_compile.py` (above `_function_symbol`), and change `_function_symbol` so `flow=flow_ops(node.body)`: - -```python -_RISK_CALL = [ - ("regex", re.compile(r"^re\.(compile|match|search|sub|split|fullmatch|findall|finditer)$")), - ("crypto", re.compile(r"^(hashlib|hmac|secrets)\.")), - ("concurrency", re.compile(r"^(threading|asyncio|multiprocessing|concurrent)\.")), -] -_RISK_MATH = re.compile(r"(\*\*|<<|>>)") -_TRY_TYPES = (ast.Try, getattr(ast, "TryStar", ast.Try)) - - -def expr_risk(node: ast.AST | None) -> str: - if node is None: - return "" - for ch in ast.walk(node): - if isinstance(ch, ast.Call): - try: - fn = ast.unparse(ch.func) - except Exception: - continue - for name, rx in _RISK_CALL: - if rx.search(fn): - return name - elif isinstance(ch, ast.withitem): - try: - fn = ast.unparse(ch.context_expr) - except Exception: - continue - if fn.startswith(("threading.", "asyncio.", "multiprocessing.")): - return "concurrency" - try: - if _RISK_MATH.search(ast.unparse(node)): - return "math" - except Exception: - pass - return "" - - -def flow_ops(statements: list, depth: int = 0, budget: int = 200) -> list: - ops: list[FlowOp] = [] - - def add(node, op, detail="", binds="", risk=""): - if len(ops) < budget: - ops.append(FlowOp(op=op, detail=detail, binds=binds, depth=depth, - line=getattr(node, "lineno", 0), risk=risk)) - - def sub(body): - return flow_ops(body, depth + 1, budget - len(ops)) - - for s in statements: - if len(ops) >= budget: - break - if isinstance(s, ast.Expr) and isinstance(s.value, ast.Constant) and isinstance(s.value.value, str): - continue # docstring - if isinstance(s, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - add(s, "NESTED", s.name) - elif isinstance(s, (ast.Assign, ast.AnnAssign)) and isinstance(getattr(s, "value", None), (ast.Call, ast.Await)): - targets = s.targets if isinstance(s, ast.Assign) else [s.target] - names = ",".join(_target(t) for t in targets) - call = s.value.value if isinstance(s.value, ast.Await) else s.value - hint = bool(SECRET_NAME.search(names)) - add(s, "CALL", expr_text(call, 120, hint), binds=names, risk=expr_risk(call)) - elif isinstance(s, ast.If): - add(s, "IF", expr_text(s.test, 100), risk=expr_risk(s.test)) - ops.extend(sub(s.body)) - if s.orelse: - add(s, "ELSE") - ops.extend(sub(s.orelse)) - elif isinstance(s, (ast.For, ast.AsyncFor)): - add(s, "LOOP", f"{_target(s.target)} in {expr_text(s.iter, 100)}") - ops.extend(sub(s.body)) - elif isinstance(s, ast.While): - add(s, "WHILE", expr_text(s.test, 100), risk=expr_risk(s.test)) - ops.extend(sub(s.body)) - elif isinstance(s, (ast.With, ast.AsyncWith)): - detail = ", ".join(expr_text(i.context_expr, 80) for i in s.items) - risk = "" - for i in s.items: - risk = risk or expr_risk(i.context_expr) - add(s, "WITH", detail, risk=risk) - ops.extend(sub(s.body)) - elif isinstance(s, _TRY_TYPES): - add(s, "TRY") - ops.extend(sub(s.body)) - for h in s.handlers: - add(h, "CATCH", expr_text(h.type, 60)) - ops.extend(sub(h.body)) - if s.finalbody: - add(s, "FINALLY") - ops.extend(sub(s.finalbody)) - elif isinstance(s, ast.Return): - add(s, "RET", expr_text(s.value, 100), risk=expr_risk(s.value)) - elif isinstance(s, ast.Raise): - add(s, "RAISE", expr_text(s.exc, 80)) - elif isinstance(s, ast.Match): - add(s, "MATCH", expr_text(s.subject, 80)) - for case in s.cases: - add(case, "CASE", expr_text(case.pattern, 60)) - ops.extend(sub(case.body)) - elif isinstance(s, ast.Expr): - v = s.value - if isinstance(v, ast.Await): - add(s, "AWAIT", expr_text(v.value, 120)) - elif isinstance(v, ast.Call): - add(s, "CALL", expr_text(v, 120), risk=expr_risk(v)) - elif isinstance(v, (ast.Yield, ast.YieldFrom)): - add(s, "YIELD", expr_text(getattr(v, "value", None), 80)) - elif isinstance(s, (ast.Break, ast.Continue)): - add(s, s.__class__.__name__.upper()) - return ops[:budget] -``` - -Note: `ast.Match` requires Python 3.10+ (global constraint). In `_function_symbol`, replace `flow=[]` with `flow=flow_ops(node.body)`. - -- [ ] **Step 4: Run tests** - -Run: `python3 -m unittest tests.test_kern_compile -v` -Expected: all PASS (Task 1 tests still green). - -- [ ] **Step 5: Commit** - -```bash -git add skills/kern/scripts/kern_compile.py tests/test_kern_compile.py -git commit -m "feat: control-flow extraction with risk tags" -``` - ---- - -### Task 3: Effect tables and effect/raise propagation - -**Files:** -- Modify: `skills/kern/scripts/kern_compile.py` (add `EFFECT_RULES`, `classify_call`, `propagate`) -- Test: `tests/test_effects.py` - -**Interfaces:** -- Consumes: `ModuleIR`, `Symbol` from Task 1. -- Produces: `classify_call(call_name: str) -> list[str]`; `propagate(module: ModuleIR) -> None` filling `Symbol.effects: dict[str, list[str]]` (effect → sorted via-names, `[]` = direct), `Symbol.raises_all: dict[str, list[str]]`, `Symbol.unknown_calls: int`. Idempotent: calling twice changes nothing (emitter calls it on every emit). - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_effects.py`: - -```python -import sys -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) -import kern_compile # noqa: E402 - -SAMPLE = ''' -def read_expected(root): - pin = (root / ".pin").read_text().strip() - if len(pin) != 64: - raise ValueError(root) - return pin - - -def load_entry(path, expected_sha): - data = path.read_bytes() - if not data: - raise StaleSource(path) - return data - - -def find_entries(root): - manifest = load_entry(root / "m.json", read_expected(root)) - frobnicate(manifest) - return manifest -''' - - -class TestClassify(unittest.TestCase): - def test_fs_read(self): - self.assertIn("fs:read", kern_compile.classify_call("path.read_bytes")) - self.assertIn("fs:read", kern_compile.classify_call("open")) - def test_fs_write(self): - self.assertIn("fs:write", kern_compile.classify_call("os.replace")) - def test_proc(self): - self.assertIn("proc", kern_compile.classify_call("subprocess.run")) - def test_net(self): - self.assertIn("net", kern_compile.classify_call("requests.post")) - def test_unknown(self): - self.assertEqual(kern_compile.classify_call("frobnicate"), []) - - -class TestPropagate(unittest.TestCase): - def setUp(self): - self.mod = kern_compile.parse_python(SAMPLE) - kern_compile.propagate(self.mod) - - def sym(self, name): - return next(s for s in self.mod.symbols if s.name == name) - - def test_direct_effect(self): - self.assertEqual(self.sym("load_entry").effects.get("fs:read"), []) - - def test_inherited_effect_with_via(self): - eff = self.sym("find_entries").effects - self.assertIn("fs:read", eff) - self.assertIn("load_entry", eff["fs:read"]) - - def test_raises_propagate(self): - ra = self.sym("find_entries").raises_all - self.assertIn("StaleSource", ra) - self.assertIn("ValueError", ra) - self.assertIn("read_expected", ra["ValueError"]) - - def test_unknown_counted(self): - self.assertGreaterEqual(self.sym("find_entries").unknown_calls, 1) - - def test_idempotent(self): - before = {s.name: dict(s.effects) for s in self.mod.symbols} - kern_compile.propagate(self.mod) - after = {s.name: dict(s.effects) for s in self.mod.symbols} - self.assertEqual(before, after) - - -if __name__ == "__main__": - unittest.main() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python3 -m unittest tests.test_effects -v` -Expected: FAIL — `AttributeError: module 'kern_compile' has no attribute 'classify_call'` - -- [ ] **Step 3: Write implementation** - -Add to `kern_compile.py`: - -```python -EFFECT_RULES = [ - ("fs:read", re.compile( - r"^(open|fitz\.open|os\.(walk|listdir|stat|scandir)|os\.path\.(exists|isfile|isdir|getsize|getmtime|basename)" - r")$|\.(read|readlines|read_text|read_bytes|exists|is_file|is_dir|stat|iterdir|glob)$")), - ("fs:write", re.compile( - r"^(os\.(replace|remove|makedirs|mkdir|rename|unlink|chmod)|shutil\.\w+|tempfile\.\w+)$" - r"|\.(write|writelines|write_text|write_bytes|save|unlink|mkdir|touch|chmod)$")), - ("net", re.compile(r"^(requests|urllib|socket|http)\.")), - ("proc", re.compile(r"^(subprocess\.\w+|os\.(system|popen|execv|execve|spawnl))$")), - ("env", re.compile(r"^os\.(getenv|putenv|environ\.get)$")), - ("time", re.compile(r"^(time\.(time|sleep|monotonic|time_ns)|datetime\.(now|utcnow|datetime\.now))$|\.sleep$")), - ("random", re.compile(r"^(random|uuid|secrets)\.")), - ("console", re.compile(r"^(print|input)$|^logging\.")), - ("thread", re.compile(r"^(threading|asyncio|multiprocessing|concurrent)\.|^ThreadPoolExecutor$")), -] - - -def classify_call(call_name: str) -> list: - name = call_name.split("(")[0].strip() - return [effect for effect, rx in EFFECT_RULES if rx.search(name)] - - -def propagate(module: ModuleIR) -> None: - funcs = [s for s in module.symbols if s.kind == "function"] - by_tail: dict[str, list] = {} - for s in funcs: - by_tail.setdefault(s.name.split(".")[-1], []).append(s) - for s in funcs: - if not s.effects: - s.effects = {e: [] for c in s.calls for e in classify_call(c)} - if not s.raises_all: - s.raises_all = {r: [] for r in s.raises} - unknown = 0 - for c in s.calls: - tail = c.split("(")[0].split(".")[-1] - if not classify_call(c) and tail not in by_tail: - unknown += 1 - s.unknown_calls = unknown - changed, rounds = True, 0 - while changed and rounds < 32: - changed, rounds = False, rounds + 1 - for s in funcs: - for c in s.calls: - tail = c.split("(")[0].split(".")[-1] - cands = by_tail.get(tail, []) - if len(cands) != 1 or cands[0] is s: - continue - callee = cands[0] - for eff in callee.effects: - if eff not in s.effects: - s.effects[eff] = [tail] - changed = True - elif s.effects[eff] and tail not in s.effects[eff]: - s.effects[eff] = sorted(s.effects[eff] + [tail]) - for exc in callee.raises_all: - if exc not in s.raises_all: - s.raises_all[exc] = [tail] - changed = True - elif s.raises_all[exc] and tail not in s.raises_all[exc]: - s.raises_all[exc] = sorted(s.raises_all[exc] + [tail]) -``` - -Note the `if not s.effects:` guards — they make `propagate` idempotent. - -- [ ] **Step 4: Run tests** - -Run: `python3 -m unittest tests.test_effects tests.test_kern_compile -v` -Expected: all PASS. - -- [ ] **Step 5: Commit** - -```bash -git add skills/kern/scripts/kern_compile.py tests/test_effects.py -git commit -m "feat: effect tables and effect/raise propagation" -``` - ---- - -### Task 4: Tiered emitter - -**Files:** -- Modify: `skills/kern/scripts/kern_compile.py` (add `emit_il`, `_function_lines`, `_render_provenanced`) -- Test: `tests/test_emitter.py` - -**Interfaces:** -- Consumes: `ModuleIR`, `propagate` (called inside `emit_il`). -- Produces: `emit_il(module: ModuleIR, source_rel: str, source_sha256: str, repo_revision: str = "none", tier: str = "L2") -> str`. Output format per spec: 5 header lines + blank, `IMPORTS`, `C`, `CLASS`, `F` blocks, `OMIT`, `FAULT-BEFORE`. Task 6 calls this from `kern_cache.baseline_for`. - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_emitter.py`: - -```python -import sys -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) -import kern_compile # noqa: E402 - -SAMPLE = ''' -import re -from pathlib import Path - -TOKEN = "ghp_abcdefghijklmnop1234" -PATTERN = re.compile(r"^[a-z]+$") - - -def load_entry(path: Path, expected_sha: str) -> dict: - data = path.read_bytes() - if not data: - raise ValueError(path) - return parse(data) - - -def parse(data): - return data.decode() -''' - - -def emit(tier): - mod = kern_compile.parse_python(SAMPLE) - return kern_compile.emit_il(mod, "src/x.py", "a" * 64, "d7e8242", tier) - - -class TestEmitter(unittest.TestCase): - def test_header(self): - il = emit("L2").splitlines() - self.assertEqual(il[0], "KERN-IL/0.2") - self.assertEqual(il[1], "source_rel=src/x.py") - self.assertEqual(il[2], "source_sha256=" + "a" * 64) - self.assertEqual(il[3], "repo_revision=d7e8242") - self.assertIn("generator=kern-det/0.2 lang=python frontend=pyast tier=L2", il[4]) - - def test_function_line_format(self): - il = emit("L2") - fline = next(l for l in il.splitlines() if l.startswith("F load_entry")) - self.assertRegex(fline, r"^F load_entry\(.+\) -> dict @L\d+-\d+ \^[0-9a-f]{8} ~L2$") - - def test_tier_l1_has_no_flow(self): - il = emit("L1") - self.assertNotIn(" IF", il) - self.assertIn("CALLS", il) - self.assertIn("EFFECTS fs:read", il) - self.assertIn("RAISES ValueError", il) - - def test_tier_l2_flow_without_expressions(self): - il = emit("L2") - body = [l for l in il.splitlines() if l.startswith(" ")] - joined = "\n".join(body) - self.assertIn("IF", joined) - self.assertIn("RAISE", joined) - self.assertNotIn("not data", joined) - - def test_tier_l3_flow_with_expressions_and_binds(self): - il = emit("L3") - self.assertIn("CALL path.read_bytes() -> data", il) - self.assertIn("IF not data", il) - - def test_l3_is_larger_than_l2_is_larger_than_l1(self): - self.assertGreater(len(emit("L3")), len(emit("L2"))) - self.assertGreater(len(emit("L2")), len(emit("L1"))) - - def test_secret_never_in_output(self): - for tier in ("L1", "L2", "L3"): - self.assertNotIn("ghp_abcdefghijklmnop1234", emit(tier)) - - def test_regex_const_fault_tagged(self): - il = emit("L2") - cline = next(l for l in il.splitlines() if l.startswith("C PATTERN")) - self.assertIn("!FAULT(regex)", cline) - self.assertIn("regex(L", il.splitlines()[-1]) - - def test_omit_counts_and_fault_before(self): - lines = emit("L2").splitlines() - self.assertTrue(lines[-2].startswith("OMIT ")) - self.assertIn("bodies-tier=L2", lines[-2]) - self.assertTrue(lines[-1].startswith("FAULT-BEFORE edit(any), exact-literals")) - - def test_deterministic(self): - self.assertEqual(emit("L2"), emit("L2")) - - -if __name__ == "__main__": - unittest.main() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python3 -m unittest tests.test_emitter -v` -Expected: FAIL — `AttributeError: ... no attribute 'emit_il'` - -- [ ] **Step 3: Write implementation** - -Add to `kern_compile.py`. Also extend `parse_python` const handling to set `risk`: after building the const `Symbol`, set `symbol.risk = expr_risk(node.value)` (requires `expr_risk` from Task 2 — it is already defined above `parse_python` after Task 2 reordering; if not, move it above). - -```python -_TIER_LEVEL = {"L1": 1, "L2": 2, "L3": 3} -_ELIDED = (" str: - parts = [] - for key in sorted(mapping): - vias = mapping[key] - parts.append(key + (f" (via {', '.join(sorted(vias))})" if vias else "")) - if unknown: - parts.append(f"unknown-calls={unknown}") - return ", ".join(parts) - - -def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: - head = "ASYNC F" if s.is_async else "F" - lines = [f"{head} {s.name}({s.signature}) -> {s.returns or 'Any'} " - f"@L{s.span[0]}-{s.span[1]} ^{s.slice8} ~{tier}"] - if s.decorators: - lines.append(" DECORATORS " + ", ".join(s.decorators)) - if s.calls: - shown = s.calls[:25] - extra = f" …+{len(s.calls) - 25}" if len(s.calls) > 25 else "" - lines.append(" CALLS " + ", ".join(shown) + extra) - effects = _render_provenanced(s.effects, s.unknown_calls) - if effects: - lines.append(" EFFECTS " + effects) - raises = _render_provenanced(s.raises_all, 0) - if raises: - lines.append(" RAISES " + raises) - if level >= 2: - for op in s.flow: - pad = " " * (op.depth + 2) - piece = op.op - if level == 3 and op.detail: - piece += f" {op.detail}" - elif level == 2 and op.op in ("CATCH", "NESTED", "CASE") and op.detail: - piece += f" {op.detail}" - if level == 3 and op.binds: - piece += f" -> {op.binds}" - risk = op.risk - if not risk and level == 3 and any(m in op.detail for m in _ELIDED): - risk = "elided-literal" - if risk: - piece += f" !FAULT({risk})" - faults.append(f"{risk}(L{op.line})") - lines.append(pad + piece) - return lines - - -def emit_il(module: ModuleIR, source_rel: str, source_sha256: str, - repo_revision: str = "none", tier: str = "L2") -> str: - level = _TIER_LEVEL[tier] - propagate(module) - out = [ - "KERN-IL/0.2", - f"source_rel={source_rel}", - f"source_sha256={source_sha256}", - f"repo_revision={repo_revision}", - f"generator={GENERATOR} lang={module.lang} frontend={module.frontend} tier={tier}", - "", - ] - faults: list[str] = [] - imports = [s for s in module.symbols if s.kind == "import"] - if imports: - lo = min(s.span[0] for s in imports) - hi = max(s.span[1] for s in imports) - out.append(f"IMPORTS {'; '.join(s.detail for s in imports)} @L{lo}-{hi}") - for s in module.symbols: - if s.kind == "const": - tag = "" - if s.risk: - tag = f" !FAULT({s.risk})" - faults.append(f"{s.risk}(L{s.span[0]})") - out.append(f"C {s.name}={s.detail} @L{s.span[0]}{tag}") - for s in module.symbols: - if s.kind == "class": - out.extend(["", f"CLASS {s.name}({s.bases}) @L{s.span[0]}-{s.span[1]} ^{s.slice8}"]) - elif s.kind == "function": - out.append("") - out.extend(_function_lines(s, level, tier, faults)) - omit = " ".join(f"{k}={v}" for k, v in sorted(module.omit.items())) - out.extend([ - "", - f"OMIT {omit} bodies-tier={tier}", - "FAULT-BEFORE edit(any), exact-literals" - + "".join(f", {f}" for f in dict.fromkeys(faults)), - ]) - return "\n".join(out).rstrip() + "\n" -``` - -- [ ] **Step 4: Run all tests** - -Run: `python3 -m unittest discover -s tests -v` -Expected: all PASS. - -- [ ] **Step 5: Commit** - -```bash -git add skills/kern/scripts/kern_compile.py tests/test_emitter.py -git commit -m "feat: tiered KERN-IL/0.2 emitter with fault tags and omit counts" -``` - ---- - -### Task 5: TS/JS tree-sitter frontend (optional dependency) - -**Files:** -- Modify: `skills/kern/scripts/kern_compile.py` (add `tsjs_available`, `parse_tsjs`) -- Test: `tests/test_tsjs.py` - -**Interfaces:** -- Consumes: `Symbol`, `FlowOp`, `ModuleIR`, `emit_il`. -- Produces: `tsjs_available() -> bool`; `parse_tsjs(text: str, typescript: bool = False) -> ModuleIR` with `frontend="tree-sitter"`, `lang="typescript"|"javascript"`. Tests skip when tree-sitter absent. Task 6 dispatches on file suffix. - -- [ ] **Step 1: Try installing the optional dependency** - -Run: `python3 -m pip install --user tree-sitter tree-sitter-javascript tree-sitter-typescript` -If install fails (no network/permission), proceed anyway — tests skip and the fallback path is exercised instead. Do not add these to any requirements file; they stay optional. - -- [ ] **Step 2: Write the failing test** - -Create `tests/test_tsjs.py`: - -```python -import sys -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) -import kern_compile # noqa: E402 - -TS_SAMPLE = ''' -import { readFile } from "fs/promises"; - -const MAX = 10; - -export class Loader { - async fetch(url: string): Promise { - const data = await readFile(url); - if (!data) { - throw new Error(url); - } - return data.toString(); - } -} - -export function parse(raw: string): number { - return Number(raw); -} -''' - - -class TestAvailability(unittest.TestCase): - def test_available_returns_bool(self): - self.assertIsInstance(kern_compile.tsjs_available(), bool) - - -@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") -class TestTsFrontend(unittest.TestCase): - def setUp(self): - self.mod = kern_compile.parse_tsjs(TS_SAMPLE, typescript=True) - - def sym(self, name): - return next(s for s in self.mod.symbols if s.name == name) - - def test_module_metadata(self): - self.assertEqual(self.mod.lang, "typescript") - self.assertEqual(self.mod.frontend, "tree-sitter") - - def test_function_and_method(self): - f = self.sym("parse") - self.assertEqual(f.kind, "function") - self.assertIn("raw: string", f.signature) - m = self.sym("Loader.fetch") - self.assertTrue(m.is_async) - self.assertIn("readFile", " ".join(m.calls)) - - def test_flow_and_raises(self): - m = self.sym("Loader.fetch") - ops = [o.op for o in m.flow] - self.assertIn("IF", ops) - self.assertIn("RAISE", ops) - self.assertIn("RET", ops) - self.assertIn("Error", m.raises) - - def test_slice_hash_present(self): - self.assertEqual(len(self.sym("parse").slice8), 8) - - def test_emit_works(self): - il = kern_compile.emit_il(self.mod, "src/x.ts", "b" * 64, "none", "L2") - self.assertTrue(il.startswith("KERN-IL/0.2")) - self.assertIn("F parse", il) - - -if __name__ == "__main__": - unittest.main() -``` - -- [ ] **Step 3: Run test to verify it fails** - -Run: `python3 -m unittest tests.test_tsjs -v` -Expected: FAIL — `AttributeError: ... no attribute 'tsjs_available'` (or, with tree-sitter installed, missing `parse_tsjs`). - -- [ ] **Step 4: Write implementation** - -Add to `kern_compile.py`: - -```python -def tsjs_available() -> bool: - try: - import tree_sitter # noqa: F401 - import tree_sitter_javascript # noqa: F401 - import tree_sitter_typescript # noqa: F401 - return True - except ImportError: - return False - - -_TS_FLOW = { - "if_statement": "IF", "for_statement": "LOOP", "for_in_statement": "LOOP", - "while_statement": "WHILE", "do_statement": "WHILE", "try_statement": "TRY", - "return_statement": "RET", "throw_statement": "RAISE", "switch_statement": "MATCH", -} -_TS_FUNC_NODES = {"function_declaration", "generator_function_declaration", "method_definition"} - - -def parse_tsjs(text: str, typescript: bool = False) -> ModuleIR: - from tree_sitter import Language, Parser - if typescript: - import tree_sitter_typescript as ts_lang - language = Language(ts_lang.language_typescript()) - lang_name = "typescript" - else: - import tree_sitter_javascript as js_lang - language = Language(js_lang.language()) - lang_name = "javascript" - parser = Parser(language) - tree = parser.parse(text.encode("utf-8")) - raw = text.encode("utf-8") - - def ntext(n, cap=120): - piece = raw[n.start_byte:n.end_byte].decode("utf-8", "replace") - piece = SPACE.sub(" ", piece).strip() - if SECRET_VALUE.search(piece): - return sanitize_string(piece, secret_hint=True) - return piece[:cap] - - def span(n): - return (n.start_point[0] + 1, n.end_point[0] + 1) - - def field_node(n, name): - return n.child_by_field_name(name) - - def collect_calls(n, acc): - if n.type == "call_expression": - fn = field_node(n, "function") - if fn is not None: - name = ntext(fn, 80) - if name not in acc: - acc.append(name) - for c in n.children: - collect_calls(c, acc) - - def collect_raises(n, acc): - if n.type == "throw_statement": - body = ntext(n, 80) - name = body.removeprefix("throw").strip().removeprefix("new").strip() - name = name.split("(")[0].rstrip(";").strip() - if name and name not in acc: - acc.append(name) - for c in n.children: - collect_raises(c, acc) - - def flow(n, depth=0, budget=200): - ops = [] - - def add(node, op, detail="", binds=""): - if len(ops) < budget: - ops.append(FlowOp(op=op, detail=detail, binds=binds, depth=depth, - line=node.start_point[0] + 1)) - - for c in n.named_children: - if len(ops) >= budget: - break - t = c.type - if t in _TS_FLOW: - cond = field_node(c, "condition") - detail = ntext(cond, 100).strip("()") if cond is not None else "" - if t == "return_statement": - detail = ntext(c, 100).removeprefix("return").strip().rstrip(";") - if t == "throw_statement": - detail = ntext(c, 80).removeprefix("throw").strip().rstrip(";") - add(c, _TS_FLOW[t], detail) - for name in ("body", "consequence"): - inner = field_node(c, name) - if inner is not None: - ops.extend(flow(inner, depth + 1, budget - len(ops))) - alt = field_node(c, "alternative") - if alt is not None: - add(c, "ELSE") - ops.extend(flow(alt, depth + 1, budget - len(ops))) - handler = field_node(c, "handler") - if handler is not None: - add(handler, "CATCH", ntext(field_node(handler, "parameter") or handler, 40)) - hbody = field_node(handler, "body") - if hbody is not None: - ops.extend(flow(hbody, depth + 1, budget - len(ops))) - elif t == "expression_statement" and c.named_children and c.named_children[0].type in ("call_expression", "await_expression"): - add(c, "CALL", ntext(c.named_children[0], 120)) - elif t in ("lexical_declaration", "variable_declaration"): - for d in c.named_children: - if d.type == "variable_declarator": - value = field_node(d, "value") - if value is not None and value.type in ("call_expression", "await_expression"): - name = field_node(d, "name") - add(d, "CALL", ntext(value, 120), binds=ntext(name, 40) if name is not None else "") - else: - ops.extend(flow(c, depth, budget - len(ops))) - return ops[:budget] - - def function_symbol(node, qualified): - calls, raises = [], [] - collect_calls(node, calls) - collect_raises(node, raises) - params = field_node(node, "parameters") - rtype = field_node(node, "return_type") - body = field_node(node, "body") - a, b = span(node) - is_async = any(ch.type == "async" for ch in node.children) - return Symbol( - kind="function", name=qualified, span=(a, b), - signature=ntext(params, 200).strip("()") if params is not None else "", - returns=ntext(rtype, 60).lstrip(": ") if rtype is not None else "", - slice8=slice_sha8(text, a, b), calls=calls, raises=raises, - flow=flow(body) if body is not None else [], is_async=is_async, - ) - - symbols: list[Symbol] = [] - - def top(n, class_prefix=""): - for c in n.named_children: - t = c.type - if t in ("import_statement",): - symbols.append(Symbol(kind="import", name="", detail=ntext(c, 120), span=span(c))) - elif t in ("lexical_declaration", "variable_declaration") and not class_prefix: - for d in c.named_children: - if d.type == "variable_declarator": - name = field_node(d, "name") - value = field_node(d, "value") - if value is not None and value.type in ("arrow_function", "function_expression"): - symbols.append(function_symbol(value, ntext(name, 60))) - elif name is not None: - detail = ntext(value, 100) if value is not None else "" - hint = bool(SECRET_NAME.search(ntext(name, 60))) - if hint: - detail = sanitize_string(detail, secret_hint=True) - symbols.append(Symbol(kind="const", name=ntext(name, 60), detail=detail, span=span(c))) - elif t in _TS_FUNC_NODES: - name = field_node(c, "name") - qual = (class_prefix + ntext(name, 60)) if name is not None else class_prefix + "" - symbols.append(function_symbol(c, qual)) - elif t == "class_declaration": - name = field_node(c, "name") - cname = ntext(name, 60) if name is not None else "" - a, b = span(c) - symbols.append(Symbol(kind="class", name=cname, span=(a, b), slice8=slice_sha8(text, a, b))) - body = field_node(c, "body") - if body is not None: - top(body, class_prefix=cname + ".") - elif t in ("export_statement", "program", "statement_block"): - top(c, class_prefix) - top(tree.root_node) - - lines = text.splitlines() - omit = { - "docstrings": 0, - "comments": sum(1 for l in lines if l.strip().startswith("//")), - "blank": sum(1 for l in lines if not l.strip()), - "assignments": 0, - } - return ModuleIR(lang_name, "tree-sitter", symbols, omit) -``` - -- [ ] **Step 5: Run tests** - -Run: `python3 -m unittest tests.test_tsjs -v` -Expected: PASS (or SKIP with "tree-sitter not installed" — both acceptable; `TestAvailability` must pass either way). If tree-sitter IS installed and node-type names mismatch the grammar version, debug with `python3 -c "import kern_compile; m = kern_compile.parse_tsjs(open('tests/test_tsjs.py').read().split(\"'''\")[1], True); print([s.name for s in m.symbols])"` and adjust `_TS_FLOW`/node names — grammar node types are stable across recent versions but `parameter` vs `parameters` on catch clauses varies. - -- [ ] **Step 6: Commit** - -```bash -git add skills/kern/scripts/kern_compile.py tests/test_tsjs.py -git commit -m "feat: TS/JS tree-sitter frontend with graceful absence" -``` - ---- - -### Task 6: Integrate compiler into kern_cache (codec bump, dispatch, size floor, --tier) - -**Files:** -- Modify: `skills/kern/scripts/kern_cache.py` -- Test: `tests/test_cache_integration.py` - -**Interfaces:** -- Consumes: `kern_compile.parse_python`, `parse_tsjs`, `tsjs_available`, `emit_il` (local import inside `baseline_for`). -- Produces: `baseline_for(root, source, relative, digest, config, tier=None) -> str` (new signature); `git_revision(root: Path) -> str`; `ensure_file(root, paths, relative, source, config, tier=None)`; `prepare_file(root, paths, relative, source, config)`; constants `CODEC_VERSION = "kern-il/0.2"`, `BASELINE_GENERATOR = "kern-det/0.2"`, `TSJS_SUFFIXES = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"}`; config keys `min_ir_tokens: 600`, `default_tier: "L2"`; manifest field `ir_tier`. - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_cache_integration.py`: - -```python -import json -import shutil -import sys -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) -import kern_cache # noqa: E402 - -BIG_PY = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( - f'def fn_{i}(path, n):\n' - f' """Doc {i}."""\n' - f' data = path.read_bytes()\n' - f' if not data:\n' - f' raise ValueError(n)\n' - f' return json.loads(data)\n' - for i in range(30) -) - - -class TestCacheIntegration(unittest.TestCase): - def setUp(self): - self.root = Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) - (self.root / "big.py").write_text(BIG_PY) - (self.root / "tiny.py").write_text("X = 1\n") - self.paths, self.config = kern_cache.initialize(self.root) - - def ensure(self, name, tier=None): - rel, src = kern_cache.normalize_rel(self.root, name) - return kern_cache.ensure_file(self.root, self.paths, rel, src, self.config, tier=tier) - - def test_codec_is_0_2(self): - self.assertEqual(kern_cache.CODEC_VERSION, "kern-il/0.2") - - def test_big_python_file_gets_deterministic_il(self): - result = self.ensure("big.py") - il = Path(result["ir"]).read_text() - self.assertTrue(il.startswith("KERN-IL/0.2")) - self.assertIn("tier=L2", il) - self.assertIn("F fn_0(", il) - self.assertIn("EFFECTS fs:read", il) - self.assertIn("RAISES ValueError", il) - - def test_tier_override(self): - result = self.ensure("big.py", tier="L1") - il = Path(result["ir"]).read_text() - self.assertIn("tier=L1", il) - self.assertNotIn(" IF", il) - manifest = json.loads((self.paths["manifest"]).read_text()) - self.assertEqual(manifest["files"]["big.py"]["ir_tier"], "L1") - - def test_tiny_file_gets_source_cheaper_stub(self): - result = self.ensure("tiny.py") - il = Path(result["ir"]).read_text() - self.assertIn("mode=source-cheaper", il) - self.assertNotIn("F ", il) - - def test_syntax_error_falls_back_to_generic(self): - (self.root / "broken.py").write_text("def broken(:\n pass\n" * 300) - il = Path(self.ensure("broken.py")["ir"]).read_text() - self.assertIn("mode=generic-line-baseline", il) - - def test_repo_revision_header_present(self): - il = Path(self.ensure("big.py")["ir"]).read_text() - self.assertIn("repo_revision=", il) - - -if __name__ == "__main__": - unittest.main() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python3 -m unittest tests.test_cache_integration -v` -Expected: FAIL — `TypeError: ensure_file() got an unexpected keyword argument` and codec assertion failure. - -- [ ] **Step 3: Modify `kern_cache.py`** - -Apply these changes: - -1. Constants (`kern_cache.py:22-24`): -```python -CODEC_VERSION = "kern-il/0.2" -BASELINE_GENERATOR = "kern-det/0.2" -TSJS_SUFFIXES = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"} -``` -2. `DEFAULT_CONFIG`: add `"min_ir_tokens": 600,` and `"default_tier": "L2",` after `"image_profile": "dense",`. -3. Add after `git_files`: -```python -def git_revision(root: Path) -> str: - try: - head = subprocess.run( - ["git", "-C", str(root), "rev-parse", "--short", "HEAD"], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, text=True, - ) - if head.returncode != 0: - return "none" - sha = head.stdout.strip() - dirty = subprocess.run( - ["git", "-C", str(root), "status", "--porcelain"], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, text=True, - ) - return f"dirty:{sha}" if dirty.stdout.strip() else sha - except OSError: - return "none" -``` -4. Delete `sanitize_string`, `LiteralSanitizer`, `expr`, `target_text`, `call_name`, `outline`, `function_card`, `python_ir` (lines ~366-568). Keep `SECRET_NAME`, `SECRET_VALUE`, `SPACE`, `redact_line`, `generic_ir`, `GENERIC_KEEP`. -5. Replace `baseline_for` with: -```python -def stub_ir(text: str, relative: str, digest: str) -> str: - lines = [ - CODEC_VERSION.upper(), - f"source_rel={relative}", - f"source_sha256={digest}", - f"generator={BASELINE_GENERATOR}", - "mode=source-cheaper", - f"QA source is ~{max(1, len(text) // 4)} tokens ({len(text.splitlines())} lines), below the IL floor; fault exact source.", - ] - return "\n".join(lines) + "\n" - - -def baseline_for(root: Path, source: Path, relative: str, digest: str, - config: dict[str, Any], tier: str | None = None) -> tuple[str, str]: - """Return (il_text, tier_used).""" - text = source.read_text(encoding="utf-8", errors="replace") - if max(1, len(text) // 4) < int(config.get("min_ir_tokens", 600)): - return stub_ir(text, relative, digest), "stub" - selected = tier or str(config.get("default_tier", "L2")) - note = "generic language fallback" - try: - import kern_compile - suffix = source.suffix.lower() - module = None - if suffix == ".py": - module = kern_compile.parse_python(text) - elif suffix in TSJS_SUFFIXES and kern_compile.tsjs_available(): - module = kern_compile.parse_tsjs(text, typescript=suffix in {".ts", ".tsx"}) - if module is not None: - if module.parse_error: - note = f"parse failed: {module.parse_error}" - else: - revision = git_revision(root) - return kern_compile.emit_il(module, relative, digest, revision, selected), selected - except Exception as exc: - note = f"deterministic compiler failed: {exc}" - return generic_ir(text, relative, digest, note), "generic" -``` -Note: the IL first line is `KERN-IL/0.2` from both `emit_il` and `generic_ir`/`stub_ir` (they use `CODEC_VERSION.upper()`), so `commit_file`'s existing first-line check keeps working unchanged. -6. `ensure_file`: change signature to `def ensure_file(root, paths, relative, source, config, tier=None)`; replace the two lines -```python - ir = baseline_for(source, relative, digest) -``` -with -```python - ir, tier_used = baseline_for(root, source, relative, digest, config, tier) -``` -and inside the manifest update dict add `"ir_tier": tier_used,` next to `"ir_generator": BASELINE_GENERATOR,`. Also: when `tier` is explicitly passed and differs from the recorded `record.get("ir_tier")`, treat the cache as unusable (add `and (tier is None or record.get("ir_tier") == tier)` to the `usable = (...)` expression). -7. `prepare_file`: change signature to `(root, paths, relative, source, config)` and its call to `ensure_file(root, paths, relative, source, config)`. -8. `sync_cache`: change its `ensure_file(root, paths, relative, source)` call to `ensure_file(root, paths, relative, source, config)`. -9. `parse_args`: on the `ensure` subparser add `command.add_argument("--tier", choices=("L1", "L2", "L3"))` — restructure the `for name in ("ensure", "prepare", "paths"):` loop into individual subparser blocks so only `ensure` gets `--tier`. -10. `main`: `ensure` branch becomes `result = ensure_file(root, paths, relative, source, config, tier=getattr(args, "tier", None))`; `prepare` branch passes `config`. - -- [ ] **Step 4: Run all tests + compile check** - -Run: `python3 -m unittest discover -s tests -v && python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/kern_compile.py skills/kern/scripts/render_ir.py` -Expected: all PASS, no compile errors. - -- [ ] **Step 5: Smoke-test on this repo** - -Run: `python3 skills/kern/scripts/kern_cache.py --repo . scan && python3 skills/kern/scripts/kern_cache.py --repo . ensure skills/kern/scripts/kern_cache.py && head -20 .kern/ir/skills/kern/scripts/kern_cache.py.kern-il.txt && rm -rf .kern` -Expected: JSON success output; IL starts with `KERN-IL/0.2`, contains `tier=L2` and `F ` lines. - -- [ ] **Step 6: Commit** - -```bash -git add skills/kern/scripts/kern_cache.py tests/test_cache_integration.py -git commit -m "feat: kern-il/0.2 codec, deterministic baseline dispatch, size floor, --tier" -``` - ---- - -### Task 7: `verify` CLI verb - -**Files:** -- Modify: `skills/kern/scripts/kern_cache.py` (add `verify_symbol`, CLI wiring) -- Test: `tests/test_verify.py` - -**Interfaces:** -- Consumes: `kern_compile.parse_python` / `parse_tsjs` (local import), Task 6 constants. -- Produces: `verify_symbol(root, paths, relative, source, symbol: str, expected_hash: str, expected_span: str | None) -> dict` with `result` ∈ `"ok" | "moved" | "stale"`; CLI `kern_cache.py --repo R verify FILE --symbol NAME --hash HHHHHHHH [--span La-b]`. Task 9 documents it in SKILL.md. - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_verify.py`: - -```python -import shutil -import sys -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) -import kern_cache # noqa: E402 -import kern_compile # noqa: E402 - -SRC = '''import json - - -def load_entry(path, expected_sha): - data = path.read_bytes() - if not data: - raise ValueError(path) - return json.loads(data) -''' - - -class TestVerify(unittest.TestCase): - def setUp(self): - self.root = Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) - self.file = self.root / "mod.py" - self.file.write_text(SRC) - self.paths, self.config = kern_cache.initialize(self.root) - mod = kern_compile.parse_python(SRC) - self.sym = next(s for s in mod.symbols if s.name == "load_entry") - - def verify(self, expected_hash, span=None): - rel, src = kern_cache.normalize_rel(self.root, "mod.py") - return kern_cache.verify_symbol(self.root, self.paths, rel, src, - "load_entry", expected_hash, span) - - def test_ok(self): - r = self.verify(self.sym.slice8, f"L{self.sym.span[0]}-{self.sym.span[1]}") - self.assertEqual(r["result"], "ok") - - def test_moved_when_file_shifts(self): - self.file.write_text("# new comment line\n" + SRC) - r = self.verify(self.sym.slice8, f"L{self.sym.span[0]}-{self.sym.span[1]}") - self.assertEqual(r["result"], "moved") - self.assertIn("current_span", r) - - def test_stale_when_body_changes(self): - self.file.write_text(SRC.replace("json.loads(data)", "json.loads(data.strip())")) - r = self.verify(self.sym.slice8) - self.assertEqual(r["result"], "stale") - self.assertEqual(r["reason"], "symbol-bytes-changed") - - def test_stale_when_symbol_deleted(self): - self.file.write_text("import json\n") - r = self.verify(self.sym.slice8) - self.assertEqual(r["result"], "stale") - self.assertEqual(r["reason"], "symbol-not-found") - - def test_unsupported_suffix_raises(self): - (self.root / "x.rb").write_text("def x; end\n" * 200) - rel, src = kern_cache.normalize_rel(self.root, "x.rb") - with self.assertRaises(ValueError): - kern_cache.verify_symbol(self.root, self.paths, rel, src, "x", "deadbeef", None) - - -if __name__ == "__main__": - unittest.main() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python3 -m unittest tests.test_verify -v` -Expected: FAIL — `AttributeError: ... no attribute 'verify_symbol'` - -- [ ] **Step 3: Write implementation** - -Add to `kern_cache.py` (after `fault_source`): - -```python -def verify_symbol(root: Path, paths: dict[str, Path], relative: str, source: Path, - symbol: str, expected_hash: str, expected_span: str | None = None) -> dict[str, Any]: - import kern_compile - text = source.read_text(encoding="utf-8", errors="replace") - suffix = source.suffix.lower() - if suffix == ".py": - module = kern_compile.parse_python(text) - elif suffix in TSJS_SUFFIXES and kern_compile.tsjs_available(): - module = kern_compile.parse_tsjs(text, typescript=suffix in {".ts", ".tsx"}) - else: - raise ValueError(f"verify does not support {suffix or 'this file type'}; use fault with --expect-sha") - if module.parse_error: - raise RuntimeError(f"current source does not parse ({module.parse_error}); fault exact source") - base = {"ok": True, "operation": "verify", "source_rel": relative, "symbol": symbol, - "source_sha256": sha256_bytes(text.encode("utf-8", "surrogatepass"))} - matches = [s for s in module.symbols if s.kind in {"function", "class"} and s.name == symbol] - if not matches: - return {**base, "result": "stale", "reason": "symbol-not-found"} - found = matches[0] - current_span = f"L{found.span[0]}-{found.span[1]}" - if found.slice8 != expected_hash: - return {**base, "result": "stale", "reason": "symbol-bytes-changed", - "current_hash": found.slice8, "current_span": current_span} - if expected_span and expected_span != current_span: - return {**base, "result": "moved", "current_span": current_span} - return {**base, "result": "ok", "current_span": current_span} -``` - -CLI wiring in `parse_args`: -```python - verify = sub.add_parser("verify") - verify.add_argument("file") - verify.add_argument("--symbol", required=True) - verify.add_argument("--hash", required=True) - verify.add_argument("--span") -``` -and in `main`, inside the file-command branch: -```python - elif args.command == "verify": - result = verify_symbol(root, paths, relative, source, args.symbol, args.hash, args.span) -``` - -- [ ] **Step 4: Run all tests** - -Run: `python3 -m unittest discover -s tests -v` -Expected: all PASS. - -- [ ] **Step 5: Commit** - -```bash -git add skills/kern/scripts/kern_cache.py tests/test_verify.py -git commit -m "feat: verify verb traps stale symbol reads" -``` - ---- - -### Task 8: Token benchmark harness - -**Files:** -- Create: `benchmarks/token_bench.py` -- Test: `tests/test_token_bench.py` - -**Interfaces:** -- Consumes: `kern_compile.parse_python`, `emit_il`. -- Produces: `benchmarks/token_bench.py` CLI: `python3 benchmarks/token_bench.py FILE [FILE...] [--out results.json]`. Report schema: `{"schema": "kern-bench/0.2", "estimator": "chars/4", "files": [{"file", "source_tokens", "bucket", "tiers": {"L1": {"tokens", "ratio"}, ...}, "fidelity_missing": [...]}]}`. Importable functions `estimate_tokens(text)`, `bench_file(path)`. - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_token_bench.py`: - -```python -import importlib.util -import sys -import tempfile -import unittest -from pathlib import Path - -REPO = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO / "skills" / "kern" / "scripts")) - -spec = importlib.util.spec_from_file_location("token_bench", REPO / "benchmarks" / "token_bench.py") -token_bench = importlib.util.module_from_spec(spec) -spec.loader.exec_module(token_bench) - -BIG = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( - f'def fn_{i}(path):\n' - f' """Doc {i} with a somewhat longer explanatory sentence to add bulk."""\n' - f' # a comment line adding source-only weight\n' - f' data = path.read_bytes()\n' - f' if not data:\n' - f' raise ValueError(path)\n' - f' return json.loads(data)\n' - for i in range(40) -) - - -class TestTokenBench(unittest.TestCase): - def setUp(self): - self.tmp = Path(tempfile.mkdtemp()) - self.f = self.tmp / "big.py" - self.f.write_text(BIG) - - def test_bench_file_shape(self): - row = token_bench.bench_file(self.f) - self.assertIn("source_tokens", row) - self.assertEqual(set(row["tiers"]), {"L1", "L2", "L3"}) - for tier in row["tiers"].values(): - self.assertGreater(tier["ratio"], 1.0) - - def test_tier_ordering(self): - row = token_bench.bench_file(self.f) - self.assertGreater(row["tiers"]["L1"]["ratio"], row["tiers"]["L2"]["ratio"]) - self.assertGreater(row["tiers"]["L2"]["ratio"], row["tiers"]["L3"]["ratio"]) - - def test_fidelity_no_missing_functions(self): - row = token_bench.bench_file(self.f) - self.assertEqual(row["fidelity_missing"], []) - - def test_parse_error_reported_not_raised(self): - bad = self.tmp / "bad.py" - bad.write_text("def broken(:\n") - row = token_bench.bench_file(bad) - self.assertIn("error", row) - - -if __name__ == "__main__": - unittest.main() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python3 -m unittest tests.test_token_bench -v` -Expected: FAIL — `FileNotFoundError` for `benchmarks/token_bench.py`. - -- [ ] **Step 3: Write implementation** - -Create `benchmarks/token_bench.py`: - -```python -#!/usr/bin/env python3 -"""Token benchmark: source vs deterministic KERN-IL per tier, bucketed by size.""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) -import kern_compile # noqa: E402 - - -def estimate_tokens(text: str) -> int: - return max(1, len(text) // 4) - - -def bucket(tokens: int) -> str: - if tokens < 2_000: - return "small(<2k)" - if tokens < 10_000: - return "medium(2k-10k)" - return "large(>10k)" - - -def fidelity_missing(module, il: str) -> list[str]: - missing = [] - for symbol in module.symbols: - if symbol.kind == "function" and symbol.name.split(".")[-1] not in il: - missing.append(symbol.name) - return missing - - -def bench_file(path: Path) -> dict: - text = path.read_text(encoding="utf-8", errors="replace") - module = kern_compile.parse_python(text) - if module.parse_error: - return {"file": str(path), "error": module.parse_error} - source_tokens = estimate_tokens(text) - row = {"file": str(path), "source_tokens": source_tokens, - "bucket": bucket(source_tokens), "tiers": {}, "fidelity_missing": []} - for tier in ("L1", "L2", "L3"): - il = kern_compile.emit_il(module, path.name, "0" * 64, "none", tier) - il_tokens = estimate_tokens(il) - row["tiers"][tier] = {"tokens": il_tokens, "ratio": round(source_tokens / il_tokens, 2)} - if tier == "L2": - row["fidelity_missing"] = fidelity_missing(module, il) - return row - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("files", nargs="+", type=Path) - parser.add_argument("--out", type=Path, help="Write JSON report here as well as stdout") - args = parser.parse_args() - report = {"schema": "kern-bench/0.2", "estimator": "chars/4", - "files": [bench_file(f) for f in args.files]} - payload = json.dumps(report, indent=2, sort_keys=True) - if args.out: - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(payload + "\n") - print(payload) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) -``` - -- [ ] **Step 4: Run tests and a real benchmark** - -Run: `python3 -m unittest tests.test_token_bench -v` -Expected: PASS. -Run: `python3 benchmarks/token_bench.py skills/kern/scripts/kern_cache.py --out benchmarks/results/python-det-v2.json` -Expected: JSON report; L2 ratio on `kern_cache.py` should be ≥ 3× (it is a medium file; the ≥6× acceptance target applies to the large bucket only). - -- [ ] **Step 5: Commit** - -```bash -git add benchmarks/token_bench.py benchmarks/results/python-det-v2.json tests/test_token_bench.py -git commit -m "feat: per-tier token benchmark with fidelity check" -``` - ---- - -### Task 9: Documentation and skill contract - -**Files:** -- Modify: `skills/kern/SKILL.md` -- Modify: `docs/architecture.md` -- Modify: `README.md` (Development section + Language coverage sentence) -- Modify: `CHANGELOG.md` - -**Interfaces:** -- Consumes: everything shipped in Tasks 1-8. -- Produces: user-facing contract for `verify`, tiers, and the deterministic pipeline. - -- [ ] **Step 1: Update `skills/kern/SKILL.md`** - -In the "JIT a requested file" section, after the `ensure` code block, add: - -```markdown -`ensure` accepts `--tier L1|L2|L3` (default from config `default_tier`, `L2`). -L1 = signatures + calls + effects + raises; L2 = + control-flow skeleton; -L3 = + expressions and dataflow. Files below the `min_ir_tokens` floor get a -`mode=source-cheaper` stub — read exact source instead. -``` - -In the "Fault exact source before edits" section, add before the existing code block: - -```markdown -Before editing any symbol read from IL or an image, verify its source-map handle: - - python3 /scripts/kern_cache.py --repo verify path/to/file \ - --symbol --hash [--span L-L] - -`ok` — proceed. `moved` — same bytes at a new span; use the returned span. -`stale` — the symbol changed; the IL page is invalid, fault exact source. -Lines tagged `!FAULT(reason)` (regex, math, concurrency, elided-literal) may not -support a claim or an edit without an exact-source fault, regardless of verify. -``` - -- [ ] **Step 2: Update `docs/architecture.md`** - -Add invariant `7. Every IL symbol carries a slice hash; a passing verify or fresh fault is required before that symbol is edited.` and change the Lifecycle line to `scan → hash → invalidate → compile(tiered) → render → page in → verify/fault → write → invalidate`. - -- [ ] **Step 3: Update `README.md`** - -In "Language and runtime coverage", replace the first sentence with: - -```markdown -Python and (when tree-sitter is installed) JavaScript/TypeScript receive a deterministic -AST-based compiler with tiered detail, computed side effects, and exception propagation. -Other recognized source formats receive a deterministic generic baseline. -``` - -In "Development", replace the commands block with: - -```bash -npm ci -npm run build -python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/render_ir.py skills/kern/scripts/kern_compile.py -python3 -m unittest discover -s tests -python3 skills/kern/scripts/kern_cache.py --repo . scan -``` - -- [ ] **Step 4: Update `CHANGELOG.md`** - -Add an entry at the top following the file's existing format: codec `kern-il/0.2`, deterministic tiered compiler, effects/raises, verify verb, size floor, token benchmark. Bump the minor version consistent with the file's convention. - -- [ ] **Step 5: Full check** - -Run: `python3 -m unittest discover -s tests -v && python3 -m py_compile skills/kern/scripts/kern_cache.py skills/kern/scripts/render_ir.py skills/kern/scripts/kern_compile.py` -Expected: all PASS. - -- [ ] **Step 6: Commit** - -```bash -git add skills/kern/SKILL.md docs/architecture.md README.md CHANGELOG.md -git commit -m "docs: kern-il/0.2 tiers, verify contract, dev commands" -``` - ---- - -### Task 10: Enrichment is append-only (commit_file contract) - -**Files:** -- Modify: `skills/kern/scripts/kern_cache.py:691-764` (`commit_file`) -- Modify: `skills/kern/references/compiler-worker.md` -- Test: `tests/test_enrichment.py` - -**Interfaces:** -- Consumes: Task 6 `ensure_file`/`baseline_for` (deterministic IL on disk before enrichment). -- Produces: `commit_file` rejects any staging IL that does not consist of the current committed deterministic IL followed by an `ENRICHMENT model=` section containing only `INTENT : ` lines. Spec rule: model output never replaces deterministic facts. - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_enrichment.py`: - -```python -import shutil -import sys -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) -import kern_cache # noqa: E402 - -BIG_PY = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( - f'def fn_{i}(path):\n' - f' data = path.read_bytes()\n' - f' return json.loads(data)\n' - for i in range(40) -) - - -class TestEnrichmentAppendOnly(unittest.TestCase): - def setUp(self): - self.root = Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) - (self.root / "mod.py").write_text(BIG_PY) - self.paths, self.config = kern_cache.initialize(self.root) - rel, src = kern_cache.normalize_rel(self.root, "mod.py") - self.rel, self.src = rel, src - ensured = kern_cache.ensure_file(self.root, self.paths, rel, src, self.config) - self.digest = ensured["source_sha256"] - self.baseline = Path(ensured["ir"]).read_text() - - def commit(self, staging_text): - staging = self.root / "staging.kern-il.txt" - staging.write_text(staging_text) - return kern_cache.commit_file(self.root, self.paths, self.rel, self.src, - staging, self.digest) - - def test_valid_append_accepted(self): - staged = self.baseline + "\nENRICHMENT model=test-model\nINTENT fn_0: reads and parses a JSON file\n" - result = self.commit(staged) - self.assertEqual(result["status"], "ready") - - def test_replacement_rejected(self): - rogue = self.baseline.replace("F fn_0(", "F totally_different(") - with self.assertRaises(ValueError): - self.commit(rogue + "\nENRICHMENT model=test-model\nINTENT fn_0: x\n") - - def test_missing_enrichment_header_rejected(self): - with self.assertRaises(ValueError): - self.commit(self.baseline + "\nINTENT fn_0: no header line\n") - - def test_non_intent_lines_rejected(self): - staged = self.baseline + "\nENRICHMENT model=test-model\nF injected_fact() -> Any @L1-1 ^deadbeef ~L2\n" - with self.assertRaises(ValueError): - self.commit(staged) - - -if __name__ == "__main__": - unittest.main() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python3 -m unittest tests.test_enrichment -v` -Expected: FAIL — `test_replacement_rejected` and the others fail because `commit_file` currently accepts any well-headed IL. - -- [ ] **Step 3: Modify `commit_file`** - -In `commit_file`, after the existing header validation (`generator` check) and before `atomic_write`, add: - -```python - baseline_path = artifact_paths(paths, relative)["ir"] - if not baseline_path.is_file(): - raise ValueError("No deterministic baseline IL exists; run ensure before commit") - baseline_text = baseline_path.read_text(encoding="utf-8") - if not text.startswith(baseline_text.rstrip("\n")): - raise ValueError("Enrichment must preserve the deterministic IL verbatim as a prefix") - appended = text[len(baseline_text.rstrip("\n")):].strip("\n") - if appended: - appended_lines = appended.splitlines() - if not appended_lines[0].startswith("ENRICHMENT model="): - raise ValueError("Appended section must start with 'ENRICHMENT model='") - for line in appended_lines[1:]: - if line.strip() and not line.startswith("INTENT "): - raise ValueError(f"Enrichment may only append INTENT lines, found: {line[:60]!r}") -``` - -Note: `artifact_paths` is already imported in scope; `text` is the decoded staging payload already available in `commit_file`. - -- [ ] **Step 4: Rewrite `skills/kern/references/compiler-worker.md`** - -Replace the enrichment contract with the new one (keep the file's existing tone/format): - -```markdown -# Compiler worker contract (enrichment) - -The deterministic IL is authoritative and already committed. Your job is ONLY to -append intent summaries. Output the baseline IL verbatim, then: - - ENRICHMENT model= - INTENT : - -Rules: -- Never modify, reorder, or omit any deterministic line. The commit will be - rejected if the baseline is not a byte-exact prefix of your output. -- Only `INTENT` lines may follow the `ENRICHMENT` header. -- One INTENT line per symbol, at most; skip symbols whose purpose is obvious. -- Never include secrets, credentials, or long literals. -``` - -- [ ] **Step 5: Run all tests** - -Run: `python3 -m unittest discover -s tests -v` -Expected: all PASS. - -- [ ] **Step 6: Commit** - -```bash -git add skills/kern/scripts/kern_cache.py skills/kern/references/compiler-worker.md tests/test_enrichment.py -git commit -m "feat: enrichment is append-only INTENT lines over deterministic IL" -``` From 69165e79734ca2311203a26e8e6ceaa0e57250bb Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 18:25:48 +0200 Subject: [PATCH 29/55] feat: operation log and log verb for KERN cache Co-Authored-By: Claude Fable 5 --- skills/kern/SKILL.md | 6 ++ skills/kern/scripts/kern_cache.py | 132 ++++++++++++++++++++++++++++++ tests/test_log.py | 114 ++++++++++++++++++++++++++ 3 files changed, 252 insertions(+) create mode 100644 tests/test_log.py diff --git a/skills/kern/SKILL.md b/skills/kern/SKILL.md index fd83b1c..260c519 100644 --- a/skills/kern/SKILL.md +++ b/skills/kern/SKILL.md @@ -116,6 +116,12 @@ python3 /scripts/kern_cache.py --repo status python3 /scripts/kern_cache.py --repo paths path/to/file ``` +To show recent KERN activity, run the operation log: + +```bash +python3 /scripts/kern_cache.py --repo log --tail 20 +``` + If parsing fails, use the generic baseline, record the limitation, and fault raw source. If Pillow is unavailable, retain textual IL and report the renderer dependency rather than installing packages without permission. The cache mirrors repository paths under: diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 2b48d08..4889453 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -140,6 +140,7 @@ def cache_paths(root: Path) -> dict[str, Path]: "images": cache / "images", "jobs": cache / "jobs", "staging": cache / "staging", + "log": cache / "log.jsonl", } @@ -854,6 +855,93 @@ def span_of(sym) -> str: "candidates": [{"span": span_of(m), "hash": m.slice8} for m in matches]} +def log_event(paths: dict[str, Path], entry: dict[str, Any]) -> None: + """Append one JSON line to the operation log. Never raises.""" + try: + with paths["log"].open("a", encoding="utf-8") as handle: + handle.write(json.dumps(entry, sort_keys=True) + "\n") + except OSError: + pass + + +def log_fields_from_result(result: dict[str, Any]) -> dict[str, Any]: + fields: dict[str, Any] = {} + for key in ("source_rel", "status", "result", "reason", "counts"): + if key in result: + fields[key] = result[key] + return fields + + +def ensure_log_fields( + paths: dict[str, Path], relative: str, source: Path, result: dict[str, Any] +) -> dict[str, Any]: + """Extra telemetry for the ensure command: tier and cheap token estimates.""" + fields: dict[str, Any] = {} + try: + tier = result.get("tier") + if tier is None: + manifest = json.loads(paths["manifest"].read_text(encoding="utf-8")) + record = manifest.get("files", {}).get(relative, {}) + tier = record.get("ir_tier") + if tier is not None: + fields["tier"] = tier + except (OSError, json.JSONDecodeError): + pass + try: + fields["source_tokens"] = source.stat().st_size // 4 + except OSError: + pass + try: + artifacts = artifact_paths(paths, relative) + fields["il_tokens"] = artifacts["ir"].stat().st_size // 4 + except OSError: + pass + return fields + + +def read_log_entries(paths: dict[str, Path], tail: int, op_filter: str | None) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + try: + with paths["log"].open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + if op_filter and entry.get("op") != op_filter: + continue + entries.append(entry) + except OSError: + return [] + if tail is not None and tail >= 0: + entries = entries[-tail:] + return entries + + +def print_log(entries: list[dict[str, Any]], as_json: bool) -> None: + if not entries: + print("no log entries") + return + if as_json: + for entry in entries: + print(json.dumps(entry, sort_keys=True)) + return + print(f"{'TS':<21} {'OP':<8} {'FILE':<32} {'STATUS/RESULT':<14} {'MS':>8}") + for entry in entries: + ts = str(entry.get("ts", "-")) + op = str(entry.get("op", "-")) + file_ = str(entry.get("source_rel", "-")) + status = entry.get("status") or entry.get("result") or "-" + duration = entry.get("duration_ms") + ms = str(duration) if isinstance(duration, int) else "-" + print(f"{ts:<21} {op:<8} {file_:<32} {str(status):<14} {ms:>8}") + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo", default=".", help="Repository root (default: current directory)") @@ -888,14 +976,22 @@ def parse_args() -> argparse.Namespace: verify.add_argument("--symbol", required=True) verify.add_argument("--hash", required=True) verify.add_argument("--span") + log_cmd = sub.add_parser("log") + log_cmd.add_argument("--tail", type=int, default=20) + log_cmd.add_argument("--op") + log_cmd.add_argument("--json", action="store_true") return parser.parse_args() def main() -> int: args = parse_args() + paths: dict[str, Path] | None = None + started = time.monotonic() try: root = repo_root(args.repo) paths, config = initialize(root) + relative: str | None = None + source: Path | None = None if args.command == "init": result = {"ok": True, "operation": "init", "repo": str(root), "cache": str(paths["cache"])} elif args.command == "scan": @@ -904,6 +1000,10 @@ def main() -> int: result = status(root, paths) elif args.command == "sync": result = sync_cache(root, paths, config, args.eager, args.limit) + elif args.command == "log": + entries = read_log_entries(paths, args.tail, args.op) + print_log(entries, args.json) + return 0 else: relative, source = normalize_rel(root, args.file) if args.command == "ensure": @@ -918,6 +1018,18 @@ def main() -> int: result = render_file(root, paths, config, relative, source, args.profile) elif args.command == "fault": sys.stdout.write(fault_source(source, relative, args.start, args.end, args.expect_sha)) + log_event( + paths, + { + "ts": now_iso(), + "op": "fault", + "source_rel": relative, + "start": args.start, + "end": args.end, + "ok": True, + "duration_ms": int((time.monotonic() - started) * 1000), + }, + ) return 0 elif args.command == "verify": result = verify_symbol(root, paths, relative, source, args.symbol, args.hash, args.span) @@ -925,8 +1037,28 @@ def main() -> int: raise RuntimeError(f"Unknown command: {args.command}") json.dump(result, sys.stdout, indent=2, sort_keys=True) sys.stdout.write("\n") + entry = { + "ts": now_iso(), + "op": args.command, + "duration_ms": int((time.monotonic() - started) * 1000), + "ok": True, + } + entry.update(log_fields_from_result(result)) + if args.command == "ensure" and relative is not None and source is not None: + entry.update(ensure_log_fields(paths, relative, source, result)) + log_event(paths, entry) return 0 except Exception as exc: + if paths is not None: + log_event( + paths, + { + "ts": now_iso(), + "op": getattr(args, "command", "?"), + "ok": False, + "error": str(exc), + }, + ) json.dump({"ok": False, "error": str(exc), "type": exc.__class__.__name__}, sys.stderr, indent=2) sys.stderr.write("\n") return 2 diff --git a/tests/test_log.py b/tests/test_log.py new file mode 100644 index 0000000..a554b8d --- /dev/null +++ b/tests/test_log.py @@ -0,0 +1,114 @@ +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPTS = Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts" +sys.path.insert(0, str(SCRIPTS)) +import kern_cache # noqa: E402 + +KERN_CACHE = SCRIPTS / "kern_cache.py" + +BIG_PY = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( + f'def fn_{i}(path, n):\n' + f' """Doc {i}."""\n' + f' data = path.read_bytes()\n' + f' if not data:\n' + f' raise ValueError(n)\n' + f' return json.loads(data)\n' + for i in range(30) +) + + +def run_cli(*args): + return subprocess.run( + [sys.executable, str(KERN_CACHE), *args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + + +class TestOperationLogging(unittest.TestCase): + def setUp(self): + self.root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + (self.root / "big.py").write_text(BIG_PY) + + def log_lines(self): + log_path = self.root / ".kern" / "log.jsonl" + self.assertTrue(log_path.is_file(), "expected .kern/log.jsonl to exist") + return [line for line in log_path.read_text().splitlines() if line.strip()] + + def test_ensure_writes_log_entry(self): + result = run_cli("--repo", str(self.root), "ensure", "big.py") + self.assertEqual(result.returncode, 0, result.stderr) + lines = self.log_lines() + self.assertGreaterEqual(len(lines), 1) + entry = json.loads(lines[-1]) + self.assertEqual(entry["op"], "ensure") + self.assertIs(entry["ok"], True) + self.assertEqual(entry["source_rel"], "big.py") + self.assertIsInstance(entry["duration_ms"], int) + + def test_error_logged(self): + result = run_cli("--repo", str(self.root), "ensure", "does-not-exist.py") + self.assertEqual(result.returncode, 2) + lines = self.log_lines() + entry = json.loads(lines[-1]) + self.assertIs(entry["ok"], False) + self.assertIn("error", entry) + self.assertTrue(entry["error"]) + + +class TestLogCommand(unittest.TestCase): + def setUp(self): + self.root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root, ignore_errors=True) + self.paths, _ = kern_cache.initialize(self.root) + + def seed_log(self, n=30): + ops = ["ensure", "scan", "status", "render", "commit"] + lines = [] + for i in range(n): + op = ops[i % len(ops)] + entry = { + "ts": f"2026-01-01T00:00:{i:02d}Z", + "op": op, + "ok": True, + "duration_ms": i, + "source_rel": f"file_{i}.py", + "status": "ready", + } + lines.append(json.dumps(entry, sort_keys=True)) + lines.append("{this is not valid json") + self.paths["log"].write_text("\n".join(lines) + "\n") + + def test_log_tail_and_filter(self): + self.seed_log() + result = run_cli("--repo", str(self.root), "log", "--tail", "5", "--op", "ensure", "--json") + self.assertEqual(result.returncode, 0, result.stderr) + lines = [line for line in result.stdout.splitlines() if line.strip()] + self.assertEqual(len(lines), 5) + for line in lines: + entry = json.loads(line) + self.assertEqual(entry["op"], "ensure") + + def test_log_human_table_and_empty(self): + result = run_cli("--repo", str(self.root), "log") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("no log entries", result.stdout.lower()) + + self.seed_log() + result2 = run_cli("--repo", str(self.root), "log", "--tail", "10") + self.assertEqual(result2.returncode, 0, result2.stderr) + self.assertIn("ensure", result2.stdout) + self.assertIn("file_", result2.stdout) + + +if __name__ == "__main__": + unittest.main() From 4601bd6f7c24a1bd5aec6aec277c9029963aa25d Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 18:34:05 +0200 Subject: [PATCH 30/55] fix: redact logged errors, harden log_event, tail-0, error file context Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 37 ++++++++++++++++--------------- tests/test_log.py | 35 +++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 4889453..79fc3fb 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -595,11 +595,13 @@ def commit_file( if not appended_lines[0].startswith("ENRICHMENT model="): raise ValueError("Appended section must start with 'ENRICHMENT model='") for line in appended_lines: - if SECRET_VALUE.search(line): + if SECRET_VALUE.search(line) or (SECRET_NAME.search(line) and re.search(r"[:=]", line)): raise ValueError("Enrichment line contains a likely credential") for line in appended_lines[1:]: if line.strip() and not line.startswith("INTENT "): - raise ValueError(f"Enrichment may only append INTENT lines, found: {line[:60]!r}") + raise ValueError( + f"Enrichment may only append INTENT lines, found: {redact_line(line)[:60]!r}" + ) payload = payload.rstrip(b"\n") + b"\n" atomic_write(artifacts["ir"], payload) if sha256_file(source) != expected_sha: @@ -860,7 +862,7 @@ def log_event(paths: dict[str, Path], entry: dict[str, Any]) -> None: try: with paths["log"].open("a", encoding="utf-8") as handle: handle.write(json.dumps(entry, sort_keys=True) + "\n") - except OSError: + except (OSError, TypeError, ValueError): pass @@ -878,11 +880,9 @@ def ensure_log_fields( """Extra telemetry for the ensure command: tier and cheap token estimates.""" fields: dict[str, Any] = {} try: - tier = result.get("tier") - if tier is None: - manifest = json.loads(paths["manifest"].read_text(encoding="utf-8")) - record = manifest.get("files", {}).get(relative, {}) - tier = record.get("ir_tier") + manifest = json.loads(paths["manifest"].read_text(encoding="utf-8")) + record = manifest.get("files", {}).get(relative, {}) + tier = record.get("ir_tier") if tier is not None: fields["tier"] = tier except (OSError, json.JSONDecodeError): @@ -919,7 +919,7 @@ def read_log_entries(paths: dict[str, Path], tail: int, op_filter: str | None) - except OSError: return [] if tail is not None and tail >= 0: - entries = entries[-tail:] + entries = entries[-tail:] if tail > 0 else [] return entries @@ -1050,15 +1050,16 @@ def main() -> int: return 0 except Exception as exc: if paths is not None: - log_event( - paths, - { - "ts": now_iso(), - "op": getattr(args, "command", "?"), - "ok": False, - "error": str(exc), - }, - ) + error_entry: dict[str, Any] = { + "ts": now_iso(), + "op": getattr(args, "command", "?"), + "ok": False, + "error": redact_line(str(exc)), + } + source_rel = getattr(args, "file", None) + if source_rel is not None: + error_entry["source_rel"] = source_rel + log_event(paths, error_entry) json.dump({"ok": False, "error": str(exc), "type": exc.__class__.__name__}, sys.stderr, indent=2) sys.stderr.write("\n") return 2 diff --git a/tests/test_log.py b/tests/test_log.py index a554b8d..639a9c8 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -64,6 +64,26 @@ def test_error_logged(self): self.assertIn("error", entry) self.assertTrue(entry["error"]) + def test_error_log_never_contains_credential_shaped_content(self): + ensure_result = run_cli("--repo", str(self.root), "ensure", "big.py") + self.assertEqual(ensure_result.returncode, 0, ensure_result.stderr) + ensured = json.loads(ensure_result.stdout) + baseline = Path(ensured["ir"]).read_text() + source_sha = ensured["source_sha256"] + + staging = self.root / "staging.kern-il.txt" + staging.write_text(baseline + "\nENRICHMENT model=m\nDB_PASSWORD=hunter2secretvalue\n") + + commit_result = run_cli( + "--repo", str(self.root), "commit", "big.py", + "--ir-file", str(staging), "--source-sha", source_sha, + ) + self.assertEqual(commit_result.returncode, 2) + self.assertNotIn("hunter2secretvalue", commit_result.stderr) + + log_contents = (self.root / ".kern" / "log.jsonl").read_text() + self.assertNotIn("hunter2secretvalue", log_contents) + class TestLogCommand(unittest.TestCase): def setUp(self): @@ -98,6 +118,21 @@ def test_log_tail_and_filter(self): entry = json.loads(line) self.assertEqual(entry["op"], "ensure") + def test_log_event_does_not_raise_on_non_serializable_field(self): + # A set is not JSON serializable; log_event must swallow the resulting + # TypeError rather than letting it abort the command. + try: + kern_cache.log_event( + self.paths, {"ts": "2026-01-01T00:00:00Z", "op": "ensure", "ok": True, "weird": {1, 2, 3}} + ) + except Exception as exc: # pragma: no cover - failure path + self.fail(f"log_event raised unexpectedly: {exc!r}") + + def test_read_log_entries_tail_zero_is_empty(self): + self.seed_log() + entries = kern_cache.read_log_entries(self.paths, tail=0, op_filter=None) + self.assertEqual(entries, []) + def test_log_human_table_and_empty(self): result = run_cli("--repo", str(self.root), "log") self.assertEqual(result.returncode, 0, result.stderr) From aa33d78710cfe127166fb6ba9380e0442a539194 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 18:39:26 +0200 Subject: [PATCH 31/55] fix: value-scrubbing redaction and assignment-adjacent secret gate Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 12 +++++++----- tests/test_enrichment.py | 10 ++++++++++ tests/test_log.py | 4 ++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 79fc3fb..3b3f858 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -43,13 +43,14 @@ "default_tier": "L2", } -SECRET_NAME = re.compile( - r"(?i)(api[_-]?key|access[_-]?key|auth|bearer|credential|passwd|password|private[_-]?key|secret|token)" -) SECRET_VALUE = re.compile( r"(?i)(?:sk|rk|pk|s2)[_-][A-Za-z0-9_-]{12,}|-----BEGIN [A-Z ]*PRIVATE KEY-----|" r"(?:aws|ghp|github_pat)_[A-Za-z0-9_-]{12,}" ) +SECRET_ASSIGNMENT = re.compile( + r"(?i)(api[_-]?key|access[_-]?key|auth|bearer|credential|passwd|password|private[_-]?key|secret|token)" + r"['\"]?\s*[:=]\s*\S" +) SPACE = re.compile(r"\s+") @@ -393,7 +394,8 @@ def refresh_one(root: Path, paths: dict[str, Path], relative: str, source: Path) def redact_line(line: str) -> str: - if SECRET_VALUE.search(line) or (SECRET_NAME.search(line) and re.search(r"[:=]", line)): + line = SECRET_VALUE.sub(lambda m: f"", line) + if SECRET_ASSIGNMENT.search(line): digest = sha256_bytes(line.encode("utf-8", "replace"))[:12] left = re.split(r"[:=]", line, maxsplit=1)[0] return f"{left}=" @@ -595,7 +597,7 @@ def commit_file( if not appended_lines[0].startswith("ENRICHMENT model="): raise ValueError("Appended section must start with 'ENRICHMENT model='") for line in appended_lines: - if SECRET_VALUE.search(line) or (SECRET_NAME.search(line) and re.search(r"[:=]", line)): + if SECRET_VALUE.search(line) or SECRET_ASSIGNMENT.search(line): raise ValueError("Enrichment line contains a likely credential") for line in appended_lines[1:]: if line.strip() and not line.startswith("INTENT "): diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py index c4a5ac5..59a7d26 100644 --- a/tests/test_enrichment.py +++ b/tests/test_enrichment.py @@ -84,6 +84,16 @@ def test_enrichment_header_with_secret_rejected(self): with self.assertRaises(ValueError): self.commit(staged) + def test_intent_prose_mentioning_token_accepted(self): + staged = self.baseline + "\nENRICHMENT model=test-model\nINTENT fn_0: parses bearer token from Authorization header\n" + result = self.commit(staged) + self.assertEqual(result["status"], "ready") + + def test_intent_assignment_shaped_secret_rejected(self): + staged = self.baseline + "\nENRICHMENT model=test-model\nINTENT fn_0: uses password=hunter2secretvalue here\n" + with self.assertRaises(ValueError): + self.commit(staged) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_log.py b/tests/test_log.py index 639a9c8..3bba9f1 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -84,6 +84,10 @@ def test_error_log_never_contains_credential_shaped_content(self): log_contents = (self.root / ".kern" / "log.jsonl").read_text() self.assertNotIn("hunter2secretvalue", log_contents) + def test_redact_line_scrubs_value_without_delimiter(self): + out = kern_cache.redact_line("push rejected using ghp_ABCDEFGHIJKLMNOP12345 by client") + self.assertNotIn("ghp_ABCDEFGHIJKLMNOP12345", out) + class TestLogCommand(unittest.TestCase): def setUp(self): From 393fd3c4f5ec64dbf1810832fe8a4e3292ef7e2b Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 18:45:01 +0200 Subject: [PATCH 32/55] fix: broad redaction trigger for redact_line, strict gate unchanged redact_line used the strict SECRET_ASSIGNMENT adjacency pattern as its cut trigger, so lines like "Authorization: Bearer eyJ..." passed through unredacted (auth is 9 chars from the colon, and JWTs don't match SECRET_VALUE prefixes). Reintroduce the broad SECRET_NAME keyword pattern for redact_line's trigger (over-redaction there is harmless); commit_file's enrichment gate keeps the strict SECRET_VALUE/SECRET_ASSIGNMENT check so INTENT prose mentioning "token" or "bearer" is still accepted. Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 5 ++++- tests/test_log.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 3b3f858..e07e4d0 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -47,6 +47,9 @@ r"(?i)(?:sk|rk|pk|s2)[_-][A-Za-z0-9_-]{12,}|-----BEGIN [A-Z ]*PRIVATE KEY-----|" r"(?:aws|ghp|github_pat)_[A-Za-z0-9_-]{12,}" ) +SECRET_NAME = re.compile( + r"(?i)(api[_-]?key|access[_-]?key|auth|bearer|credential|passwd|password|private[_-]?key|secret|token)" +) SECRET_ASSIGNMENT = re.compile( r"(?i)(api[_-]?key|access[_-]?key|auth|bearer|credential|passwd|password|private[_-]?key|secret|token)" r"['\"]?\s*[:=]\s*\S" @@ -395,7 +398,7 @@ def refresh_one(root: Path, paths: dict[str, Path], relative: str, source: Path) def redact_line(line: str) -> str: line = SECRET_VALUE.sub(lambda m: f"", line) - if SECRET_ASSIGNMENT.search(line): + if SECRET_NAME.search(line) and re.search(r"[:=]", line): digest = sha256_bytes(line.encode("utf-8", "replace"))[:12] left = re.split(r"[:=]", line, maxsplit=1)[0] return f"{left}=" diff --git a/tests/test_log.py b/tests/test_log.py index 3bba9f1..7d8819c 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -88,6 +88,11 @@ def test_redact_line_scrubs_value_without_delimiter(self): out = kern_cache.redact_line("push rejected using ghp_ABCDEFGHIJKLMNOP12345 by client") self.assertNotIn("ghp_ABCDEFGHIJKLMNOP12345", out) + def test_redact_line_catches_bearer_header(self): + out = kern_cache.redact_line("Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.payload.sig") + self.assertNotIn("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", out) + self.assertIn("REDACTED", out) + class TestLogCommand(unittest.TestCase): def setUp(self): From 2bff8217508001a2270ce781b985ae2aa89da786 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 19:32:34 +0200 Subject: [PATCH 33/55] feat: structure-only L2 skips bare CALL flow lines At L2 a bare CALL line carries no call name (CALLS already lists them), yet it accounted for 23% of L2 page bytes on a 3,704-line corpus file. L2 now emits only risk-tagged CALL lines (!FAULT positions must stay); L3 dataflow is unchanged. Measured L2 on the corpus file: 4.5x -> 5.8x. Co-Authored-By: Claude Fable 5 --- benchmarks/results/python-det-v2.json | 16 ++++++++-------- docs/deterministic-compiler.md | 16 ++++++++++++++++ skills/kern/scripts/kern_compile.py | 4 ++++ tests/test_emitter.py | 17 +++++++++++++++++ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/benchmarks/results/python-det-v2.json b/benchmarks/results/python-det-v2.json index 7fefd56..1a5f510 100644 --- a/benchmarks/results/python-det-v2.json +++ b/benchmarks/results/python-det-v2.json @@ -2,22 +2,22 @@ "estimator": "chars/4", "files": [ { - "bucket": "medium(2k-10k)", + "bucket": "large(>10k)", "fidelity_missing": [], "file": "skills/kern/scripts/kern_cache.py", - "source_tokens": 9150, + "source_tokens": 10612, "tiers": { "L1": { - "ratio": 2.74, - "tokens": 3336 + "ratio": 2.92, + "tokens": 3639 }, "L2": { - "ratio": 1.95, - "tokens": 4686 + "ratio": 2.32, + "tokens": 4578 }, "L3": { - "ratio": 1.11, - "tokens": 8249 + "ratio": 1.15, + "tokens": 9255 } } } diff --git a/docs/deterministic-compiler.md b/docs/deterministic-compiler.md index f516bea..7b7997f 100644 --- a/docs/deterministic-compiler.md +++ b/docs/deterministic-compiler.md @@ -347,6 +347,22 @@ Two conclusions: Redaction verified on the same run: a hardcoded `s2_…` API key present in the source appears nowhere in the IL (6 `` markers emitted). +### Shipped emitter, measured on the same file + +The table above is the design-phase prototype. The shipped `kern-det/0.2` +emitter keeps more facts per page (`CALLS` names at L1/L2, per-symbol slice +hashes, `EFFECTS` provenance) and emits, at L2, only risk-tagged `CALL` flow +lines — a bare `CALL` carries no name, so structure-only tiers skip it: + +| Tier | Tokens | Compression | +|---|---:|---:| +| L1 | 5,451 | **8.6×** | +| L2 | 8,130 | **5.8×** | +| L3 | 25,817 | 1.8× | + +L3 is larger than the prototype because it preserves full `CALL expr -> var` +dataflow with real expressions instead of truncating them. + ### On micro-compressing keywords Replacing IL keywords with shorter codes (`CALLS` → `K`, `EFFECTS` → `E`) is diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index c79a3e8..9650abe 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -460,6 +460,10 @@ def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: lines.append(" RAISES " + raises) if level >= 2: for op in s.flow: + # L2 is structure-only: bare CALL carries no name (CALLS has them), + # so only risk-tagged calls earn a line at this tier. + if level == 2 and op.op == "CALL" and not op.risk: + continue pad = " " * (op.depth + 2) piece = op.op if level == 3 and op.detail: diff --git a/tests/test_emitter.py b/tests/test_emitter.py index 5259810..5f12663 100644 --- a/tests/test_emitter.py +++ b/tests/test_emitter.py @@ -59,6 +59,23 @@ def test_tier_l2_flow_without_expressions(self): self.assertIn("RAISE", joined) self.assertNotIn("not data", joined) + def test_tier_l2_drops_bare_call_keeps_risky_call(self): + src = ( + "import hashlib\n\n" + "def f(x):\n" + " a = helper(x)\n" + " d = hashlib.sha1(x).digest()\n" + " return a\n" + ) + mod = kern_compile.parse_python(src) + il = kern_compile.emit_il(mod, "src/x.py", "a" * 64, "none", "L2") + flow = [l.strip() for l in il.splitlines() if l.startswith(" ")] + self.assertNotIn("CALL", flow) + self.assertIn("CALL !FAULT(crypto)", flow) + self.assertIn("CALLS", il) + l3 = kern_compile.emit_il(mod, "src/x.py", "a" * 64, "none", "L3") + self.assertIn("CALL helper(x) -> a", l3) + def test_tier_l3_flow_with_expressions_and_binds(self): il = emit("L3") self.assertIn("CALL path.read_bytes() -> data", il) From 6d394fb49e11e392c056d11ecb7731df7698cdc7 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 20:42:32 +0200 Subject: [PATCH 34/55] docs: React frontend IR design spec TSX grammar routing fix + React semantic lowering (COMPONENT/PROPS/ STATE/EFFECT/EVENT/RENDER) as KERN-IL extension via kern_react.py adapter post-pass. Tiers L1-L3 mapped, ambiguity faults enumerated. Co-Authored-By: Claude Fable 5 --- .../2026-07-11-react-frontend-ir-design.md | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md diff --git a/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md b/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md new file mode 100644 index 0000000..0e6621b --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md @@ -0,0 +1,124 @@ +# React Frontend IR — Design + +**Date:** 2026-07-11 +**Scope:** TSX/JSX grammar routing fix + React semantic extraction (Stage 1 MVP). Next.js adapter, type enrichment, and other frameworks are later specs. + +## Problem + +KERN-IL/0.2 lowers TS/JS through Tree-sitter, but: + +1. `.tsx` files are routed to the plain TypeScript grammar (`kern_cache.py` passes `typescript=True`; `parse_tsjs` loads `language_typescript()`). JSX is a syntax error in that grammar, so JSX-bearing `.tsx` files degrade to a parse-error IR. `tree_sitter_typescript.language_tsx()` exists and is unused. +2. `.jsx` parses (the JavaScript grammar includes JSX) but the extractor has no JSX semantics: components render as `FN` symbols whose return statements are truncated JSX text. Hooks, state, effects, events, and render structure — the behavioral core of a React file — are invisible. + +React source is a verbose encoding of compact behavior. A deterministic semantic lowering recovers that behavior at a fraction of the tokens. + +## Decision summary + +- **No new parser.** Tree-sitter JS and TSX grammars provide syntax; KERN supplies React meaning. Writing or committing a custom React grammar is explicitly out of scope — React is semantics on top of JS/TSX syntax, not a new syntax. +- **Extend KERN-IL, not a new format.** `COMPONENT` becomes a new symbol kind inside the existing `ModuleIR`/`Symbol`/`emit_il` pipeline. Spans, slice hashes, tiers, faults, verify, cache, and redaction are inherited verbatim. The runtime model learns one format. +- **Adapter seam, not query packs.** React lowering lives in a new module (`skills/kern/scripts/kern_react.py`) as a post-pass over the same Tree-sitter tree, in the codebase's existing manual-visitor style. Future frameworks (Next.js, Vue, Svelte) become sibling adapters emitting the same vocabulary. Declarative `.scm` query packs are deferred until a second framework proves the need. +- **Tiers stay the compression lever.** Component detail maps onto existing L1/L2/L3; no new knobs. + +## Architecture + +```text +TSX/JSX source + ↓ +parse_tsjs(text, dialect) # dialect: "js" | "ts" | "tsx" + ↓ tree-sitter CST + base symbols (FN/const/class/import) +kern_react.py post-pass # same tree, upgrades qualifying FN → COMPONENT + ↓ +ModuleIR → emit_il (existing) # tier rendering, faults, hashes +``` + +### Grammar routing + +| suffix | grammar | +| --- | --- | +| `.js` `.jsx` `.mjs` `.cjs` | `tree_sitter_javascript` (JSX built in) | +| `.ts` | `tree_sitter_typescript.language_typescript()` | +| `.tsx` | `tree_sitter_typescript.language_tsx()` | + +`parse_tsjs(text, typescript: bool)` becomes `parse_tsjs(text, dialect: str)`. Both call sites (`kern_cache.py` compile and verify paths) updated. `generator=` header line reports `lang=tsx frontend=tree-sitter+react` when the adapter fires. + +### Component detection (deterministic) + +A symbol is upgraded to `kind="component"` when: + +- it is a function declaration, function expression, or arrow function, AND +- its name matches `^[A-Z]`, AND +- a `return` (or arrow expression body) contains `jsx_element`, `jsx_self_closing_element`, or `jsx_fragment`. + +Also detected: `memo(Fn)`, `forwardRef(Fn)` wrappers — the inner function is lowered, the wrapper noted. Capitalized functions with no JSX stay plain `FN`. Lowercase functions returning JSX stay `FN` (not components by React convention). + +## Extraction vocabulary + +```text +COMPONENT UserCard L3-18 #a3f9c2d1 + PROPS user, onClose?=noop + STATE open=false + STATE [state, dispatch]=useReducer(reducer, init) + CTX theme=useContext(ThemeContext) + REF inputRef + HOOK data=useUserData(id) + EFFECT deps=[user.id] + EVENT Card.onClick -> set open=true + RENDER + Card + Avatar src=user.avatar + span {user.name} + IF open > UserDetails user=user + FOR item in items > Row key=item.id +``` + +Rules: + +- **PROPS** — first parameter. Destructured pattern lists names with defaults (`onClose?=noop` when a default exists; `?` when the TS type marks it optional and that is syntactically visible). Non-destructured param renders as its name (`props`). +- **STATE** — `const [x, setX] = useState(init)` → `STATE x=init`; setter name recorded internally for EVENT lowering. `useReducer` renders the pair and arguments. +- **CTX / REF / HOOK** — `useContext`, `useRef`, and custom `use[A-Z]\w*` calls respectively. Custom hooks are opaque: call text only, no cross-file resolution. +- **EFFECT** — `useEffect` / `useLayoutEffect`. Dependency array rendered verbatim (`deps=[user.id]`, `deps=[]`); a missing array renders `deps=EVERY-RENDER`. Effect body: L2 shows the head only; L3 summarizes via existing `flow()`. +- **EVENT** — JSX attribute matching `on[A-Z]\w+={expr}`. If the handler body is a single known-setter call, lower to `set =`; otherwise render the callee name or `flow()`-style summary at L3. +- **RENDER** — JSX tree, indentation = nesting: + - `{cond && }` → `IF cond > X`; ternary → `IF cond > X ELSE > Y` + - `.map(` callback returning JSX → `FOR param in receiver > X` + - Host elements (lowercase) are structure; text/expression children render as `{expr}` capped by existing `ntext` (secret redaction inherited) + - Capitalized JSX names are component dependencies; names that match imports cross-link naturally through the existing import lines +- Non-hook, non-render statements in the component body flow through the existing L3 flow-op rendering unchanged. +- All line math is `\n`-only, matching the repo rule (never `str.splitlines()`). + +## Tier mapping + +| Tier | Component detail | +| --- | --- | +| L1 | `COMPONENT name (props) span #hash` — one line, like current FN heads | +| L2 | + STATE/CTX/REF/HOOK/EFFECT/EVENT heads; RENDER collapsed to components-only tree (host elements and attributes dropped; IF/FOR structure kept) | +| L3 | Full render tree with attributes, effect bodies and handler bodies via `flow()` | + +## Faulting + +Ambiguity never disappears silently. Reuses the existing `!FAULT(...)` inline channel and FAULTS footer: + +| Construct | Marker | +| --- | --- | +| Hook called via alias or namespace (`R.useState`, renamed import) | `!FAULT(aliased-hook)` | +| Dynamic component (`` where Tag is a lowercase variable or member expression) | `!FAULT(dynamic-component)` | +| Spread props as the sole prop source (`{...rest}`) | rendered `...rest` + `!FAULT(spread-props)` | +| Render prop / children-as-function | `!FAULT(render-prop)`, body summarized as flow | +| Hook call inside conditional | `!FAULT(conditional-hook)` | +| Render tree exceeding op budget | explicit `…+N` + `!FAULT(render-truncated)` | + +Frontend IR remains a reasoning representation, not the write authority: edits still require faulting exact current source and verifying the slice hash, per the existing contract. + +## Testing + +- **`tests/test_react.py`** — golden fixtures per construct: useState, useReducer, useContext, useRef, custom hook, effect with/without deps, events (setter and non-setter), IF/FOR render lowering, fragments, memo/forwardRef, spread props, dynamic component, aliased hook, conditional hook, `.jsx` and `.tsx` dialects, TSX generics ambiguity (`` arrow). +- **Regression** — existing `test_tsjs.py` fixtures (no React constructs) must emit byte-identical IR; the adapter must be a strict no-op on non-component code. +- **Corpus** — run compile over KERN's own `app/*.jsx` and a cloned real-world Next.js repo: zero crashes, all ambiguous constructs surface as faults (grep the FAULTS footer), compression ratios recorded per tier for the record. No hard ratio gate — compression is a large-file property and the `min_ir_tokens=600` source-cheaper stub already handles small files. + +## Non-goals (this spec) + +- Next.js concepts: routes, layouts, `use client`/`use server` boundaries, server actions, metadata (Stage 2 spec) +- Type-aware enrichment via the TypeScript compiler API or LSP (Stage 3 spec) +- Vue / Svelte / Angular / Astro adapters +- Declarative `.scm` query-pack architecture +- A custom Tree-sitter React grammar (never needed) From feb4a37a818c5c30a61883a9bf825fa35b346868 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:02:00 +0200 Subject: [PATCH 35/55] fix: route .tsx to tree-sitter TSX grammar via dialect param Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 14 ++++++-- skills/kern/scripts/kern_compile.py | 8 +++-- tests/test_react.py | 52 +++++++++++++++++++++++++++++ tests/test_tsjs.py | 4 +-- 4 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 tests/test_react.py diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index e07e4d0..74e0e4f 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -22,6 +22,16 @@ CODEC_VERSION = "kern-il/0.2" BASELINE_GENERATOR = "kern-det/0.2" TSJS_SUFFIXES = {".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx"} + + +def tsjs_dialect(suffix: str) -> str: + if suffix == ".tsx": + return "tsx" + if suffix == ".ts": + return "ts" + return "js" + + CACHE_DIRNAME = ".kern" DEFAULT_CONFIG: dict[str, Any] = { "schema": SCHEMA, @@ -462,7 +472,7 @@ def baseline_for(root: Path, source: Path, relative: str, digest: str, if suffix == ".py": module = kern_compile.parse_python(text) elif suffix in TSJS_SUFFIXES and kern_compile.tsjs_available(): - module = kern_compile.parse_tsjs(text, typescript=suffix in {".ts", ".tsx"}) + module = kern_compile.parse_tsjs(text, dialect=tsjs_dialect(suffix)) if module is not None: if module.parse_error: note = f"parse failed: {module.parse_error}" @@ -828,7 +838,7 @@ def verify_symbol(root: Path, paths: dict[str, Path], relative: str, source: Pat if suffix == ".py": module = kern_compile.parse_python(text) elif suffix in TSJS_SUFFIXES and kern_compile.tsjs_available(): - module = kern_compile.parse_tsjs(text, typescript=suffix in {".ts", ".tsx"}) + module = kern_compile.parse_tsjs(text, dialect=tsjs_dialect(suffix)) else: raise ValueError(f"verify does not support {suffix or 'this file type'}; use fault with --expect-sha") if module.parse_error: diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 9650abe..ca917b5 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -500,9 +500,13 @@ def tsjs_available() -> bool: _TS_FUNC_NODES = {"function_declaration", "generator_function_declaration", "method_definition"} -def parse_tsjs(text: str, typescript: bool = False) -> ModuleIR: +def parse_tsjs(text: str, dialect: str = "js") -> ModuleIR: from tree_sitter import Language, Parser - if typescript: + if dialect == "tsx": + import tree_sitter_typescript as ts_lang + language = Language(ts_lang.language_tsx()) + lang_name = "tsx" + elif dialect == "ts": import tree_sitter_typescript as ts_lang language = Language(ts_lang.language_typescript()) lang_name = "typescript" diff --git a/tests/test_react.py b/tests/test_react.py new file mode 100644 index 0000000..4e8537f --- /dev/null +++ b/tests/test_react.py @@ -0,0 +1,52 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + +TSX_SAMPLE = ''' +import { useState, useEffect } from "react"; +import { Card, Avatar, UserDetails } from "./ui"; + +export function UserCard({ user, onClose = noop }) { + const [open, setOpen] = useState(false); + + useEffect(() => { + analytics.track("view_user", user.id); + }, [user.id]); + + return ( + setOpen(true)}> + + {user.name} + {open && } + + ); +} + +export function formatName(name: string): string { + return name.trim(); +} +''' + + +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestDialectRouting(unittest.TestCase): + def test_tsx_dialect_parses_jsx_clean(self): + mod = kern_compile.parse_tsjs(TSX_SAMPLE, dialect="tsx") + self.assertEqual(mod.parse_error, "") + self.assertEqual(mod.lang, "tsx") + + def test_plain_ts_dialect_chokes_on_jsx(self): + # Documents the routed-around limitation: TS grammar has no JSX. + mod = kern_compile.parse_tsjs(TSX_SAMPLE, dialect="ts") + self.assertNotEqual(mod.parse_error, "") + + def test_js_dialect_still_default(self): + mod = kern_compile.parse_tsjs("function f() { return 1; }\n") + self.assertEqual(mod.lang, "javascript") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tsjs.py b/tests/test_tsjs.py index c6fe699..908a1f4 100644 --- a/tests/test_tsjs.py +++ b/tests/test_tsjs.py @@ -34,7 +34,7 @@ def test_available_returns_bool(self): @unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") class TestTsFrontend(unittest.TestCase): def setUp(self): - self.mod = kern_compile.parse_tsjs(TS_SAMPLE, typescript=True) + self.mod = kern_compile.parse_tsjs(TS_SAMPLE, dialect="ts") def sym(self, name): return next(s for s in self.mod.symbols if s.name == name) @@ -69,7 +69,7 @@ def test_emit_works(self): def test_broken_source_sets_parse_error(self): broken = "export function f( {\n return 1;\n}\n" - mod = kern_compile.parse_tsjs(broken, typescript=True) + mod = kern_compile.parse_tsjs(broken, dialect="ts") self.assertNotEqual(mod.parse_error, "") def test_clean_source_has_no_parse_error(self): From 9f28e0f694d2883e4c7078310df52684e6235d5d Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:07:05 +0200 Subject: [PATCH 36/55] feat: React component detection adapter with props extraction Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 34 +++++++- skills/kern/scripts/kern_react.py | 118 ++++++++++++++++++++++++++++ tests/test_react.py | 53 +++++++++++++ 3 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 skills/kern/scripts/kern_react.py diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index ca917b5..cffabeb 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -140,6 +140,7 @@ class Symbol: effects: dict = field(default_factory=dict) # effect -> list of via names ([] = direct) raises_all: dict = field(default_factory=dict) # exception -> list of via names unknown_calls: int = 0 + react: dict = field(default_factory=dict) # React adapter payload (kern_react) @dataclass @@ -601,6 +602,8 @@ def add(node, op, detail="", binds=""): ops.extend(flow(c, depth, budget - len(ops))) return ops[:budget] + fn_nodes: list = [] + def function_symbol(node, qualified): calls, raises = [], [] collect_calls(node, calls) @@ -610,13 +613,15 @@ def function_symbol(node, qualified): body = field_node(node, "body") a, b = span(node) is_async = any(ch.type == "async" for ch in node.children) - return Symbol( + sym = Symbol( kind="function", name=qualified, span=(a, b), signature=ntext(params, 200).strip("()") if params is not None else "", returns=ntext(rtype, 60).lstrip(": ") if rtype is not None else "", slice8=slice_sha8(text, a, b), calls=calls, raises=raises, flow=flow(body) if body is not None else [], is_async=is_async, ) + fn_nodes.append((sym, node)) + return sym symbols: list[Symbol] = [] @@ -632,6 +637,25 @@ def top(n, class_prefix=""): value = field_node(d, "value") if value is not None and value.type in ("arrow_function", "function_expression"): symbols.append(function_symbol(value, ntext(name, 60))) + elif value is not None and value.type == "call_expression": + callee = field_node(value, "function") + wrapper = ntext(callee, 60) if callee is not None else "" + inner = None + if wrapper in ("memo", "forwardRef", "React.memo", "React.forwardRef"): + args = field_node(value, "arguments") + if args is not None: + inner = next((ch for ch in args.named_children + if ch.type in ("arrow_function", "function_expression")), None) + if inner is not None: + sym = function_symbol(inner, ntext(name, 60)) + sym.decorators = [wrapper.split(".")[-1]] + symbols.append(sym) + elif name is not None: + rendered = ntext(value, 100) + hint = bool(SECRET_NAME.search(ntext(name, 60))) + if hint: + rendered = sanitize_string(rendered, secret_hint=True) + symbols.append(Symbol(kind="const", name=ntext(name, 60), detail="=" + rendered, span=span(c))) elif name is not None: if value is not None: rendered = ntext(value, 100) @@ -658,6 +682,12 @@ def top(n, class_prefix=""): top(c, class_prefix) top(tree.root_node) + frontend = "tree-sitter" + if dialect in ("js", "tsx"): + import kern_react + if kern_react.lower_components(fn_nodes, ntext=ntext, flow_fn=flow): + frontend = "tree-sitter+react" + lines = text.splitlines() omit = { "docstrings": 0, @@ -681,7 +711,7 @@ def first_error_line(n): line = 1 parse_error = f"tree-sitter reported syntax errors (first at L{line})" - return ModuleIR(lang_name, "tree-sitter", symbols, omit, parse_error=parse_error) + return ModuleIR(lang_name, frontend, symbols, omit, parse_error=parse_error) def emit_il(module: ModuleIR, source_rel: str, source_sha256: str, diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py new file mode 100644 index 0000000..b093f45 --- /dev/null +++ b/skills/kern/scripts/kern_react.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""React semantic adapter: upgrades tree-sitter function symbols to KERN-IL components.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +JSX_TYPES = {"jsx_element", "jsx_self_closing_element", "jsx_fragment"} +HOOK_RE = re.compile(r"^use[A-Z]\w*$") +COMPONENT_NAME_RE = re.compile(r"^[A-Z]") +FN_TYPES = {"arrow_function", "function_expression", "function_declaration", + "generator_function_declaration", "method_definition"} + + +@dataclass +class HookUse: + kind: str # STATE | CTX | REF | HOOK | EFFECT + detail: str + line: int + risk: str = "" + flow: list = field(default_factory=list) # EFFECT body ops for L3 + + +@dataclass +class EventUse: + target: str # "Card.onClick" + action: str # "set open=true" | callee name + line: int + + +@dataclass +class RenderNode: + tag: str # "Card" | "IF open" | "FOR item in items" | "ELSE" | "{expr}" | text + attrs: str = "" + line: int = 0 + is_component: bool = False + is_structure: bool = False # IF/ELSE/FOR — survives L2 collapse + risk: str = "" + children: list = field(default_factory=list) + + +def _unwrap_parens(n): + while n is not None and n.type == "parenthesized_expression": + inner = n.named_children + n = inner[0] if inner else None + return n + + +def _returns_jsx(fn_node) -> bool: + body = fn_node.child_by_field_name("body") + if body is None: + return False + if body.type != "statement_block": + u = _unwrap_parens(body) + return u is not None and u.type in JSX_TYPES + stack = list(body.named_children) + while stack: + n = stack.pop() + if n.type in FN_TYPES: + continue # returns inside nested functions don't count + if n.type == "return_statement": + for ch in n.named_children: + u = _unwrap_parens(ch) + if u is not None and u.type in JSX_TYPES: + return True + stack.extend(n.named_children) + return False + + +def _extract_props(fn_node, ntext) -> list: + params = fn_node.child_by_field_name("parameters") + if params is None: + single = fn_node.child_by_field_name("parameter") + first = single + else: + named = params.named_children + first = named[0] if named else None + if first is None: + return [] + if first.type in ("required_parameter", "optional_parameter"): + pattern = first.child_by_field_name("pattern") + if pattern is not None: + first = pattern + if first.type == "object_pattern": + out = [] + for p in first.named_children: + if p.type == "shorthand_property_identifier_pattern": + out.append(ntext(p, 60)) + elif p.type == "object_assignment_pattern": + left = p.child_by_field_name("left") + right = p.child_by_field_name("right") + out.append(f"{ntext(left, 60)}={ntext(right, 60)}") + elif p.type == "pair_pattern": + out.append(ntext(p, 60)) + elif p.type == "rest_pattern": + out.append(ntext(p, 60)) + return out + return [ntext(first, 60)] + + +def lower_components(fn_nodes, ntext, flow_fn) -> bool: + upgraded = False + for sym, node in fn_nodes: + short = sym.name.split(".")[-1] + if not COMPONENT_NAME_RE.match(short) or not _returns_jsx(node): + continue + sym.kind = "component" + sym.react = { + "wrapper": sym.decorators[0] if sym.decorators else "", + "props": _extract_props(node, ntext), + "hooks": [], + "events": [], + "render": [], + "faults": [], + } + upgraded = True + return upgraded diff --git a/tests/test_react.py b/tests/test_react.py index 4e8537f..c71f216 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -48,5 +48,58 @@ def test_js_dialect_still_default(self): self.assertEqual(mod.lang, "javascript") +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestComponentDetection(unittest.TestCase): + def parse(self, src, dialect="tsx"): + return kern_compile.parse_tsjs(src, dialect=dialect) + + def sym(self, mod, name): + return next(s for s in mod.symbols if s.name == name) + + def test_capitalized_jsx_function_is_component(self): + mod = self.parse(TSX_SAMPLE) + s = self.sym(mod, "UserCard") + self.assertEqual(s.kind, "component") + self.assertEqual(mod.frontend, "tree-sitter+react") + + def test_props_extracted(self): + mod = self.parse(TSX_SAMPLE) + self.assertEqual(self.sym(mod, "UserCard").react["props"], + ["user", "onClose=noop"]) + + def test_capitalized_without_jsx_stays_function(self): + mod = self.parse("export function Parse(x) { return x + 1; }\n") + self.assertEqual(self.sym(mod, "Parse").kind, "function") + + def test_lowercase_with_jsx_stays_function(self): + mod = self.parse("function helper() { return
; }\n") + self.assertEqual(self.sym(mod, "helper").kind, "function") + + def test_arrow_component_detected(self): + mod = self.parse("export const Badge = ({ label }) => {label};\n") + self.assertEqual(self.sym(mod, "Badge").kind, "component") + + def test_memo_wrapper_unwrapped(self): + mod = self.parse("import { memo } from 'react';\n" + "const Row = memo(({ id }) =>
  • {id}
    • );\n") + s = self.sym(mod, "Row") + self.assertEqual(s.kind, "component") + self.assertEqual(s.react["wrapper"], "memo") + + def test_non_react_file_untouched(self): + mod = self.parse("export function parse(raw) { return Number(raw); }\n", + dialect="js") + self.assertEqual(mod.frontend, "tree-sitter") + self.assertEqual(self.sym(mod, "parse").kind, "function") + + def test_nested_return_jsx_in_inner_fn_not_component(self): + src = ("function Outer() {\n" + " const inner = () =>
    ;\n" + " return 42;\n" + "}\n") + mod = self.parse(src) + self.assertEqual(self.sym(mod, "Outer").kind, "function") + + if __name__ == "__main__": unittest.main() From 2239f395a02ecdff3f8ca1a83ede0f2320f89ca0 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:13:54 +0200 Subject: [PATCH 37/55] fix: memo/forwardRef unwrap only considers first argument Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 7 ++++--- tests/test_react.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index cffabeb..2fef339 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -643,9 +643,10 @@ def top(n, class_prefix=""): inner = None if wrapper in ("memo", "forwardRef", "React.memo", "React.forwardRef"): args = field_node(value, "arguments") - if args is not None: - inner = next((ch for ch in args.named_children - if ch.type in ("arrow_function", "function_expression")), None) + if args is not None and args.named_children: + first = args.named_children[0] + if first.type in ("arrow_function", "function_expression"): + inner = first if inner is not None: sym = function_symbol(inner, ntext(name, 60)) sym.decorators = [wrapper.split(".")[-1]] diff --git a/tests/test_react.py b/tests/test_react.py index c71f216..7df5fb8 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -86,6 +86,22 @@ def test_memo_wrapper_unwrapped(self): self.assertEqual(s.kind, "component") self.assertEqual(s.react["wrapper"], "memo") + def test_memo_with_comparator_identifier_not_hijacked(self): + src = ("const Row = ({ id }) =>
  • {id}
    • ;\n" + "const MemoRow = memo(Row, (a, b) => a.id === b.id);\n") + mod = self.parse(src) + memo_syms = [s for s in mod.symbols if s.name == "MemoRow"] + self.assertEqual(len(memo_syms), 1) + self.assertEqual(memo_syms[0].kind, "const") # falls to const fallback + + def test_memo_inline_with_comparator_uses_first_arg(self): + src = "const Row = memo(({ id }) =>
  • {id}
    • , (a, b) => a.id === b.id);\n" + mod = self.parse(src) + s = self.sym(mod, "Row") + self.assertEqual(s.kind, "component") + self.assertEqual(s.react["wrapper"], "memo") + self.assertEqual(s.react["props"], ["id"]) + def test_non_react_file_untouched(self): mod = self.parse("export function parse(raw) { return Number(raw); }\n", dialect="js") From 3778c4be28fb1acd65ba78b357f7df7ebad8a97e Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:17:18 +0200 Subject: [PATCH 38/55] feat: React hook extraction with aliased and conditional-hook faults Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 98 +++++++++++++++++++++++++++++++ tests/test_react.py | 56 ++++++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index b093f45..ab3a67a 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -99,6 +99,99 @@ def _extract_props(fn_node, ntext) -> list: return [ntext(first, 60)] +BUILTIN_STATE = {"useState", "useReducer"} +EFFECT_HOOKS = {"useEffect", "useLayoutEffect"} +COND_TYPES = {"if_statement", "for_statement", "for_in_statement", + "while_statement", "do_statement", "switch_statement", + "ternary_expression", "binary_expression"} + + +def _call_parts(value, ntext): + """(callee_text, args_nodes) for a call_expression, else (None, []).""" + if value is None or value.type != "call_expression": + return None, [] + callee = value.child_by_field_name("function") + args = value.child_by_field_name("arguments") + return (ntext(callee, 80) if callee is not None else None, + list(args.named_children) if args is not None else []) + + +def _extract_hooks(body, react, ntext, flow_fn): + hooks, setters, faults = react["hooks"], {}, react["faults"] + for stmt in body.named_children: + line = stmt.start_point[0] + 1 + if stmt.type in ("lexical_declaration", "variable_declaration"): + for d in stmt.named_children: + if d.type != "variable_declarator": + continue + name_node = d.child_by_field_name("name") + value = d.child_by_field_name("value") + callee, args = _call_parts(value, ntext) + if callee is None: + continue + tail = callee.split(".")[-1] + if not HOOK_RE.match(tail): + continue + risk = "aliased-hook" if "." in callee else "" + if risk: + faults.append((risk, line)) + name_txt = ntext(name_node, 60) + if tail == "useState" and name_node.type == "array_pattern": + elems = [ntext(e, 40) for e in name_node.named_children] + state = elems[0] if elems else "?" + init = ntext(args[0], 80) if args else "undefined" + if len(elems) > 1: + setters[elems[1]] = state + hooks.append(HookUse("STATE", f"{state}={init}", line, risk)) + elif tail == "useReducer": + hooks.append(HookUse("STATE", f"{name_txt}={ntext(value, 120)}", line, risk)) + elif tail == "useContext": + hooks.append(HookUse("CTX", f"{name_txt}={ntext(value, 80)}", line, risk)) + elif tail == "useRef": + hooks.append(HookUse("REF", name_txt, line, risk)) + else: + hooks.append(HookUse("HOOK", f"{name_txt}={ntext(value, 120)}", line, risk)) + elif stmt.type == "expression_statement" and stmt.named_children: + value = stmt.named_children[0] + callee, args = _call_parts(value, ntext) + if callee is None: + continue + tail = callee.split(".")[-1] + if tail in EFFECT_HOOKS: + risk = "aliased-hook" if "." in callee else "" + if risk: + faults.append((risk, line)) + deps = f"deps={ntext(args[1], 100)}" if len(args) >= 2 else "deps=EVERY-RENDER" + ops = [] + if args and args[0].type in ("arrow_function", "function_expression"): + cb_body = args[0].child_by_field_name("body") + if cb_body is not None and cb_body.type == "statement_block": + ops = flow_fn(cb_body) + hooks.append(HookUse("EFFECT", deps, line, risk, flow=ops)) + elif HOOK_RE.match(tail): + risk = "aliased-hook" if "." in callee else "" + if risk: + faults.append((risk, line)) + hooks.append(HookUse("HOOK", ntext(value, 120), line, risk)) + else: + _fault_conditional_hooks(stmt, faults, ntext) + react["setters"] = setters + + +def _fault_conditional_hooks(node, faults, ntext): + stack = [node] + while stack: + n = stack.pop() + if n.type in FN_TYPES or n.type in JSX_TYPES: + continue + if n.type == "call_expression": + callee = n.child_by_field_name("function") + txt = ntext(callee, 80) if callee is not None else "" + if HOOK_RE.match(txt.split(".")[-1] or ""): + faults.append(("conditional-hook", n.start_point[0] + 1)) + stack.extend(n.named_children) + + def lower_components(fn_nodes, ntext, flow_fn) -> bool: upgraded = False for sym, node in fn_nodes: @@ -114,5 +207,10 @@ def lower_components(fn_nodes, ntext, flow_fn) -> bool: "render": [], "faults": [], } + body = node.child_by_field_name("body") + if body is not None and body.type == "statement_block": + _extract_hooks(body, sym.react, ntext, flow_fn) + else: + sym.react["setters"] = {} upgraded = True return upgraded diff --git a/tests/test_react.py b/tests/test_react.py index 7df5fb8..0c2ef22 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -117,5 +117,61 @@ def test_nested_return_jsx_in_inner_fn_not_component(self): self.assertEqual(self.sym(mod, "Outer").kind, "function") +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestHooks(unittest.TestCase): + def component(self, src): + mod = kern_compile.parse_tsjs(src, dialect="tsx") + return next(s for s in mod.symbols if s.kind == "component") + + def hooks(self, src): + return self.component(src).react["hooks"] + + def test_usestate_setter_and_init(self): + s = self.component(TSX_SAMPLE) + h = [x for x in s.react["hooks"] if x.kind == "STATE"] + self.assertEqual(h[0].detail, "open=false") + self.assertEqual(s.react["setters"], {"setOpen": "open"}) + + def test_effect_deps_verbatim(self): + h = [x for x in self.hooks(TSX_SAMPLE) if x.kind == "EFFECT"] + self.assertEqual(h[0].detail, "deps=[user.id]") + self.assertTrue(h[0].flow) # body captured for L3 + + def test_effect_missing_deps(self): + src = ("function T() {\n useEffect(() => { tick(); });\n" + " return
    ;\n}\n") + h = [x for x in self.hooks(src) if x.kind == "EFFECT"] + self.assertEqual(h[0].detail, "deps=EVERY-RENDER") + + def test_reducer_context_ref_custom(self): + src = ("function T() {\n" + " const [state, dispatch] = useReducer(reducer, init);\n" + " const theme = useContext(ThemeContext);\n" + " const inputRef = useRef(null);\n" + " const data = useUserData(id);\n" + " return
    ;\n}\n") + kinds = [(h.kind, h.detail) for h in self.hooks(src)] + self.assertEqual(kinds, [ + ("STATE", "[state, dispatch]=useReducer(reducer, init)"), + ("CTX", "theme=useContext(ThemeContext)"), + ("REF", "inputRef"), + ("HOOK", "data=useUserData(id)"), + ]) + + def test_aliased_hook_faulted(self): + src = ("import * as R from 'react';\n" + "function T() {\n const [a, setA] = R.useState(0);\n" + " return
    ;\n}\n") + h = self.hooks(src)[0] + self.assertEqual(h.risk, "aliased-hook") + + def test_conditional_hook_faulted(self): + src = ("function T({ on }) {\n" + " if (on) { useEffect(() => {}); }\n" + " return
    ;\n}\n") + faults = self.component(src).react["faults"] + self.assertIn("conditional-hook", [f[0] for f in faults]) + + if __name__ == "__main__": unittest.main() From cff9e21bba6ba7ba15db7137a2ce7562ae9b1a03 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:23:41 +0200 Subject: [PATCH 39/55] fix: fault hooks guarded by && and ternary at component top level Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 8 ++++---- tests/test_react.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index ab3a67a..78788d1 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -99,11 +99,8 @@ def _extract_props(fn_node, ntext) -> list: return [ntext(first, 60)] -BUILTIN_STATE = {"useState", "useReducer"} EFFECT_HOOKS = {"useEffect", "useLayoutEffect"} -COND_TYPES = {"if_statement", "for_statement", "for_in_statement", - "while_statement", "do_statement", "switch_statement", - "ternary_expression", "binary_expression"} +COND_EXPR_TYPES = {"binary_expression", "ternary_expression"} def _call_parts(value, ntext): @@ -153,6 +150,9 @@ def _extract_hooks(body, react, ntext, flow_fn): hooks.append(HookUse("HOOK", f"{name_txt}={ntext(value, 120)}", line, risk)) elif stmt.type == "expression_statement" and stmt.named_children: value = stmt.named_children[0] + if value.type in COND_EXPR_TYPES: + _fault_conditional_hooks(value, faults, ntext) + continue callee, args = _call_parts(value, ntext) if callee is None: continue diff --git a/tests/test_react.py b/tests/test_react.py index 0c2ef22..73da8e1 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -172,6 +172,27 @@ def test_conditional_hook_faulted(self): faults = self.component(src).react["faults"] self.assertIn("conditional-hook", [f[0] for f in faults]) + def test_logical_and_guarded_hook_faulted(self): + src = ("function T({ on }) {\n" + " on && useEffect(() => {});\n" + " return
    ;\n}\n") + faults = self.component(src).react["faults"] + self.assertIn("conditional-hook", [f[0] for f in faults]) + + def test_ternary_guarded_hook_faulted(self): + src = ("function T({ on }) {\n" + " on ? useState(0) : null;\n" + " return
    ;\n}\n") + faults = self.component(src).react["faults"] + self.assertIn("conditional-hook", [f[0] for f in faults]) + + def test_plain_toplevel_call_not_faulted(self): + src = ("function T() {\n" + " analytics.track(useContextValue());\n" + " return
    ;\n}\n") + faults = self.component(src).react["faults"] + self.assertEqual(faults, []) + if __name__ == "__main__": unittest.main() From f98e490c04f3f35147e1781d9907216111476352 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:28:34 +0200 Subject: [PATCH 40/55] feat: JSX render tree lowering with IF/FOR and ambiguity faults Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 176 ++++++++++++++++++++++++++++++ tests/test_react.py | 71 ++++++++++++ 2 files changed, 247 insertions(+) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index 78788d1..475c975 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -192,6 +192,179 @@ def _fault_conditional_hooks(node, faults, ntext): stack.extend(n.named_children) +RENDER_BUDGET = 200 + + +def _jsx_name(el, ntext): + """(name_text, is_component, risk) for a jsx element node.""" + opening = el + if el.type == "jsx_element": + opening = el.named_children[0] if el.named_children else el + name = opening.child_by_field_name("name") + if name is None: + return ("<>", False, "") + txt = ntext(name, 60) + if name.type in ("member_expression", "jsx_namespace_name"): + return (txt, True, "dynamic-component") + if COMPONENT_NAME_RE.match(txt): + return (txt, True, "") + return (txt, False, "") + + +def _jsx_attrs(el, ntext): + """(attrs_text, spread_only, event_attrs) from an element's opening tag. + event_attrs: list of (attr_name, expr_node, element_name_text).""" + opening = el + if el.type == "jsx_element": + opening = el.named_children[0] if el.named_children else el + parts, named, spread, events = [], 0, 0, [] + for a in opening.named_children: + if a.type == "jsx_attribute": + aname_node = a.named_children[0] if a.named_children else None + aname = ntext(aname_node, 40) if aname_node is not None else "" + value = a.named_children[1] if len(a.named_children) > 1 else None + if re.match(r"^on[A-Z]", aname) and value is not None: + events.append((aname, value)) + continue + named += 1 + parts.append(ntext(a, 60)) + elif a.type == "jsx_expression": # {...spread} + spread += 1 + parts.append(ntext(a, 40).strip("{}")) + return " ".join(parts), (spread > 0 and named == 0), events + + +def _lower_jsx(node, ntext, counter, events_out, element_name=""): + """Lower one JSX node (or jsx child) to list[RenderNode].""" + if counter["n"] >= RENDER_BUDGET: + return [] + n = _unwrap_parens(node) + if n is None: + return [] + out = [] + + def make(tag, **kw): + counter["n"] += 1 + return RenderNode(tag=tag, line=n.start_point[0] + 1, **kw) + + if n.type in ("jsx_element", "jsx_self_closing_element"): + name, is_comp, risk = _jsx_name(n, ntext) + attrs, spread_only, ev = _jsx_attrs(n, ntext) + if spread_only and not risk: + risk = "spread-props" + rn = make(name, attrs=attrs, is_component=is_comp, risk=risk) + for aname, value in ev: + events_out.append((name, aname, value)) + if n.type == "jsx_element": + for ch in n.named_children[1:-1]: + rn.children.extend(_lower_jsx(ch, ntext, counter, events_out, name)) + out.append(rn) + elif n.type == "jsx_fragment": + for ch in n.named_children: + out.extend(_lower_jsx(ch, ntext, counter, events_out, element_name)) + elif n.type == "jsx_text": + txt = ntext(n, 60) + if txt: + out.append(make(txt)) + elif n.type == "jsx_expression": + inner = n.named_children[0] if n.named_children else None + out.extend(_lower_expr(inner, ntext, counter, events_out, element_name)) + elif n.type in ("binary_expression", "ternary_expression", "call_expression", + "arrow_function", "function_expression"): + out.extend(_lower_expr(n, ntext, counter, events_out, element_name)) + return out + + +def _lower_expr(n, ntext, counter, events_out, element_name): + n = _unwrap_parens(n) + if n is None or counter["n"] >= RENDER_BUDGET: + return [] + + def make(tag, **kw): + counter["n"] += 1 + return RenderNode(tag=tag, line=n.start_point[0] + 1, **kw) + + if n.type in JSX_TYPES: + return _lower_jsx(n, ntext, counter, events_out, element_name) + if n.type == "binary_expression": + op = n.child_by_field_name("operator") + right = _unwrap_parens(n.child_by_field_name("right")) + if op is not None and op.type == "&&" and right is not None and right.type in JSX_TYPES: + left = n.child_by_field_name("left") + rn = make(f"IF {ntext(left, 80)}", is_structure=True) + rn.children = _lower_jsx(right, ntext, counter, events_out, element_name) + return [rn] + elif n.type == "ternary_expression": + cond = n.child_by_field_name("condition") + cons = _unwrap_parens(n.child_by_field_name("consequence")) + alt = _unwrap_parens(n.child_by_field_name("alternative")) + branches = [] + if cons is not None and cons.type in JSX_TYPES: + rn = make(f"IF {ntext(cond, 80)}", is_structure=True) + rn.children = _lower_jsx(cons, ntext, counter, events_out, element_name) + branches.append(rn) + if alt is not None and alt.type in JSX_TYPES: + el = make("ELSE", is_structure=True) + el.children = _lower_jsx(alt, ntext, counter, events_out, element_name) + branches.append(el) + if branches: + return branches + elif n.type == "call_expression": + callee = n.child_by_field_name("function") + if callee is not None and callee.type == "member_expression": + prop = callee.child_by_field_name("property") + if prop is not None and ntext(prop, 20) == "map": + receiver = callee.child_by_field_name("object") + args = n.child_by_field_name("arguments") + cb = next((a for a in args.named_children + if a.type in ("arrow_function", "function_expression")), None) if args is not None else None + if cb is not None: + params = cb.child_by_field_name("parameters") + single = cb.child_by_field_name("parameter") + if single is not None: + param_txt = ntext(single, 40) + elif params is not None and params.named_children: + param_txt = ntext(params.named_children[0], 40) + else: + param_txt = "_" + rn = make(f"FOR {param_txt} in {ntext(receiver, 60)}", is_structure=True) + body = cb.child_by_field_name("body") + rn.children = _lower_jsx(body, ntext, counter, events_out, element_name) if body is not None else [] + return [rn] + elif n.type in ("arrow_function", "function_expression"): + return [make("{" + ntext(n, 60) + "}", risk="render-prop")] + return [make("{" + ntext(n, 80) + "}")] + + +def _extract_render(node, react, ntext, events_out): + body = node.child_by_field_name("body") + if body is None: + return + counter = {"n": 0} + jsx_root = None + if body.type != "statement_block": + jsx_root = _unwrap_parens(body) + else: + stack = list(body.named_children) + while stack: + s = stack.pop(0) + if s.type in FN_TYPES: + continue + if s.type == "return_statement": + for ch in s.named_children: + u = _unwrap_parens(ch) + if u is not None and u.type in JSX_TYPES: + jsx_root = u # last JSX return wins + else: + stack.extend(s.named_children) + if jsx_root is None: + return + react["render"] = _lower_jsx(jsx_root, ntext, counter, events_out) + if counter["n"] >= RENDER_BUDGET: + react["render"].append(RenderNode(tag="…", risk="render-truncated", + line=jsx_root.start_point[0] + 1)) + + def lower_components(fn_nodes, ntext, flow_fn) -> bool: upgraded = False for sym, node in fn_nodes: @@ -212,5 +385,8 @@ def lower_components(fn_nodes, ntext, flow_fn) -> bool: _extract_hooks(body, sym.react, ntext, flow_fn) else: sym.react["setters"] = {} + raw_events = [] + _extract_render(node, sym.react, ntext, raw_events) + sym.react["_raw_events"] = raw_events # consumed by Task 5 upgraded = True return upgraded diff --git a/tests/test_react.py b/tests/test_react.py index 73da8e1..3148ed8 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -194,5 +194,76 @@ def test_plain_toplevel_call_not_faulted(self): self.assertEqual(faults, []) +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestRenderTree(unittest.TestCase): + def render(self, src): + mod = kern_compile.parse_tsjs(src, dialect="tsx") + comp = next(s for s in mod.symbols if s.kind == "component") + return comp.react["render"] + + def flat(self, nodes, depth=0): + out = [] + for n in nodes: + out.append((depth, n.tag, n.risk)) + out.extend(self.flat(n.children, depth + 1)) + return out + + def test_hierarchy_and_conditional(self): + tags = self.flat(self.render(TSX_SAMPLE)) + self.assertIn((0, "Card", ""), tags) + self.assertIn((1, "Avatar", ""), tags) + self.assertIn((1, "span", ""), tags) + self.assertIn((1, "IF open", ""), tags) + self.assertIn((2, "UserDetails", ""), tags) + + def test_component_flag_and_attrs(self): + nodes = self.render(TSX_SAMPLE) + card = nodes[0] + self.assertTrue(card.is_component) + avatar = next(c for c in card.children if c.tag == "Avatar") + self.assertEqual(avatar.attrs, "src={user.avatar}") + + def test_map_becomes_for(self): + src = ("function L({ items }) {\n" + " return
      {items.map(item => )}
    ;\n}\n") + tags = [t for _, t, _ in self.flat(self.render(src))] + self.assertIn("FOR item in items", tags) + self.assertIn("Row", tags) + + def test_ternary_if_else(self): + src = ("function T({ ok }) {\n" + " return
    {ok ? : }
    ;\n}\n") + tags = [t for _, t, _ in self.flat(self.render(src))] + self.assertIn("IF ok", tags) + self.assertIn("ELSE", tags) + + def test_dynamic_component_faulted(self): + src = ("function T() {\n return ;\n}\n") + flat = self.flat(self.render(src)) + self.assertIn("dynamic-component", [r for _, _, r in flat]) + + def test_spread_sole_prop_source_faulted(self): + src = ("function T({ rest }) {\n return ;\n}\n") + node = self.render(src)[0] + self.assertEqual(node.risk, "spread-props") + self.assertIn("...rest", node.attrs) + + def test_spread_with_named_attrs_not_faulted(self): + src = ("function T({ rest }) {\n return ;\n}\n") + self.assertEqual(self.render(src)[0].risk, "") + + def test_render_prop_faulted(self): + src = ("function T() {\n" + " return {item => };\n}\n") + flat = self.flat(self.render(src)) + self.assertIn("render-prop", [r for _, _, r in flat]) + + def test_fragment(self): + src = "function T() {\n return <>;\n}\n" + tags = [t for _, t, _ in self.flat(self.render(src))] + self.assertIn("A", tags) + self.assertIn("B", tags) + + if __name__ == "__main__": unittest.main() From a6104da518096755195652c075f8c991ce8c0987 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:37:39 +0200 Subject: [PATCH 41/55] fix: render selection last-return, truncation off-by-one, fault stacking Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 56 ++++++++++++++++++++++--------- tests/test_react.py | 34 +++++++++++++++++++ 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index 475c975..e440a5d 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -237,6 +237,7 @@ def _jsx_attrs(el, ntext): def _lower_jsx(node, ntext, counter, events_out, element_name=""): """Lower one JSX node (or jsx child) to list[RenderNode].""" if counter["n"] >= RENDER_BUDGET: + counter["dropped"] = True return [] n = _unwrap_parens(node) if n is None: @@ -244,15 +245,20 @@ def _lower_jsx(node, ntext, counter, events_out, element_name=""): out = [] def make(tag, **kw): + if counter["n"] >= RENDER_BUDGET: + counter["dropped"] = True + return None counter["n"] += 1 return RenderNode(tag=tag, line=n.start_point[0] + 1, **kw) if n.type in ("jsx_element", "jsx_self_closing_element"): name, is_comp, risk = _jsx_name(n, ntext) attrs, spread_only, ev = _jsx_attrs(n, ntext) - if spread_only and not risk: - risk = "spread-props" + if spread_only: + risk = f"{risk}+spread-props" if risk else "spread-props" rn = make(name, attrs=attrs, is_component=is_comp, risk=risk) + if rn is None: + return [] for aname, value in ev: events_out.append((name, aname, value)) if n.type == "jsx_element": @@ -265,7 +271,9 @@ def make(tag, **kw): elif n.type == "jsx_text": txt = ntext(n, 60) if txt: - out.append(make(txt)) + rn = make(txt) + if rn is not None: + out.append(rn) elif n.type == "jsx_expression": inner = n.named_children[0] if n.named_children else None out.extend(_lower_expr(inner, ntext, counter, events_out, element_name)) @@ -277,10 +285,16 @@ def make(tag, **kw): def _lower_expr(n, ntext, counter, events_out, element_name): n = _unwrap_parens(n) - if n is None or counter["n"] >= RENDER_BUDGET: + if n is None: + return [] + if counter["n"] >= RENDER_BUDGET: + counter["dropped"] = True return [] def make(tag, **kw): + if counter["n"] >= RENDER_BUDGET: + counter["dropped"] = True + return None counter["n"] += 1 return RenderNode(tag=tag, line=n.start_point[0] + 1, **kw) @@ -292,6 +306,8 @@ def make(tag, **kw): if op is not None and op.type == "&&" and right is not None and right.type in JSX_TYPES: left = n.child_by_field_name("left") rn = make(f"IF {ntext(left, 80)}", is_structure=True) + if rn is None: + return [] rn.children = _lower_jsx(right, ntext, counter, events_out, element_name) return [rn] elif n.type == "ternary_expression": @@ -301,12 +317,14 @@ def make(tag, **kw): branches = [] if cons is not None and cons.type in JSX_TYPES: rn = make(f"IF {ntext(cond, 80)}", is_structure=True) - rn.children = _lower_jsx(cons, ntext, counter, events_out, element_name) - branches.append(rn) + if rn is not None: + rn.children = _lower_jsx(cons, ntext, counter, events_out, element_name) + branches.append(rn) if alt is not None and alt.type in JSX_TYPES: el = make("ELSE", is_structure=True) - el.children = _lower_jsx(alt, ntext, counter, events_out, element_name) - branches.append(el) + if el is not None: + el.children = _lower_jsx(alt, ntext, counter, events_out, element_name) + branches.append(el) if branches: return branches elif n.type == "call_expression": @@ -328,39 +346,47 @@ def make(tag, **kw): else: param_txt = "_" rn = make(f"FOR {param_txt} in {ntext(receiver, 60)}", is_structure=True) + if rn is None: + return [] body = cb.child_by_field_name("body") rn.children = _lower_jsx(body, ntext, counter, events_out, element_name) if body is not None else [] return [rn] elif n.type in ("arrow_function", "function_expression"): - return [make("{" + ntext(n, 60) + "}", risk="render-prop")] - return [make("{" + ntext(n, 80) + "}")] + rn = make("{" + ntext(n, 60) + "}", risk="render-prop") + return [rn] if rn is not None else [] + rn = make("{" + ntext(n, 80) + "}") + return [rn] if rn is not None else [] def _extract_render(node, react, ntext, events_out): body = node.child_by_field_name("body") if body is None: return - counter = {"n": 0} + counter = {"n": 0, "dropped": False} jsx_root = None if body.type != "statement_block": jsx_root = _unwrap_parens(body) + if jsx_root is not None and jsx_root.type not in JSX_TYPES: + jsx_root = None else: + best_line = -1 stack = list(body.named_children) while stack: - s = stack.pop(0) + s = stack.pop() if s.type in FN_TYPES: continue if s.type == "return_statement": for ch in s.named_children: u = _unwrap_parens(ch) - if u is not None and u.type in JSX_TYPES: - jsx_root = u # last JSX return wins + if u is not None and u.type in JSX_TYPES and s.start_point[0] > best_line: + best_line = s.start_point[0] + jsx_root = u # textually last JSX-bearing return wins else: stack.extend(s.named_children) if jsx_root is None: return react["render"] = _lower_jsx(jsx_root, ntext, counter, events_out) - if counter["n"] >= RENDER_BUDGET: + if counter.get("dropped"): react["render"].append(RenderNode(tag="…", risk="render-truncated", line=jsx_root.start_point[0] + 1)) diff --git a/tests/test_react.py b/tests/test_react.py index 3148ed8..6a25ece 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -264,6 +264,40 @@ def test_fragment(self): self.assertIn("A", tags) self.assertIn("B", tags) + def test_guard_clause_last_return_wins(self): + src = ("function Card({ loading }) {\n" + " if (loading) { return ; }\n" + " return ;\n}\n") + tags = [t for _, t, _ in self.flat(self.render(src))] + self.assertEqual(tags, ["Content"]) + + def test_two_guards_last_return_wins(self): + src = ("function T({ a, b }) {\n" + " if (a) return ;\n" + " if (b) return ;\n" + " return ;\n}\n") + tags = [t for _, t, _ in self.flat(self.render(src))] + self.assertEqual(tags, ["C"]) + + def test_exact_budget_no_false_truncation(self): + # 1 (outer div) + 199 self-closing children == RENDER_BUDGET (200) + # exactly, with nothing dropped -- must NOT be flagged as truncated. + children = "" * 199 + src = f"function T() {{\n return
    {children}
    ;\n}}\n" + risks = [r for _, _, r in self.flat(self.render(src))] + self.assertNotIn("render-truncated", risks) + + def test_overflow_truncation_faulted(self): + children = "".join(f"{{x{i}}}" for i in range(120)) + src = f"function T() {{\n return
    {children}
    ;\n}}\n" + risks = [r for _, _, r in self.flat(self.render(src))] + self.assertIn("render-truncated", risks) + + def test_dynamic_component_with_spread_keeps_both_faults(self): + src = "function T({ p }) {\n return ;\n}\n" + risks = [r for _, _, r in self.flat(self.render(src))] + self.assertIn("dynamic-component+spread-props", risks) + if __name__ == "__main__": unittest.main() From f23faa8b18c642b2beb76e2dfd3d63b43acb3b34 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:43:07 +0200 Subject: [PATCH 42/55] feat: React event handler extraction with setter lowering Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 39 ++++++++++++++++++++++++++++++- tests/test_react.py | 30 ++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index e440a5d..0df443a 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -358,6 +358,42 @@ def make(tag, **kw): return [rn] if rn is not None else [] +def _event_action(value_node, setters, ntext): + expr = value_node + if expr.type == "jsx_expression": + expr = expr.named_children[0] if expr.named_children else None + expr = _unwrap_parens(expr) + if expr is None: + return "" + if expr.type in ("arrow_function", "function_expression"): + body = expr.child_by_field_name("body") + body = _unwrap_parens(body) + if body is not None and body.type == "statement_block": + calls = [c for c in body.named_children + if c.type == "expression_statement" and c.named_children + and c.named_children[0].type == "call_expression"] + body = calls[0].named_children[0] if len(calls) == 1 else None + if body is not None and body.type == "call_expression": + callee = body.child_by_field_name("function") + callee_txt = ntext(callee, 60) if callee is not None else "" + if callee_txt in setters: + args = body.child_by_field_name("arguments") + arg = args.named_children[0] if args is not None and args.named_children else None + arg_txt = ntext(arg, 60) if arg is not None else "undefined" + return f"set {setters[callee_txt]}={arg_txt}" + return callee_txt + return ntext(expr, 80) + return ntext(expr, 60) + + +def _extract_events(react, ntext): + setters = react.get("setters", {}) + for element, attr, value in react.pop("_raw_events", []): + action = _event_action(value, setters, ntext) + react["events"].append(EventUse(f"{element}.{attr}", action, + value.start_point[0] + 1)) + + def _extract_render(node, react, ntext, events_out): body = node.child_by_field_name("body") if body is None: @@ -413,6 +449,7 @@ def lower_components(fn_nodes, ntext, flow_fn) -> bool: sym.react["setters"] = {} raw_events = [] _extract_render(node, sym.react, ntext, raw_events) - sym.react["_raw_events"] = raw_events # consumed by Task 5 + sym.react["_raw_events"] = raw_events + _extract_events(sym.react, ntext) upgraded = True return upgraded diff --git a/tests/test_react.py b/tests/test_react.py index 6a25ece..bd46003 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -299,5 +299,35 @@ def test_dynamic_component_with_spread_keeps_both_faults(self): self.assertIn("dynamic-component+spread-props", risks) +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestEvents(unittest.TestCase): + def events(self, src): + mod = kern_compile.parse_tsjs(src, dialect="tsx") + comp = next(s for s in mod.symbols if s.kind == "component") + return comp.react["events"] + + def test_setter_arrow_lowered(self): + ev = self.events(TSX_SAMPLE) + self.assertEqual([(e.target, e.action) for e in ev], + [("Card.onClick", "set open=true")]) + + def test_handler_reference(self): + src = ("function T({ onSave }) {\n" + " return ;\n}\n") + ev = self.events(src) + self.assertEqual((ev[0].target, ev[0].action), ("button.onClick", "onSave")) + + def test_non_setter_arrow_uses_callee(self): + src = ("function T() {\n" + " return     analytics.track('x')} />;\n}\n") + ev = self.events(src) + self.assertEqual(ev[0].action, "analytics.track") + + def test_raw_events_cleaned_up(self): + mod = kern_compile.parse_tsjs(TSX_SAMPLE, dialect="tsx") + comp = next(s for s in mod.symbols if s.kind == "component") + self.assertNotIn("_raw_events", comp.react) + + if __name__ == "__main__": unittest.main() From dee4f17824782818557ca3ac53bc3e42c72931ba Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:48:11 +0200 Subject: [PATCH 43/55] fix: event setter lowering requires single-statement handler body Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 11 +++++++---- tests/test_react.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index 0df443a..9832388 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -369,10 +369,13 @@ def _event_action(value_node, setters, ntext): body = expr.child_by_field_name("body") body = _unwrap_parens(body) if body is not None and body.type == "statement_block": - calls = [c for c in body.named_children - if c.type == "expression_statement" and c.named_children - and c.named_children[0].type == "call_expression"] - body = calls[0].named_children[0] if len(calls) == 1 else None + stmts = body.named_children + if (len(stmts) == 1 and stmts[0].type == "expression_statement" + and stmts[0].named_children + and stmts[0].named_children[0].type == "call_expression"): + body = stmts[0].named_children[0] + else: + body = None if body is not None and body.type == "call_expression": callee = body.child_by_field_name("function") callee_txt = ntext(callee, 60) if callee is not None else "" diff --git a/tests/test_react.py b/tests/test_react.py index bd46003..4c5ad18 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -328,6 +328,21 @@ def test_raw_events_cleaned_up(self): comp = next(s for s in mod.symbols if s.kind == "component") self.assertNotIn("_raw_events", comp.react) + def test_guarded_setter_not_lowered_to_unconditional_set(self): + src = ("function T() {\n" + " const [open, setOpen] = useState(false);\n" + " return { if (open) return; setOpen(true); }} />;\n}\n") + ev = self.events(src) + self.assertNotEqual(ev[0].action, "set open=true") + self.assertIn("if", ev[0].action) # guard remains visible in the fallback text + + def test_single_statement_setter_still_lowered(self): + src = ("function T() {\n" + " const [open, setOpen] = useState(false);\n" + " return { setOpen(true); }} />;\n}\n") + ev = self.events(src) + self.assertEqual(ev[0].action, "set open=true") + if __name__ == "__main__": unittest.main() From de7dba71ca2c44e79156ec3a6de511d7039e1e57 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:51:25 +0200 Subject: [PATCH 44/55] feat: emit COMPONENT symbols with tiered render trees and fault footer Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_cache.py | 2 +- skills/kern/scripts/kern_compile.py | 6 ++- skills/kern/scripts/kern_react.py | 67 +++++++++++++++++++++++++++++ tests/test_react.py | 44 +++++++++++++++++++ 4 files changed, 117 insertions(+), 2 deletions(-) diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 74e0e4f..ea06ed4 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -845,7 +845,7 @@ def verify_symbol(root: Path, paths: dict[str, Path], relative: str, source: Pat raise RuntimeError(f"current source does not parse ({module.parse_error}); fault exact source") base = {"ok": True, "operation": "verify", "source_rel": relative, "symbol": symbol, "source_sha256": sha256_bytes(data)} - matches = [s for s in module.symbols if s.kind in {"function", "class"} and s.name == symbol] + matches = [s for s in module.symbols if s.kind in {"function", "class", "component"} and s.name == symbol] if not matches: return {**base, "result": "stale", "reason": "symbol-not-found"} diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 2fef339..01eaea5 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -202,7 +202,7 @@ def classify_call(call_name: str) -> list: def propagate(module: ModuleIR) -> None: - funcs = [s for s in module.symbols if s.kind == "function"] + funcs = [s for s in module.symbols if s.kind in ("function", "component")] by_tail: dict[str, list] = {} for s in funcs: by_tail.setdefault(s.name.split(".")[-1], []).append(s) @@ -761,6 +761,10 @@ def emit_il(module: ModuleIR, source_rel: str, source_sha256: str, elif s.kind == "function": out.append("") out.extend(_function_lines(s, level, tier, faults)) + elif s.kind == "component": + import kern_react + out.append("") + out.extend(kern_react.component_lines(s, level, tier, faults)) omit = " ".join(f"{k}={v}" for k, v in sorted(module.omit.items())) out.extend([ "", diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index 9832388..a98474f 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -430,6 +430,73 @@ def _extract_render(node, react, ntext, events_out): line=jsx_root.start_point[0] + 1)) +def _render_lines(nodes, level, depth, faults, lines): + for n in nodes: + keep = level >= 3 or n.is_component or n.is_structure + if not keep: + _render_lines(n.children, level, depth, faults, lines) + continue + piece = n.tag + if level >= 3 and n.attrs: + piece += f" {n.attrs}" + if n.risk: + piece += f" !FAULT({n.risk})" + faults.append(f"{n.risk}(L{n.line})") + kept_children = _kept(n.children, level) + if level == 2 and n.is_structure and len(kept_children) == 1 and not kept_children[0].children: + child = kept_children[0] + piece += f" > {child.tag}" + if child.risk: + piece += f" !FAULT({child.risk})" + faults.append(f"{child.risk}(L{child.line})") + lines.append(" " * depth + piece) + continue + lines.append(" " * depth + piece) + _render_lines(n.children, level, depth + 1, faults, lines) + + +def _kept(nodes, level): + out = [] + for n in nodes: + if level >= 3 or n.is_component or n.is_structure: + out.append(n) + else: + out.extend(_kept(n.children, level)) + return out + + +def component_lines(s, level, tier, faults): + lines = [f"COMPONENT {s.name}({s.signature}) @L{s.span[0]}-{s.span[1]} ^{s.slice8} ~{tier}"] + if level == 1: + return lines + r = s.react + if r.get("wrapper"): + lines.append(f" WRAP {r['wrapper']}") + if r.get("props"): + lines.append(" PROPS " + ", ".join(r["props"])) + for h in r.get("hooks", []): + tag = "" + if h.risk: + tag = f" !FAULT({h.risk})" + faults.append(f"{h.risk}(L{h.line})") + lines.append(f" {h.kind} {h.detail}{tag}") + if h.kind == "EFFECT" and level >= 3: + for op in h.flow: + piece = op.op + (f" {op.detail}" if op.detail else "") + if op.binds: + piece += f" -> {op.binds}" + lines.append(" " * (op.depth + 2) + piece) + for e in r.get("events", []): + lines.append(f" EVENT {e.target} -> {e.action}") + for risk, line in r.get("faults", []): + lines.append(f" !FAULT({risk}) @L{line}") + faults.append(f"{risk}(L{line})") + if r.get("render"): + lines.append(" RENDER") + _render_lines(r["render"], level, 2, faults, lines) + return lines + + def lower_components(fn_nodes, ntext, flow_fn) -> bool: upgraded = False for sym, node in fn_nodes: diff --git a/tests/test_react.py b/tests/test_react.py index 4c5ad18..67abc3a 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -344,5 +344,49 @@ def test_single_statement_setter_still_lowered(self): self.assertEqual(ev[0].action, "set open=true") +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestEmit(unittest.TestCase): + def il(self, tier, src=None): + mod = kern_compile.parse_tsjs(src or TSX_SAMPLE, dialect="tsx") + return kern_compile.emit_il(mod, "app/UserCard.tsx", "a" * 64, "none", tier) + + def test_l1_head_only(self): + il = self.il("L1") + self.assertIn("COMPONENT UserCard({ user, onClose = noop }) @L", il) + self.assertNotIn("STATE", il) + self.assertNotIn("RENDER", il) + + def test_l2_heads_and_collapsed_render(self): + il = self.il("L2") + self.assertIn(" PROPS user, onClose=noop", il) + self.assertIn(" STATE open=false", il) + self.assertIn(" EFFECT deps=[user.id]", il) + self.assertIn(" EVENT Card.onClick -> set open=true", il) + self.assertIn(" RENDER", il) + self.assertIn("Card", il) + self.assertIn("IF open > UserDetails", il) # single kept child inlined + self.assertNotIn("span", il) # host element collapsed at L2 + self.assertNotIn("src={user.avatar}", il) # attrs dropped at L2 + + def test_l3_full_render_and_effect_body(self): + il = self.il("L3") + self.assertIn("span", il) + self.assertIn("src={user.avatar}", il) + self.assertIn("CALL analytics.track", il) # effect body flow + + def test_frontend_header_tag(self): + self.assertIn("frontend=tree-sitter+react", self.il("L2")) + + def test_fault_footer(self): + src = "function T() {\n return ;\n}\n" + il = self.il("L2", src) + self.assertIn("!FAULT(dynamic-component)", il) + self.assertIn("dynamic-component(L", il) # FAULT-BEFORE footer + + def test_plain_function_in_same_file_unchanged(self): + il = self.il("L2") + self.assertIn("F formatName(name: string) -> string", il) + + if __name__ == "__main__": unittest.main() From 32cffcfbdfbc944fb80addb2c2606ddaf10894b8 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 21:59:19 +0200 Subject: [PATCH 45/55] fix: footer faults for collapsed render nodes, kept-child inline rule Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 5 ++++- tests/test_react.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index a98474f..bf30086 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -434,6 +434,8 @@ def _render_lines(nodes, level, depth, faults, lines): for n in nodes: keep = level >= 3 or n.is_component or n.is_structure if not keep: + if n.risk: + faults.append(f"{n.risk}(L{n.line})") _render_lines(n.children, level, depth, faults, lines) continue piece = n.tag @@ -443,7 +445,8 @@ def _render_lines(nodes, level, depth, faults, lines): piece += f" !FAULT({n.risk})" faults.append(f"{n.risk}(L{n.line})") kept_children = _kept(n.children, level) - if level == 2 and n.is_structure and len(kept_children) == 1 and not kept_children[0].children: + if (level == 2 and n.is_structure and len(kept_children) == 1 + and not _kept(kept_children[0].children, level)): child = kept_children[0] piece += f" > {child.tag}" if child.risk: diff --git a/tests/test_react.py b/tests/test_react.py index 67abc3a..8b4e194 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -387,6 +387,18 @@ def test_plain_function_in_same_file_unchanged(self): il = self.il("L2") self.assertIn("F formatName(name: string) -> string", il) + def test_l2_collapsed_node_fault_reaches_footer(self): + src = ("function T({ props }) {\n" + " return
    hi
    ;\n}\n") + il = self.il("L2", src) + self.assertIn("spread-props(L", il) # footer net holds at L2 + + def test_l2_inline_when_child_has_only_host_children(self): + src = ("function T({ open, user }) {\n" + " return
    {open && x}
    ;\n}\n") + il = self.il("L2", src) + self.assertIn("IF open > UserDetails", il) + if __name__ == "__main__": unittest.main() From 91574708832e6ba813187a1ed72b8d909874327c Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 22:04:31 +0200 Subject: [PATCH 46/55] test: React no-op regression, corpus results, frontend docs Adds the byte-identical no-op regression test (plain JS with no JSX stays frontend=tree-sitter, no COMPONENT symbols emitted) and a "React frontend (tsx/jsx)" section in docs/deterministic-compiler.md covering dialect routing, the COMPONENT vocabulary (copied verbatim from the design spec), tier behavior, and the six fault markers. Corpus run (L2, KERN's own app/ + a fresh shallow clone of vercel/commerce, node_modules excluded): files=69 crashes=0 components=61 faults=8 ratio min=1.9 median=3.3 max=10.1 Zero crashes on first run; no compiler fixes were needed. Full suite: 154 tests, all passing (153 baseline + 1 new). Co-Authored-By: Claude Fable 5 --- docs/deterministic-compiler.md | 128 +++++++++++++++++++++++++++++++++ tests/test_react.py | 17 +++++ 2 files changed, 145 insertions(+) diff --git a/docs/deterministic-compiler.md b/docs/deterministic-compiler.md index 7b7997f..ee35271 100644 --- a/docs/deterministic-compiler.md +++ b/docs/deterministic-compiler.md @@ -392,3 +392,131 @@ Tiering is the honest lever: L3→L2 alone saves 28% with no decode risk. size; publish per-language results under `benchmarks/results/`. 5. **Fallback:** corpus runs with tree-sitter absent and with syntax-broken files must degrade to the labeled generic baseline, never crash. + +--- + +## React frontend (tsx/jsx) + +Step 1's grammar table lists `tree-sitter` for TypeScript and JavaScript, but two +gaps sat under that row: `.tsx` files were routed to the plain TypeScript grammar +(no JSX support, so JSX-bearing `.tsx` degraded to a parse error), and `.jsx` +parsed fine but rendered every component as a truncated `FN` — hooks, state, +effects, events, and render structure were invisible. `kern_react.py` closes both +gaps as a post-pass over the same tree-sitter tree: no new parser, no new format. +`COMPONENT` is a symbol kind inside the existing `ModuleIR`/`Symbol`/`emit_il` +pipeline, so spans, slice hashes, tiers, faults, verify, cache, and redaction are +inherited verbatim. + +### Dialect routing + +| suffix | grammar | +| --- | --- | +| `.js` `.jsx` `.mjs` `.cjs` | `tree_sitter_javascript` (JSX built in) | +| `.ts` | `tree_sitter_typescript.language_typescript()` | +| `.tsx` | `tree_sitter_typescript.language_tsx()` | + +`parse_tsjs(text, dialect: str)` takes `"js" | "ts" | "tsx"` instead of a +`typescript: bool`. Both call sites (`kern_cache.py` compile and verify paths) +use it. The `generator=` header line reports `frontend=tree-sitter+react` when +the adapter fires, and plain `frontend=tree-sitter` otherwise — a strict no-op +on non-component code, verified by `TestNoOpOnPlainCode` in `tests/test_react.py`. + +A symbol is upgraded to `kind="component"` when it is a function declaration, +function expression, or arrow function, its name matches `^[A-Z]`, and a +`return` (or arrow expression body) contains a `jsx_element`, +`jsx_self_closing_element`, or `jsx_fragment`. `memo(Fn)` / `forwardRef(Fn)` +wrappers are unwrapped: the inner function is lowered, the wrapper noted. +Capitalized functions with no JSX stay plain `FN`; lowercase functions +returning JSX also stay `FN` (not components by React convention). + +### Extraction vocabulary + +```text +COMPONENT UserCard L3-18 #a3f9c2d1 + PROPS user, onClose?=noop + STATE open=false + STATE [state, dispatch]=useReducer(reducer, init) + CTX theme=useContext(ThemeContext) + REF inputRef + HOOK data=useUserData(id) + EFFECT deps=[user.id] + EVENT Card.onClick -> set open=true + RENDER + Card + Avatar src=user.avatar + span {user.name} + IF open > UserDetails user=user + FOR item in items > Row key=item.id +``` + +Rules: + +- **PROPS** — first parameter. Destructured pattern lists names with defaults + (`onClose?=noop` when a default exists; `?` when the TS type marks it optional + and that is syntactically visible). Non-destructured param renders as its name + (`props`). +- **STATE** — `const [x, setX] = useState(init)` → `STATE x=init`; setter name + recorded internally for EVENT lowering. `useReducer` renders the pair and + arguments. +- **CTX / REF / HOOK** — `useContext`, `useRef`, and custom `use[A-Z]\w*` calls + respectively. Custom hooks are opaque: call text only, no cross-file + resolution. +- **EFFECT** — `useEffect` / `useLayoutEffect`. Dependency array rendered + verbatim (`deps=[user.id]`, `deps=[]`); a missing array renders + `deps=EVERY-RENDER`. Effect body: L2 shows the head only; L3 summarizes via + existing `flow()`. +- **EVENT** — JSX attribute matching `on[A-Z]\w+={expr}`. If the handler body is + a single known-setter call, lower to `set =`; otherwise render the + callee name or `flow()`-style summary at L3. +- **RENDER** — JSX tree, indentation = nesting: + - `{cond && }` → `IF cond > X`; ternary → `IF cond > X ELSE > Y` + - `.map(` callback returning JSX → `FOR param in receiver > X` + - Host elements (lowercase) are structure; text/expression children render as + `{expr}` capped by existing `ntext` (secret redaction inherited) + - Capitalized JSX names are component dependencies; names that match imports + cross-link naturally through the existing import lines +- Non-hook, non-render statements in the component body flow through the + existing L3 flow-op rendering unchanged. +- All line math is `\n`-only, matching the repo rule (never `str.splitlines()`). + +### Tier mapping + +| Tier | Component detail | +| --- | --- | +| L1 | `COMPONENT name (props) span #hash` — one line, like current FN heads | +| L2 | + STATE/CTX/REF/HOOK/EFFECT/EVENT heads; RENDER collapsed to components-only tree (host elements and attributes dropped; IF/FOR structure kept) | +| L3 | Full render tree with attributes, effect bodies and handler bodies via `flow()` | + +### Faulting + +Ambiguity never disappears silently. Six markers reuse the existing +`!FAULT(...)` inline channel and FAULTS footer: + +| Construct | Marker | +| --- | --- | +| Hook called via alias or namespace (`R.useState`, renamed import) | `!FAULT(aliased-hook)` | +| Dynamic component (`` where Tag is a lowercase variable or member expression) | `!FAULT(dynamic-component)` | +| Spread props as the sole prop source (`{...rest}`) | rendered `...rest` + `!FAULT(spread-props)` | +| Render prop / children-as-function | `!FAULT(render-prop)`, body summarized as flow | +| Hook call inside conditional | `!FAULT(conditional-hook)` | +| Render tree exceeding op budget | explicit `…+N` + `!FAULT(render-truncated)` | + +Frontend IR remains a reasoning representation, not the write authority: edits +still require faulting exact current source and verifying the slice hash, per +the existing contract (Step 5). + +### Corpus run + +Compiled at L2 over two real corpora — KERN's own `app/` directory (4 files: +`layout.jsx`, `page.jsx`, `docs/layout.jsx`, `docs/page.jsx`) and a fresh +shallow clone of [`vercel/commerce`](https://github.com/vercel/commerce) (65 +`.js`/`.jsx`/`.ts`/`.tsx` files, `node_modules` excluded): + +```text +files=69 crashes=0 components=61 faults=8 +ratio min=1.9 median=3.3 max=10.1 +``` + +Zero crashes, components detected on both corpora, ambiguous constructs (spread +props, dynamic components, render-prop children) surfaced as faults rather than +silently dropped — the done bar this spec set for Stage 1 MVP. diff --git a/tests/test_react.py b/tests/test_react.py index 8b4e194..2a95b40 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -400,5 +400,22 @@ def test_l2_inline_when_child_has_only_host_children(self): self.assertIn("IF open > UserDetails", il) +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestNoOpOnPlainCode(unittest.TestCase): + PLAIN = ('import { readFile } from "fs/promises";\n\n' + "export async function load(url) {\n" + " const data = await readFile(url);\n" + " if (!data) {\n throw new Error(url);\n }\n" + " return data.toString();\n}\n") + + def test_component_kind_absent_and_frontend_plain(self): + mod = kern_compile.parse_tsjs(self.PLAIN, dialect="js") + self.assertEqual(mod.frontend, "tree-sitter") + self.assertFalse(any(s.kind == "component" for s in mod.symbols)) + il = kern_compile.emit_il(mod, "src/load.js", "c" * 64, "none", "L2") + self.assertNotIn("COMPONENT", il) + self.assertIn("frontend=tree-sitter tier=L2", il) + + if __name__ == "__main__": unittest.main() From cce8249cad3667e87c6d34e8723da75044cbf36d Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 22:21:07 +0200 Subject: [PATCH 47/55] fix: emit component body flow and EFFECTS line, spec parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit component_lines dropped s.flow and s.effects entirely: non-hook body logic (guards, computed consts, calls) vanished at every tier, violating the spec line 'non-hook, non-render statements flow through the existing L3 flow-op rendering unchanged'. Extract the flow-op loop from _function_lines into a shared flow_lines helper (function output stays byte-identical), then render body flow at L3 — skipping CALL ops whose callee is a hook, since those already surface as STATE/CTX/REF/HOOK/ EFFECT heads — and the EFFECTS provenance line at L2+, after EVENT lines and before faults/RENDER. Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_compile.py | 48 +++++++++++++++++------------ skills/kern/scripts/kern_react.py | 19 ++++++++++++ tests/test_react.py | 30 ++++++++++++++++++ 3 files changed, 77 insertions(+), 20 deletions(-) diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 01eaea5..cc71170 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -460,26 +460,34 @@ def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: if raises: lines.append(" RAISES " + raises) if level >= 2: - for op in s.flow: - # L2 is structure-only: bare CALL carries no name (CALLS has them), - # so only risk-tagged calls earn a line at this tier. - if level == 2 and op.op == "CALL" and not op.risk: - continue - pad = " " * (op.depth + 2) - piece = op.op - if level == 3 and op.detail: - piece += f" {op.detail}" - elif level == 2 and op.op in ("CATCH", "NESTED", "CASE") and op.detail: - piece += f" {op.detail}" - if level == 3 and op.binds: - piece += f" -> {op.binds}" - risk = op.risk - if not risk and level == 3 and any(m in op.detail for m in _ELIDED): - risk = "elided-literal" - if risk: - piece += f" !FAULT({risk})" - faults.append(f"{risk}(L{op.line})") - lines.append(pad + piece) + lines.extend(flow_lines(s, level, tier, faults)) + return lines + + +def flow_lines(s: Symbol, level: int, tier: str, faults: list, ops=None) -> list: + """Render flow ops at the given tier. `ops` overrides `s.flow` so callers + (the React adapter) can pre-filter without copying the symbol.""" + lines = [] + for op in (s.flow if ops is None else ops): + # L2 is structure-only: bare CALL carries no name (CALLS has them), + # so only risk-tagged calls earn a line at this tier. + if level == 2 and op.op == "CALL" and not op.risk: + continue + pad = " " * (op.depth + 2) + piece = op.op + if level == 3 and op.detail: + piece += f" {op.detail}" + elif level == 2 and op.op in ("CATCH", "NESTED", "CASE") and op.detail: + piece += f" {op.detail}" + if level == 3 and op.binds: + piece += f" -> {op.binds}" + risk = op.risk + if not risk and level == 3 and any(m in op.detail for m in _ELIDED): + risk = "elided-literal" + if risk: + piece += f" !FAULT({risk})" + faults.append(f"{risk}(L{op.line})") + lines.append(pad + piece) return lines diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index bf30086..3634976 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -6,6 +6,10 @@ import re from dataclasses import dataclass, field +# Safe: kern_compile imports kern_react only lazily inside functions, so there +# is no cycle at module-exec time. +from kern_compile import SECRET_NAME, _render_provenanced, flow_lines, sanitize_string + JSX_TYPES = {"jsx_element", "jsx_self_closing_element", "jsx_fragment"} HOOK_RE = re.compile(r"^use[A-Z]\w*$") COMPONENT_NAME_RE = re.compile(r"^[A-Z]") @@ -468,6 +472,15 @@ def _kept(nodes, level): return out +def _hook_call_op(op) -> bool: + """True for CALL flow ops whose callee is a hook: already surfaced as + STATE/CTX/REF/HOOK/EFFECT lines, so they must not duplicate as flow ops.""" + if op.op != "CALL": + return False + callee = op.detail.split("(")[0].strip() + return bool(HOOK_RE.match(callee.split(".")[-1])) + + def component_lines(s, level, tier, faults): lines = [f"COMPONENT {s.name}({s.signature}) @L{s.span[0]}-{s.span[1]} ^{s.slice8} ~{tier}"] if level == 1: @@ -491,6 +504,12 @@ def component_lines(s, level, tier, faults): lines.append(" " * (op.depth + 2) + piece) for e in r.get("events", []): lines.append(f" EVENT {e.target} -> {e.action}") + effects = _render_provenanced(s.effects, s.unknown_calls) + if effects: + lines.append(" EFFECTS " + effects) + if level >= 3 and s.flow: + body_ops = [op for op in s.flow if not _hook_call_op(op)] + lines.extend(flow_lines(s, level, tier, faults, ops=body_ops)) for risk, line in r.get("faults", []): lines.append(f" !FAULT({risk}) @L{line}") faults.append(f"{risk}(L{line})") diff --git a/tests/test_react.py b/tests/test_react.py index 2a95b40..dc2be9a 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -399,6 +399,36 @@ def test_l2_inline_when_child_has_only_host_children(self): il = self.il("L2", src) self.assertIn("IF open > UserDetails", il) + def test_l3_component_body_flow_visible(self): + src = ("function Card({ items, loading }) {\n" + " const total = computeTotal(items);\n" + " if (loading) { return ; }\n" + " return
    {total}
    ;\n}\n") + il = self.il("L3", src) + self.assertIn("computeTotal", il) + self.assertIn("IF loading", il) + self.assertIn("Spinner", il) # guard render visible via RET flow op + + def test_l2_component_effects_line(self): + # No TSJS call classifies into a named effect class today; the + # unknown-calls counter still renders, so EFFECTS must appear. + src = ("function T() {\n" + " useEffect(() => { fetch('/api'); }, []);\n" + " console.log('x');\n" + " return
    ;\n}\n") + il = self.il("L2", src) + self.assertIn("EFFECTS", il) + self.assertIn("unknown-calls=", il) + + def test_l3_hook_calls_not_duplicated_as_flow_ops(self): + # useState appears once as STATE, not again as a CALL flow op. + src = ("function T() {\n" + " const [open, setOpen] = useState(false);\n" + " return
    ;\n}\n") + il = self.il("L3", src) + self.assertEqual(il.count("useState"), 0) # STATE open=false covers it + self.assertIn("STATE open=false", il) + @unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") class TestNoOpOnPlainCode(unittest.TestCase): From 957eaaa1fa09ed868cd68f3ec16ea0876b128393 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 22:22:11 +0200 Subject: [PATCH 48/55] fix: detect ternary and logical-and returns as components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _returns_jsx required the return value to BE a JSX node, so 'return loading ? : ' and '=> ok && hi' silently stayed kind=function. New _jsx_bearing helper accepts, one level deep (deterministic), a ternary whose consequence or alternative is JSX and a '&&' binary whose right is JSX — on both the arrow expression-body path and return statements. _extract_render counts these shapes as JSX-bearing for last-return-wins selection and routes them through _lower_expr, which already lowers both to IF/ELSE nodes. Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 36 +++++++++++++++++++++++++------ tests/test_react.py | 24 +++++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index 3634976..70cab57 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -51,13 +51,32 @@ def _unwrap_parens(n): return n +def _jsx_bearing(u) -> bool: + """True for a JSX node, or a one-level conditional expression yielding JSX: + `cond ? : ` or `cond && `. One level only — deterministic.""" + if u is None: + return False + if u.type in JSX_TYPES: + return True + if u.type == "ternary_expression": + cons = _unwrap_parens(u.child_by_field_name("consequence")) + alt = _unwrap_parens(u.child_by_field_name("alternative")) + return ((cons is not None and cons.type in JSX_TYPES) + or (alt is not None and alt.type in JSX_TYPES)) + if u.type == "binary_expression": + op = u.child_by_field_name("operator") + right = _unwrap_parens(u.child_by_field_name("right")) + return (op is not None and op.type == "&&" + and right is not None and right.type in JSX_TYPES) + return False + + def _returns_jsx(fn_node) -> bool: body = fn_node.child_by_field_name("body") if body is None: return False if body.type != "statement_block": - u = _unwrap_parens(body) - return u is not None and u.type in JSX_TYPES + return _jsx_bearing(_unwrap_parens(body)) stack = list(body.named_children) while stack: n = stack.pop() @@ -65,8 +84,7 @@ def _returns_jsx(fn_node) -> bool: continue # returns inside nested functions don't count if n.type == "return_statement": for ch in n.named_children: - u = _unwrap_parens(ch) - if u is not None and u.type in JSX_TYPES: + if _jsx_bearing(_unwrap_parens(ch)): return True stack.extend(n.named_children) return False @@ -409,7 +427,7 @@ def _extract_render(node, react, ntext, events_out): jsx_root = None if body.type != "statement_block": jsx_root = _unwrap_parens(body) - if jsx_root is not None and jsx_root.type not in JSX_TYPES: + if not _jsx_bearing(jsx_root): jsx_root = None else: best_line = -1 @@ -421,14 +439,18 @@ def _extract_render(node, react, ntext, events_out): if s.type == "return_statement": for ch in s.named_children: u = _unwrap_parens(ch) - if u is not None and u.type in JSX_TYPES and s.start_point[0] > best_line: + if _jsx_bearing(u) and s.start_point[0] > best_line: best_line = s.start_point[0] jsx_root = u # textually last JSX-bearing return wins else: stack.extend(s.named_children) if jsx_root is None: return - react["render"] = _lower_jsx(jsx_root, ntext, counter, events_out) + if jsx_root.type in JSX_TYPES: + react["render"] = _lower_jsx(jsx_root, ntext, counter, events_out) + else: + # ternary / `&&` return: _lower_expr lowers both shapes to IF/ELSE nodes + react["render"] = _lower_expr(jsx_root, ntext, counter, events_out, "") if counter.get("dropped"): react["render"].append(RenderNode(tag="…", risk="render-truncated", line=jsx_root.start_point[0] + 1)) diff --git a/tests/test_react.py b/tests/test_react.py index dc2be9a..5845e22 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -108,6 +108,15 @@ def test_non_react_file_untouched(self): self.assertEqual(mod.frontend, "tree-sitter") self.assertEqual(self.sym(mod, "parse").kind, "function") + def test_ternary_return_is_component(self): + mod = self.parse("function Card({ loading }) {\n" + " return loading ? : ;\n}\n") + self.assertEqual(self.sym(mod, "Card").kind, "component") + + def test_logical_and_return_is_component(self): + mod = self.parse("const Hint = ({ ok }) => ok && hi;\n") + self.assertEqual(self.sym(mod, "Hint").kind, "component") + def test_nested_return_jsx_in_inner_fn_not_component(self): src = ("function Outer() {\n" " const inner = () =>
    ;\n" @@ -237,6 +246,21 @@ def test_ternary_if_else(self): self.assertIn("IF ok", tags) self.assertIn("ELSE", tags) + def test_ternary_return_lowered_to_if_else(self): + src = ("function Card({ loading }) {\n" + " return loading ? : ;\n}\n") + tags = [t for _, t, _ in self.flat(self.render(src))] + self.assertIn("IF loading", tags) + self.assertIn("ELSE", tags) + self.assertIn("Spinner", tags) + self.assertIn("Content", tags) + + def test_logical_and_return_lowered_to_if(self): + src = "const Hint = ({ ok }) => ok && hi;\n" + tags = [t for _, t, _ in self.flat(self.render(src))] + self.assertIn("IF ok", tags) + self.assertIn("b", tags) + def test_dynamic_component_faulted(self): src = ("function T() {\n return ;\n}\n") flat = self.flat(self.render(src)) From 87a9736da8b25f250618182486b0efb6bdd01072 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 22:23:06 +0200 Subject: [PATCH 49/55] fix: collect dropped descendant faults under L2 inline rule The 'IF open > UserDetails' inline branch in _render_lines returned via continue without walking the inlined child's dropped descendants, so e.g. a spread-props risk on '
    ' inside the child never reached the FAULT-BEFORE footer at L2. New _collect_faults helper walks child.children recursively before the continue. Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 10 ++++++++++ tests/test_react.py | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index 70cab57..5e4021a 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -456,6 +456,13 @@ def _extract_render(node, react, ntext, events_out): line=jsx_root.start_point[0] + 1)) +def _collect_faults(nodes, faults): + for n in nodes: + if n.risk: + faults.append(f"{n.risk}(L{n.line})") + _collect_faults(n.children, faults) + + def _render_lines(nodes, level, depth, faults, lines): for n in nodes: keep = level >= 3 or n.is_component or n.is_structure @@ -478,6 +485,9 @@ def _render_lines(nodes, level, depth, faults, lines): if child.risk: piece += f" !FAULT({child.risk})" faults.append(f"{child.risk}(L{child.line})") + # The inlined child's descendants are dropped from the tree, but + # their risks must still reach the FAULT-BEFORE footer. + _collect_faults(child.children, faults) lines.append(" " * depth + piece) continue lines.append(" " * depth + piece) diff --git a/tests/test_react.py b/tests/test_react.py index 5845e22..e5370c5 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -423,6 +423,12 @@ def test_l2_inline_when_child_has_only_host_children(self): il = self.il("L2", src) self.assertIn("IF open > UserDetails", il) + def test_l2_inlined_child_descendant_fault_reaches_footer(self): + src = ("function T({ open, rest }) {\n" + " return
    {open &&
    }
    ;\n}\n") + il = self.il("L2", src) + self.assertIn("spread-props(L", il) + def test_l3_component_body_flow_visible(self): src = ("function Card({ items, loading }) {\n" " const total = computeTotal(items);\n" From d967fa00f199c95cec8c50aa68909c082fc2bab3 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 22:24:53 +0200 Subject: [PATCH 50/55] fix: name-hinted secret redaction on React IR surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only SECRET_VALUE-shaped tokens were redacted on the new component surfaces; the Python frontend already redacts by name hint. Now, using SECRET_NAME/sanitize_string imported from kern_compile (safe: the reverse import is lazy, no module-exec cycle): - PROPS: object_assignment_pattern defaults of secret-named props - STATE/CTX/HOOK/useReducer: value text bound to a secret-named binding - JSX attributes: secret-named attr values render as name= - COMPONENT head: signature text scrubbed of secret-named string defaults (_scrub_named_secrets) - L3 body flow ops: details scrubbed the same way, so RET cannot re-leak what RENDER redacts Plain-function output untouched. Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 46 +++++++++++++++++++++++++------ tests/test_react.py | 33 ++++++++++++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index 5e4021a..f1f6ba4 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -4,7 +4,7 @@ from __future__ import annotations import re -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace # Safe: kern_compile imports kern_react only lazily inside functions, so there # is no cycle at module-exec time. @@ -15,6 +15,18 @@ COMPONENT_NAME_RE = re.compile(r"^[A-Z]") FN_TYPES = {"arrow_function", "function_expression", "function_declaration", "generator_function_declaration", "method_definition"} +_STR_ASSIGN = re.compile( + r"(\w+)(\s*=\s*)(\"(?:[^\"\\]|\\.)*\"|'(?:[^'\\]|\\.)*'|`[^`]*`)") + + +def _scrub_named_secrets(text: str) -> str: + """Redact string literals assigned to secret-named identifiers or JSX + attributes inside already-rendered text (signatures, flow-op details).""" + def sub(m): + if SECRET_NAME.search(m.group(1)): + return m.group(1) + m.group(2) + sanitize_string(m.group(3), secret_hint=True) + return m.group(0) + return _STR_ASSIGN.sub(sub, text) @dataclass @@ -112,7 +124,11 @@ def _extract_props(fn_node, ntext) -> list: elif p.type == "object_assignment_pattern": left = p.child_by_field_name("left") right = p.child_by_field_name("right") - out.append(f"{ntext(left, 60)}={ntext(right, 60)}") + left_txt = ntext(left, 60) + right_txt = ntext(right, 60) + if SECRET_NAME.search(left_txt): + right_txt = sanitize_string(right_txt, secret_hint=True) + out.append(f"{left_txt}={right_txt}") elif p.type == "pair_pattern": out.append(ntext(p, 60)) elif p.type == "rest_pattern": @@ -155,21 +171,28 @@ def _extract_hooks(body, react, ntext, flow_fn): if risk: faults.append((risk, line)) name_txt = ntext(name_node, 60) + + def bound(value_txt, name=None): + # Name-hinted secret: redact the value bound to it. + if SECRET_NAME.search(name if name is not None else name_txt): + return sanitize_string(value_txt, secret_hint=True) + return value_txt + if tail == "useState" and name_node.type == "array_pattern": elems = [ntext(e, 40) for e in name_node.named_children] state = elems[0] if elems else "?" init = ntext(args[0], 80) if args else "undefined" if len(elems) > 1: setters[elems[1]] = state - hooks.append(HookUse("STATE", f"{state}={init}", line, risk)) + hooks.append(HookUse("STATE", f"{state}={bound(init, state)}", line, risk)) elif tail == "useReducer": - hooks.append(HookUse("STATE", f"{name_txt}={ntext(value, 120)}", line, risk)) + hooks.append(HookUse("STATE", f"{name_txt}={bound(ntext(value, 120))}", line, risk)) elif tail == "useContext": - hooks.append(HookUse("CTX", f"{name_txt}={ntext(value, 80)}", line, risk)) + hooks.append(HookUse("CTX", f"{name_txt}={bound(ntext(value, 80))}", line, risk)) elif tail == "useRef": hooks.append(HookUse("REF", name_txt, line, risk)) else: - hooks.append(HookUse("HOOK", f"{name_txt}={ntext(value, 120)}", line, risk)) + hooks.append(HookUse("HOOK", f"{name_txt}={bound(ntext(value, 120))}", line, risk)) elif stmt.type == "expression_statement" and stmt.named_children: value = stmt.named_children[0] if value.type in COND_EXPR_TYPES: @@ -249,7 +272,10 @@ def _jsx_attrs(el, ntext): events.append((aname, value)) continue named += 1 - parts.append(ntext(a, 60)) + if SECRET_NAME.search(aname) and value is not None: + parts.append(f"{aname}={sanitize_string(ntext(value, 60), secret_hint=True)}") + else: + parts.append(ntext(a, 60)) elif a.type == "jsx_expression": # {...spread} spread += 1 parts.append(ntext(a, 40).strip("{}")) @@ -540,7 +566,8 @@ def component_lines(s, level, tier, faults): if effects: lines.append(" EFFECTS " + effects) if level >= 3 and s.flow: - body_ops = [op for op in s.flow if not _hook_call_op(op)] + body_ops = [replace(op, detail=_scrub_named_secrets(op.detail)) + for op in s.flow if not _hook_call_op(op)] lines.extend(flow_lines(s, level, tier, faults, ops=body_ops)) for risk, line in r.get("faults", []): lines.append(f" !FAULT({risk}) @L{line}") @@ -558,6 +585,9 @@ def lower_components(fn_nodes, ntext, flow_fn) -> bool: if not COMPONENT_NAME_RE.match(short) or not _returns_jsx(node): continue sym.kind = "component" + # The head line renders the raw signature text; name-hinted string + # defaults (apiToken = "…") must not leak through it. + sym.signature = _scrub_named_secrets(sym.signature) sym.react = { "wrapper": sym.decorators[0] if sym.decorators else "", "props": _extract_props(node, ntext), diff --git a/tests/test_react.py b/tests/test_react.py index e5370c5..8c54b1c 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -460,6 +460,39 @@ def test_l3_hook_calls_not_duplicated_as_flow_ops(self): self.assertIn("STATE open=false", il) +@unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") +class TestRedaction(unittest.TestCase): + def _il(self, src): + mod = kern_compile.parse_tsjs(src, dialect="tsx") + return kern_compile.emit_il(mod, "app/T.tsx", "b" * 64, "none", "L3") + + def test_prop_default_secret_redacted(self): + il = self._il('function Login({ apiToken = "hunter2-plain" }) {\n return
    ;\n}\n') + self.assertNotIn("hunter2-plain", il) + self.assertIn("REDACTED", il) + + def test_state_secret_redacted(self): + il = self._il('function T() {\n const [password, setPassword] = useState("letmein123");\n return
    ;\n}\n') + self.assertNotIn("letmein123", il) + + def test_jsx_attr_secret_redacted(self): + il = self._il('function T() {\n return ;\n}\n') + self.assertNotIn("hunter2abc", il) + + def test_custom_hook_bound_secret_redacted(self): + il = self._il('function T() {\n const authToken = useToken("tok-abc-plain");\n return
    ;\n}\n') + self.assertNotIn("tok-abc-plain", il) + + def test_non_secret_names_untouched(self): + il = self._il('function T({ label = "hello" }) {\n' + ' const [open, setOpen] = useState("wide");\n' + ' return ;\n}\n') + self.assertIn('label = "hello"', il) # signature verbatim + self.assertIn("open=\"wide\"", il) + self.assertIn('placeholder="type here"', il) + self.assertNotIn("REDACTED", il) + + @unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") class TestNoOpOnPlainCode(unittest.TestCase): PLAIN = ('import { readFile } from "fs/promises";\n\n' From 2db9e81c10947592edec0dc7fed087dce44eeb2f Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 22:25:55 +0200 Subject: [PATCH 51/55] fix: paren-unwrapped guarded hooks fault, docstring and docs sync - _extract_hooks unwraps parenthesized expression statements so '(on && useHook());' faults conditional-hook like the bare form - _jsx_attrs docstring: event_attrs are (attr_name, value_node) pairs, not stale triples - docs: generator= line documents lang=tsx alongside frontend=tree-sitter+react; React section and tier table describe the EFFECTS provenance line (L2+) and L3 non-hook body flow Co-Authored-By: Claude Fable 5 --- docs/deterministic-compiler.md | 13 ++++++++----- skills/kern/scripts/kern_react.py | 6 ++++-- tests/test_react.py | 7 +++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/docs/deterministic-compiler.md b/docs/deterministic-compiler.md index ee35271..98fde61 100644 --- a/docs/deterministic-compiler.md +++ b/docs/deterministic-compiler.md @@ -417,8 +417,8 @@ inherited verbatim. `parse_tsjs(text, dialect: str)` takes `"js" | "ts" | "tsx"` instead of a `typescript: bool`. Both call sites (`kern_cache.py` compile and verify paths) -use it. The `generator=` header line reports `frontend=tree-sitter+react` when -the adapter fires, and plain `frontend=tree-sitter` otherwise — a strict no-op +use it. The `generator=` header line reports `lang=tsx frontend=tree-sitter+react` +when the adapter fires, and plain `frontend=tree-sitter` otherwise — a strict no-op on non-component code, verified by `TestNoOpOnPlainCode` in `tests/test_react.py`. A symbol is upgraded to `kind="component"` when it is a function declaration, @@ -476,7 +476,10 @@ Rules: - Capitalized JSX names are component dependencies; names that match imports cross-link naturally through the existing import lines - Non-hook, non-render statements in the component body flow through the - existing L3 flow-op rendering unchanged. + existing L3 flow-op rendering unchanged (hook calls are skipped there — + they already surface as STATE/CTX/REF/HOOK/EFFECT heads). Components also + emit the same `EFFECTS` provenance line as plain functions (effect classes + plus `unknown-calls=N`) at L2 and L3. - All line math is `\n`-only, matching the repo rule (never `str.splitlines()`). ### Tier mapping @@ -484,8 +487,8 @@ Rules: | Tier | Component detail | | --- | --- | | L1 | `COMPONENT name (props) span #hash` — one line, like current FN heads | -| L2 | + STATE/CTX/REF/HOOK/EFFECT/EVENT heads; RENDER collapsed to components-only tree (host elements and attributes dropped; IF/FOR structure kept) | -| L3 | Full render tree with attributes, effect bodies and handler bodies via `flow()` | +| L2 | + STATE/CTX/REF/HOOK/EFFECT/EVENT heads and the `EFFECTS` provenance line; RENDER collapsed to components-only tree (host elements and attributes dropped; IF/FOR structure kept) | +| L3 | Full render tree with attributes, effect bodies, handler bodies, and non-hook body statements via `flow()` | ### Faulting diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index f1f6ba4..8220b42 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -194,7 +194,9 @@ def bound(value_txt, name=None): else: hooks.append(HookUse("HOOK", f"{name_txt}={bound(ntext(value, 120))}", line, risk)) elif stmt.type == "expression_statement" and stmt.named_children: - value = stmt.named_children[0] + value = _unwrap_parens(stmt.named_children[0]) + if value is None: + continue if value.type in COND_EXPR_TYPES: _fault_conditional_hooks(value, faults, ntext) continue @@ -258,7 +260,7 @@ def _jsx_name(el, ntext): def _jsx_attrs(el, ntext): """(attrs_text, spread_only, event_attrs) from an element's opening tag. - event_attrs: list of (attr_name, expr_node, element_name_text).""" + event_attrs: list of (attr_name, value_node) pairs.""" opening = el if el.type == "jsx_element": opening = el.named_children[0] if el.named_children else el diff --git a/tests/test_react.py b/tests/test_react.py index 8c54b1c..0b8982d 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -188,6 +188,13 @@ def test_logical_and_guarded_hook_faulted(self): faults = self.component(src).react["faults"] self.assertIn("conditional-hook", [f[0] for f in faults]) + def test_paren_wrapped_guarded_hook_faulted(self): + src = ("function T({ on }) {\n" + " (on && useHook());\n" + " return
    ;\n}\n") + faults = self.component(src).react["faults"] + self.assertIn("conditional-hook", [f[0] for f in faults]) + def test_ternary_guarded_hook_faulted(self): src = ("function T({ on }) {\n" " on ? useState(0) : null;\n" From 294b1446befae7a338fdc2260c19a1351116d66e Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 22:37:03 +0200 Subject: [PATCH 52/55] fix: scrub name-hinted secrets in EVENT actions and EFFECT body flow Co-Authored-By: Claude Fable 5 --- skills/kern/scripts/kern_react.py | 10 ++++------ tests/test_react.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index 8220b42..b456308 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -557,13 +557,11 @@ def component_lines(s, level, tier, faults): faults.append(f"{h.risk}(L{h.line})") lines.append(f" {h.kind} {h.detail}{tag}") if h.kind == "EFFECT" and level >= 3: - for op in h.flow: - piece = op.op + (f" {op.detail}" if op.detail else "") - if op.binds: - piece += f" -> {op.binds}" - lines.append(" " * (op.depth + 2) + piece) + effect_ops = [replace(op, detail=_scrub_named_secrets(op.detail)) + for op in h.flow] + lines.extend(flow_lines(s, level, tier, faults, ops=effect_ops)) for e in r.get("events", []): - lines.append(f" EVENT {e.target} -> {e.action}") + lines.append(f" EVENT {e.target} -> {_scrub_named_secrets(e.action)}") effects = _render_provenanced(s.effects, s.unknown_calls) if effects: lines.append(" EFFECTS " + effects) diff --git a/tests/test_react.py b/tests/test_react.py index 0b8982d..7cf3a43 100644 --- a/tests/test_react.py +++ b/tests/test_react.py @@ -499,6 +499,28 @@ def test_non_secret_names_untouched(self): self.assertIn('placeholder="type here"', il) self.assertNotIn("REDACTED", il) + def test_event_setter_secret_redacted(self): + # data-pad pushes the RET flow op's 100-char raw-text capture past the + # onClick handler, isolating this test to the EVENT-line surface + # (a separate, pre-existing gap in RET's raw-JSX text capture for + # call-form literals is out of scope for this fix). + pad = "x" * 80 + il = self._il('function T() {\n' + ' const [apiToken, setApiToken] = useState("");\n' + f' return setApiToken("hunter2-literal")}} />;\n}}\n') + self.assertNotIn("hunter2-literal", il) + + def test_event_fallback_secret_redacted(self): + il = self._il('function T() {\n' + ' return { const password = "hunter2-ev"; save(password); }} />;\n}\n') + self.assertNotIn("hunter2-ev", il) + + def test_effect_body_secret_redacted(self): + il = self._il('function T() {\n' + ' useEffect(() => { if (window.token = "hunter2-eff") { save(); } }, []);\n' + ' return
    ;\n}\n') + self.assertNotIn("hunter2-eff", il) + @unittest.skipUnless(kern_compile.tsjs_available(), "tree-sitter not installed") class TestNoOpOnPlainCode(unittest.TestCase): From 6a08759a3df67285cafa1c6d6b4ddc8508d1079f Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 23:21:01 +0200 Subject: [PATCH 53/55] docs: historical note on spec vs shipped parse_tsjs API Co-Authored-By: Claude Fable 5 --- docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md b/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md index 0e6621b..30e8ad4 100644 --- a/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md +++ b/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md @@ -3,6 +3,8 @@ **Date:** 2026-07-11 **Scope:** TSX/JSX grammar routing fix + React semantic extraction (Stage 1 MVP). Next.js adapter, type enrichment, and other frameworks are later specs. +> **Historical note:** this spec predates the compiler-hardening merge from main. The shipped implementation adopts main's `parse_tsjs(text, typescript=, tsx=)` boolean API instead of the `dialect` parameter described below; the semantic vocabulary, tiers, and faulting are as specified. Current behavior: `docs/deterministic-compiler.md`. + ## Problem KERN-IL/0.2 lowers TS/JS through Tree-sitter, but: From c7b7368f10502ef9a8c44a9bc48bc2e6c83e478e Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sat, 11 Jul 2026 23:22:37 +0200 Subject: [PATCH 54/55] chore: refresh deterministic benchmark after kern_cache component kind Source drift only: ADDRESSABLE_SYMBOL_KINDS gained "component" (+3-4 tokens per tier). Values match CI's 3.13 regeneration exactly. Co-Authored-By: Claude Fable 5 --- benchmarks/results/python-det-v2.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/results/python-det-v2.json b/benchmarks/results/python-det-v2.json index f06821f..ede23b7 100644 --- a/benchmarks/results/python-det-v2.json +++ b/benchmarks/results/python-det-v2.json @@ -5,25 +5,25 @@ "bucket": "large(>10k)", "fidelity_missing": [], "file": "skills/kern/scripts/kern_cache.py", - "source_tokens": 15181, + "source_tokens": 15184, "tiers": { "L1": { "fidelity_missing": [], "fidelity_ok": true, "ratio": 3.0, - "tokens": 5061 + "tokens": 5064 }, "L2": { "fidelity_missing": [], "fidelity_ok": true, "ratio": 2.38, - "tokens": 6370 + "tokens": 6374 }, "L3": { "fidelity_missing": [], "fidelity_ok": true, "ratio": 1.18, - "tokens": 12867 + "tokens": 12870 } } } From 42181bbdf5c751b1457cba307a328aee3b85b0d2 Mon Sep 17 00:00:00 2001 From: julio Almeida Date: Sun, 12 Jul 2026 00:53:21 +0200 Subject: [PATCH 55/55] fix: harden React frontend IR --- benchmarks/results/python-det-v2.json | 10 +- docs/deterministic-compiler.md | 11 +- .../2026-07-11-react-frontend-ir-design.md | 7 +- skills/kern/scripts/kern_cache.py | 33 + skills/kern/scripts/kern_compile.py | 154 +++- skills/kern/scripts/kern_react.py | 782 +++++++++++++++--- tests/test_cache_integration.py | 53 ++ tests/test_react_adversarial.py | 393 +++++++++ tests/test_react_wrapper_regressions.py | 131 +++ 9 files changed, 1450 insertions(+), 124 deletions(-) create mode 100644 tests/test_react_adversarial.py create mode 100644 tests/test_react_wrapper_regressions.py diff --git a/benchmarks/results/python-det-v2.json b/benchmarks/results/python-det-v2.json index ede23b7..7a7817c 100644 --- a/benchmarks/results/python-det-v2.json +++ b/benchmarks/results/python-det-v2.json @@ -5,25 +5,25 @@ "bucket": "large(>10k)", "fidelity_missing": [], "file": "skills/kern/scripts/kern_cache.py", - "source_tokens": 15184, + "source_tokens": 15467, "tiers": { "L1": { "fidelity_missing": [], "fidelity_ok": true, - "ratio": 3.0, - "tokens": 5064 + "ratio": 3.01, + "tokens": 5139 }, "L2": { "fidelity_missing": [], "fidelity_ok": true, "ratio": 2.38, - "tokens": 6374 + "tokens": 6494 }, "L3": { "fidelity_missing": [], "fidelity_ok": true, "ratio": 1.18, - "tokens": 12870 + "tokens": 13066 } } } diff --git a/docs/deterministic-compiler.md b/docs/deterministic-compiler.md index 9242d00..b52a5de 100644 --- a/docs/deterministic-compiler.md +++ b/docs/deterministic-compiler.md @@ -447,7 +447,8 @@ Rules: resolution. - **EFFECT** — `useEffect` / `useLayoutEffect`. Dependency array rendered verbatim (`deps=[user.id]`, `deps=[]`); a missing array renders - `deps=EVERY-RENDER`. Effect body: L2 shows the head only; L3 summarizes via + `deps=EVERY-RENDER`. Effect body: L2 normally shows the head only but retains + risk-bearing operations required by the fault contract; L3 summarizes via existing `flow()`. - **EVENT** — JSX attribute matching `on[A-Z]\w+={expr}`. If the handler body is a single known-setter call, lower to `set =`; otherwise render the @@ -463,7 +464,8 @@ Rules: existing L3 flow-op rendering unchanged (hook calls are skipped there — they already surface as STATE/CTX/REF/HOOK/EFFECT heads). Components also emit the same `EFFECTS` provenance line as plain functions (effect classes - plus `unknown-calls=N`) at L2 and L3. + plus `unknown-calls=N`) at L2 and L3. L2 retains only risk-bearing body ops + so their inline markers and `FAULT-BEFORE` entries cannot disappear. - All line math is `\n`-only, matching the repo rule (never `str.splitlines()`). ### Tier mapping @@ -471,12 +473,12 @@ Rules: | Tier | Component detail | | --- | --- | | L1 | `COMPONENT name (props) span #hash` — one line, like current FN heads | -| L2 | + STATE/CTX/REF/HOOK/EFFECT/EVENT heads and the `EFFECTS` provenance line; RENDER collapsed to components-only tree (host elements and attributes dropped; IF/FOR structure kept) | +| L2 | + STATE/CTX/REF/HOOK/EFFECT/EVENT heads, risk-bearing body ops, and the `EFFECTS` provenance line; RENDER collapsed to components-only tree (host elements and attributes dropped; IF/FOR structure kept) | | L3 | Full render tree with attributes, effect bodies, handler bodies, and non-hook body statements via `flow()` | ### Faulting -Ambiguity never disappears silently. Six markers reuse the existing +Ambiguity never disappears silently. Seven markers reuse the existing `!FAULT(...)` inline channel and FAULTS footer: | Construct | Marker | @@ -486,6 +488,7 @@ Ambiguity never disappears silently. Six markers reuse the existing | Spread props as the sole prop source (`{...rest}`) | rendered `...rest` + `!FAULT(spread-props)` | | Render prop / children-as-function | `!FAULT(render-prop)`, body summarized as flow | | Hook call inside conditional | `!FAULT(conditional-hook)` | +| Multiple or nested JSX return paths collapsed to one render root | `!FAULT(render-control-flow)` | | Render tree exceeding op budget | explicit `…+N` + `!FAULT(render-truncated)` | Frontend IR remains a reasoning representation, not the write authority: edits diff --git a/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md b/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md index 30e8ad4..d17eb28 100644 --- a/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md +++ b/docs/superpowers/specs/2026-07-11-react-frontend-ir-design.md @@ -78,14 +78,14 @@ Rules: - **PROPS** — first parameter. Destructured pattern lists names with defaults (`onClose?=noop` when a default exists; `?` when the TS type marks it optional and that is syntactically visible). Non-destructured param renders as its name (`props`). - **STATE** — `const [x, setX] = useState(init)` → `STATE x=init`; setter name recorded internally for EVENT lowering. `useReducer` renders the pair and arguments. - **CTX / REF / HOOK** — `useContext`, `useRef`, and custom `use[A-Z]\w*` calls respectively. Custom hooks are opaque: call text only, no cross-file resolution. -- **EFFECT** — `useEffect` / `useLayoutEffect`. Dependency array rendered verbatim (`deps=[user.id]`, `deps=[]`); a missing array renders `deps=EVERY-RENDER`. Effect body: L2 shows the head only; L3 summarizes via existing `flow()`. +- **EFFECT** — `useEffect` / `useLayoutEffect`. Dependency array rendered verbatim (`deps=[user.id]`, `deps=[]`); a missing array renders `deps=EVERY-RENDER`. Effect body: L2 shows the head plus any risk-bearing ops required by the fault contract; L3 summarizes via existing `flow()`. - **EVENT** — JSX attribute matching `on[A-Z]\w+={expr}`. If the handler body is a single known-setter call, lower to `set =`; otherwise render the callee name or `flow()`-style summary at L3. - **RENDER** — JSX tree, indentation = nesting: - `{cond && }` → `IF cond > X`; ternary → `IF cond > X ELSE > Y` - `.map(` callback returning JSX → `FOR param in receiver > X` - Host elements (lowercase) are structure; text/expression children render as `{expr}` capped by existing `ntext` (secret redaction inherited) - Capitalized JSX names are component dependencies; names that match imports cross-link naturally through the existing import lines -- Non-hook, non-render statements in the component body flow through the existing L3 flow-op rendering unchanged. +- Non-hook, non-render statements in the component body flow through the existing L3 flow-op rendering unchanged; L2 retains only risk-bearing ops. - All line math is `\n`-only, matching the repo rule (never `str.splitlines()`). ## Tier mapping @@ -93,7 +93,7 @@ Rules: | Tier | Component detail | | --- | --- | | L1 | `COMPONENT name (props) span #hash` — one line, like current FN heads | -| L2 | + STATE/CTX/REF/HOOK/EFFECT/EVENT heads; RENDER collapsed to components-only tree (host elements and attributes dropped; IF/FOR structure kept) | +| L2 | + STATE/CTX/REF/HOOK/EFFECT/EVENT heads and risk-bearing body ops; RENDER collapsed to components-only tree (host elements and attributes dropped; IF/FOR structure kept) | | L3 | Full render tree with attributes, effect bodies and handler bodies via `flow()` | ## Faulting @@ -107,6 +107,7 @@ Ambiguity never disappears silently. Reuses the existing `!FAULT(...)` inline ch | Spread props as the sole prop source (`{...rest}`) | rendered `...rest` + `!FAULT(spread-props)` | | Render prop / children-as-function | `!FAULT(render-prop)`, body summarized as flow | | Hook call inside conditional | `!FAULT(conditional-hook)` | +| Multiple or nested JSX return paths collapsed to one render root | `!FAULT(render-control-flow)` | | Render tree exceeding op budget | explicit `…+N` + `!FAULT(render-truncated)` | Frontend IR remains a reasoning representation, not the write authority: edits still require faulting exact current source and verifying the slice hash, per the existing contract. diff --git a/skills/kern/scripts/kern_cache.py b/skills/kern/scripts/kern_cache.py index 5b37282..252d1cb 100755 --- a/skills/kern/scripts/kern_cache.py +++ b/skills/kern/scripts/kern_cache.py @@ -6,6 +6,7 @@ import argparse import copy import hashlib +import importlib import importlib.metadata import json import os @@ -546,6 +547,31 @@ def _distribution_version(name: str) -> str: return "unknown" +def _module_source_fingerprint(module_name: str, fallback: Path | None = None) -> str: + """Return a module source digest, or a deterministic availability marker. + + Optional compiler adapters are still fingerprint inputs when they cannot be + imported: if their source is present, hash the source directly; otherwise + record a stable marker instead of making cache validation fail. + """ + path = fallback + try: + module = importlib.import_module(module_name) + module_file = getattr(module, "__file__", None) + if module_file: + path = Path(module_file) + except Exception: + pass + if path is None: + return "unavailable" + try: + return sha256_file(path) + except FileNotFoundError: + return "missing" + except (OSError, ValueError): + return "unavailable" + + def compiler_fingerprint( source: Path, config: dict[str, Any], @@ -562,6 +588,10 @@ def compiler_fingerprint( "python": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", } if suffix in TSJS_SUFFIXES: + try: + react_fallback = compiler_path.with_name("kern_react.py") + except ValueError: + react_fallback = None capabilities.update( { "tree-sitter": _distribution_version("tree-sitter"), @@ -573,6 +603,9 @@ def compiler_fingerprint( capabilities["tsjs"] = kern_compile.tsjs_capability_fingerprint() except Exception: capabilities["tsjs"] = "unavailable" + capabilities["kern-react-sha256"] = _module_source_fingerprint( + "kern_react", react_fallback + ) payload = { "codec": CODEC_VERSION, "generator": BASELINE_GENERATOR, diff --git a/skills/kern/scripts/kern_compile.py b/skills/kern/scripts/kern_compile.py index 68d708c..5a2236d 100644 --- a/skills/kern/scripts/kern_compile.py +++ b/skills/kern/scripts/kern_compile.py @@ -11,7 +11,7 @@ import importlib.metadata import json import re -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace CODEC_VERSION = "kern-il/0.2" GENERATOR = "kern-det/0.2" @@ -259,6 +259,7 @@ class Symbol: raises_all: dict = field(default_factory=dict) # exception -> list of via names unknown_calls: int = 0 react: dict = field(default_factory=dict) # React adapter payload (kern_react) + redact_body: bool = False # scrub rendered body metadata @dataclass @@ -735,12 +736,14 @@ def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: f"@L{s.span[0]}-{s.span[1]} ^{handle} ~{tier}"] if s.decorators: lines.append(" DECORATORS " + ", ".join(s.decorators)) + display_call = (lambda call: sanitize_string(call, secret_hint=True) + if s.redact_body else call) if level <= 2: if s.calls: # Calls are part of the L1/L2 fidelity contract. If the complete # deduplicated set is no longer economical, the file-level size # policy should select source instead of silently discarding facts. - lines.append(" CALLS " + ", ".join(s.calls)) + lines.append(" CALLS " + ", ".join(display_call(call) for call in s.calls)) else: covered = "\n".join(op.detail for op in s.flow) leftover = [ @@ -748,7 +751,7 @@ def _function_lines(s: Symbol, level: int, tier: str, faults: list) -> list: if not re.search(rf"(? 1: + value = node.named_children[1] + if named_secret(name) and value is not None: + add(value) + elif node_type in { + "method_definition", "method_signature", "abstract_method_signature", + }: + name = field_node(node, "name") + if named_secret(name): + # The method name is useful structure, but a secret-named + # method's defaults, generic/return types, and body are all + # secret-bearing value positions. + for child in node.named_children: + if (name is None + or child.start_byte != name.start_byte + or child.end_byte != name.end_byte): + add(child) elif node_type in {"call_expression", "new_expression"}: callee = field_node(node, "function") or field_node(node, "constructor") if named_secret(callee): @@ -945,6 +990,23 @@ def walk(node): if arguments is not None: for argument in arguments.named_children: add(argument) + if callee is not None and callee.type == "member_expression": + prop = field_node(callee, "property") + if prop is not None and raw_text(prop) == "map": + arguments = field_node(node, "arguments") + callback = None + if arguments is not None and arguments.named_children: + candidate = unwrap_ts_value(arguments.named_children[0]) + if (candidate is not None + and candidate.type in { + "arrow_function", "function_expression", + }): + callback = candidate + if callback is not None: + params = (field_node(callback, "parameter") + or field_node(callback, "parameters")) + if named_secret(params): + add(field_node(callee, "object")) for child in node.named_children: walk(child) @@ -1122,6 +1184,12 @@ def add(current, op, detail="", binds="", risk=""): def function_symbol(node, qualified, decorators=None): decorators = list(decorators or []) + method_name = (field_node(node, "name") + if node.type in { + "method_definition", "method_signature", + "abstract_method_signature", + } else None) + secret_method = bool(method_name is not None and SECRET_NAME.search(raw_text(method_name))) calls, raises, effects = [], [], {} body = field_node(node, "body") if body is not None: @@ -1132,6 +1200,8 @@ def function_symbol(node, qualified, decorators=None): a, b = span_with_decorators(node, decorators) is_async = any(ch.type == "async" for ch in node.children) signature = ntext(params, 240) if params is not None else "" + if secret_method and params is not None: + signature = ntext(params, 240, secret_hint=True) if signature.startswith("(") and signature.endswith(")"): signature = signature[1:-1] if body is None: @@ -1141,13 +1211,22 @@ def function_symbol(node, qualified, decorators=None): else: body_flow = [FlowOp("RET", ntext(body, 140), depth=0, line=body.start_point[0] + 1, risk=ts_risk(body))] + if secret_method: + body_flow = [replace( + op, + detail=(sanitize_string(op.detail, secret_hint=True) + if op.detail else ""), + ) for op in body_flow] + raises = [sanitize_string(item, secret_hint=True) for item in raises] sym = Symbol( kind="function", name=qualified, span=(a, b), signature=signature, - returns=ntext(rtype, 60).lstrip(": ") if rtype is not None else "", + returns=(ntext(rtype, 60, secret_hint=secret_method).lstrip(": ") + if rtype is not None else ""), slice8=slice_sha8(text, a, b), calls=calls, raises=raises, flow=body_flow, is_async=is_async, effects=effects, decorators=[ntext(decorator, 100) for decorator in decorators], + redact_body=secret_method, ) fn_nodes.append((sym, node)) return sym @@ -1158,11 +1237,27 @@ def function_symbol(node, qualified, decorators=None): def add_simple(kind_name, name, node, detail="", decorators=None, bases=""): decorators = list(decorators or []) a, b = span_with_decorators(node, decorators) - symbols.append(Symbol( + sym = Symbol( kind=kind_name, name=name, detail=detail, bases=bases, span=(a, b), slice8=slice_sha8(text, a, b), decorators=[ntext(decorator, 100) for decorator in decorators], - )) + ) + symbols.append(sym) + return sym + + # Wrapper callbacks are only function-like when the React adapter can + # prove they render JSX. Keep the ordinary const symbol as the origin so + # probing memo/forwardRef is a no-op for non-components; successful probes + # are swapped in after React lowering. + wrapper_probes: list[tuple[Symbol, Symbol, object]] = [] + + def unwrap_ts_value(value): + while value is not None and value.type in { + "parenthesized_expression", "as_expression", "satisfies_expression", + "non_null_expression", + }: + value = value.named_children[0] if value.named_children else None + return value def commonjs_assignment(node, prefix) -> bool: left = field_node(node, "left") @@ -1219,23 +1314,32 @@ def visit(node, prefix="", in_class=False, decorators=None): name_node = field_node(declaration, "name") value = field_node(declaration, "value") name = prefix + ntext(name_node, 80) - if value is not None and value.type in {"arrow_function", "function_expression", "generator_function"}: + if value is not None and value.type in { + "arrow_function", "function_expression", "generator_function", + }: symbols.append(function_symbol(value, name)) continue + probe_value = unwrap_ts_value(value) wrapper, inner = "", None - if value is not None and value.type == "call_expression": - callee = field_node(value, "function") + if probe_value is not None and probe_value.type == "call_expression": + callee = field_node(probe_value, "function") wrapper = ntext(callee, 60) if callee is not None else "" if wrapper in ("memo", "forwardRef", "React.memo", "React.forwardRef"): - arguments = field_node(value, "arguments") + arguments = field_node(probe_value, "arguments") if arguments is not None and arguments.named_children: - first = arguments.named_children[0] - if first.type in ("arrow_function", "function_expression"): + first = unwrap_ts_value(arguments.named_children[0]) + if (first is not None + and first.type in ("arrow_function", "function_expression")): inner = first if inner is not None: - sym = function_symbol(inner, name) - sym.decorators = [wrapper.split(".")[-1]] - symbols.append(sym) + detail = "=" + ntext( + value, 160, secret_hint=bool(SECRET_NAME.search(name)) + ) + origin = add_simple("const", name, declaration, detail) + probe = function_symbol(inner, name) + probe.decorators = [wrapper.split(".")[-1]] + owner = node.parent if node.parent.type == "export_statement" else node + wrapper_probes.append((origin, probe, owner)) else: detail = "=" + ntext(value, 160, secret_hint=bool(SECRET_NAME.search(name))) if value is not None else "" add_simple("const", name, declaration, detail) @@ -1326,6 +1430,20 @@ def visit(node, prefix="", in_class=False, decorators=None): import kern_react if kern_react.lower_components(fn_nodes, ntext=ntext, flow_fn=flow): frontend = "tree-sitter+react" + for origin, probe, declaration in wrapper_probes: + if probe.kind != "component": + continue + # The component is owned by the complete declaration, not merely + # its first wrapper argument. Comparator and other wrapper-arg + # edits must therefore change its exact and semantic handles, and + # exact-source faults must cover the complete declaration. + a, b = span(declaration) + probe.span = (a, b) + probe.slice8 = slice_sha8(text, a, b) + for index, symbol in enumerate(symbols): + if symbol is origin: + symbols[index] = probe + break all_nodes = list(walk_nodes(tree.root_node)) lines = text.split("\n") diff --git a/skills/kern/scripts/kern_react.py b/skills/kern/scripts/kern_react.py index d206796..0f78028 100644 --- a/skills/kern/scripts/kern_react.py +++ b/skills/kern/scripts/kern_react.py @@ -8,13 +8,26 @@ # Safe: kern_compile imports kern_react only lazily inside functions, so there # is no cycle at module-exec time. -from kern_compile import SECRET_NAME, _render_provenanced, flow_lines, sanitize_string +from kern_compile import ( + SECRET_NAME, + SECRET_VALUE, + FlowOp, + _render_provenanced, + flow_lines, + one_line_text, + sanitize_string, + sha256_hex, +) JSX_TYPES = {"jsx_element", "jsx_self_closing_element", "jsx_fragment"} HOOK_RE = re.compile(r"^use[A-Z]\w*$") COMPONENT_NAME_RE = re.compile(r"^[A-Z]") +COMPONENT_FN_TYPES = {"arrow_function", "function_expression", "function_declaration"} FN_TYPES = {"arrow_function", "function_expression", "function_declaration", - "generator_function_declaration", "method_definition"} + "generator_function", "generator_function_declaration", "method_definition"} +NESTED_SCOPE_TYPES = FN_TYPES | { + "class", "class_declaration", "abstract_class_declaration", +} _STR_ASSIGN = re.compile( r"(\w+)(\s*=\s*)(\"(?:[^\"\\]|\\.)*\"|'(?:[^'\\]|\\.)*'|`[^`]*`)") @@ -29,12 +42,117 @@ def sub(m): return _STR_ASSIGN.sub(sub, text) +def _raw_text(node) -> str: + if node is None: + return "" + value = getattr(node, "text", b"") + if isinstance(value, bytes): + return value.decode("utf-8", "replace") + return str(value) + + +def _cap_text(text: str, cap: int) -> str: + if len(text) <= cap: + return text + digest = sha256_hex(text.encode("utf-8", "surrogatepass"))[:12] + return text[:max(0, cap - 24)] + f"…" + + +def _secret_pattern_values(root, inherited_secret=False) -> list: + """Return default-value nodes whose binding/property name is secret. + + Tree-sitter's generic text sanitizer does not treat + ``object_assignment_pattern`` as an assignment, and a secret property may + bind to a differently named local (``password: value = make(...)``). Walk + the pattern itself so compound defaults and nested destructuring are + handled by syntax rather than by rendered-text regexes. + """ + if root is None: + return [] + out = [] + + def walk(node, secret=False): + if node.type in {"string", "template_string"}: + if SECRET_VALUE.search(_raw_text(node)): + out.append(node) + return + if node.type in {"call_expression", "new_expression"}: + callee = (node.child_by_field_name("function") + or node.child_by_field_name("constructor")) + if callee is not None and SECRET_NAME.search(_raw_text(callee)): + arguments = node.child_by_field_name("arguments") + if arguments is not None: + out.extend(arguments.named_children) + return + if node.type in {"object_assignment_pattern", "assignment_pattern"}: + left = node.child_by_field_name("left") + right = node.child_by_field_name("right") + named = bool(left is not None and SECRET_NAME.search(_raw_text(left))) + if right is not None and (secret or named): + out.append(right) + return + if left is not None: + walk(left, secret) + if right is not None: + walk(right, secret) + return + if node.type == "pair_pattern": + key = node.child_by_field_name("key") + value = node.child_by_field_name("value") + named = bool(key is not None and SECRET_NAME.search(_raw_text(key))) + if value is not None: + walk(value, secret or named) + return + for child in node.named_children: + walk(child, secret) + + walk(root, inherited_secret) + return out + + +def _render_with_redactions(root, redactions, cap: int) -> str: + """Render *root* while replacing selected descendant nodes atomically.""" + if root is None: + return "" + raw = getattr(root, "text", b"") + if not isinstance(raw, bytes): + raw = str(raw).encode("utf-8", "surrogatepass") + ranges = [] + for node in redactions: + if node.start_byte < root.start_byte or node.end_byte > root.end_byte: + continue + ranges.append((node.start_byte - root.start_byte, + node.end_byte - root.start_byte)) + selected = [] + for start, end in sorted(ranges, key=lambda item: (item[0], -(item[1] - item[0]))): + if selected and start >= selected[-1][0] and end <= selected[-1][1]: + continue + if selected and start < selected[-1][1]: + continue + selected.append((start, end)) + pieces, cursor = [], 0 + for start, end in selected: + pieces.append(raw[cursor:start]) + value = raw[start:end].decode("utf-8", "replace") + pieces.append(sanitize_string(value, secret_hint=True).encode("utf-8")) + cursor = end + pieces.append(raw[cursor:]) + return _cap_text(one_line_text(b"".join(pieces).decode("utf-8", "replace")), cap) + + +def _append_fault(faults, risk: str, line: int): + item = (risk, line) + if item not in faults: + faults.append(item) + + @dataclass class HookUse: kind: str # STATE | CTX | REF | HOOK | EFFECT detail: str line: int risk: str = "" + inherited_risk: str = "" flow: list = field(default_factory=list) # EFFECT body ops for L3 @@ -57,12 +175,26 @@ class RenderNode: def _unwrap_parens(n): - while n is not None and n.type == "parenthesized_expression": + while n is not None and n.type in { + "parenthesized_expression", "as_expression", "satisfies_expression", + "non_null_expression", + }: inner = n.named_children n = inner[0] if inner else None return n +def _function_argument(arguments): + """Return a function-valued first argument after TS expression wrappers.""" + if arguments is None or not arguments.named_children: + return None + candidate = _unwrap_parens(arguments.named_children[0]) + if (candidate is not None + and candidate.type in {"arrow_function", "function_expression"}): + return candidate + return None + + def _jsx_bearing(u) -> bool: """True for a JSX node, or a one-level conditional expression yielding JSX: `cond ? : ` or `cond && `. One level only — deterministic.""" @@ -92,7 +224,7 @@ def _returns_jsx(fn_node) -> bool: stack = list(body.named_children) while stack: n = stack.pop() - if n.type in FN_TYPES: + if n.type in NESTED_SCOPE_TYPES: continue # returns inside nested functions don't count if n.type == "return_statement": for ch in n.named_children: @@ -125,12 +257,14 @@ def _extract_props(fn_node, ntext) -> list: left = p.child_by_field_name("left") right = p.child_by_field_name("right") left_txt = ntext(left, 60) - right_txt = ntext(right, 60) - if SECRET_NAME.search(left_txt): - right_txt = sanitize_string(right_txt, secret_hint=True) + values = _secret_pattern_values(p) + right_txt = (_render_with_redactions(right, [right], 60) + if right in values else ntext(right, 60)) out.append(f"{left_txt}={right_txt}") elif p.type == "pair_pattern": - out.append(ntext(p, 60)) + values = _secret_pattern_values(p) + out.append(_render_with_redactions(p, values, 60) + if values else ntext(p, 60)) elif p.type == "rest_pattern": out.append(ntext(p, 60)) return out @@ -138,11 +272,59 @@ def _extract_props(fn_node, ntext) -> list: EFFECT_HOOKS = {"useEffect", "useLayoutEffect"} -COND_EXPR_TYPES = {"binary_expression", "ternary_expression"} + + +def _optional_chain_reaches_result(node) -> bool: + """Whether an optional-chain segment can short-circuit this expression. + + Follow only the receiver/call chain. Optional expressions inside computed + property keys are evaluated values; they do not make the outer call + conditional. + """ + # Parentheses terminate optional-chain short-circuiting. Type-only TS + # wrappers are runtime-transparent, so follow those but never cross an + # explicit grouping boundary. + while node is not None and node.type in { + "as_expression", "satisfies_expression", "non_null_expression", + }: + node = node.named_children[0] if node.named_children else None + if node is None: + return False + if node.type == "parenthesized_expression": + return False + if node.type == "call_expression": + return _is_optional_call(node) + if node.type not in {"member_expression", "subscript_expression"}: + return False + if any(child.type in {"optional_chain", "?."} for child in node.children): + return True + return _optional_chain_reaches_result(node.child_by_field_name("object")) + + +def _is_optional_call(node) -> bool: + if node is None or node.type != "call_expression": + return False + if any(child.type == "?." for child in node.children): + return True + return _optional_chain_reaches_result(node.child_by_field_name("function")) + + +def _is_conditional_expression(node) -> bool: + if node is None: + return False + if node.type == "ternary_expression" or _is_optional_call(node): + return True + if node.type in {"binary_expression", "augmented_assignment_expression"}: + operator = node.child_by_field_name("operator") + return operator is not None and operator.type in { + "&&", "||", "??", "&&=", "||=", "??=", + } + return False def _call_parts(value, ntext): """(callee_text, args_nodes) for a call_expression, else (None, []).""" + value = _unwrap_parens(value) if value is None or value.type != "call_expression": return None, [] callee = value.child_by_field_name("function") @@ -151,7 +333,135 @@ def _call_parts(value, ntext): list(args.named_children) if args is not None else []) -def _extract_hooks(body, react, ntext, flow_fn): +def _hook_info(callee: str, aliases: dict, shadowed=()) -> tuple[str, bool, bool]: + tail = callee.split(".")[-1] + if "." not in callee: + if callee in shadowed: + return callee, False, False + canonical = aliases.get(callee, callee) + return canonical, canonical != callee, bool(HOOK_RE.match(canonical)) + # A renamed import is a local binding, never a property on an unrelated + # object. Keep namespace-style canonical hook calls conservative, but do + # not let `obj.useS()` inherit `import { useState as useS }` semantics. + if tail in aliases: + return tail, False, False + return tail, True, bool(HOOK_RE.match(tail)) + + +def _react_hook_aliases(fn_nodes, ntext) -> dict: + """Map renamed named imports from React back to their canonical hook.""" + if not fn_nodes: + return {} + root = fn_nodes[0][1] + while root.parent is not None: + root = root.parent + aliases = {} + for statement in root.named_children: + if statement.type != "import_statement": + continue + source = next((child for child in reversed(statement.named_children) + if child.type == "string"), None) + if source is None or _raw_text(source).strip("'\"") != "react": + continue + stack = list(statement.named_children) + while stack: + node = stack.pop() + if node.type == "import_specifier": + original = node.child_by_field_name("name") + alias = node.child_by_field_name("alias") + original_txt = ntext(original, 60) + local_txt = ntext(alias, 60) if alias is not None else original_txt + if HOOK_RE.match(original_txt) and local_txt != original_txt: + aliases[local_txt] = original_txt + continue + stack.extend(node.named_children) + return aliases + + +def _shadowed_hook_aliases(fn_node, aliases: dict) -> set[str]: + """Return imported aliases rebound by this component's function scope.""" + if not aliases: + return set() + names = set() + + def bindings(node): + if node is None or node.type == "type_annotation": + return + if node.type in {"identifier", "shorthand_property_identifier_pattern"}: + text = _raw_text(node) + if text in aliases: + names.add(text) + return + if node.type in {"required_parameter", "optional_parameter"}: + bindings(node.child_by_field_name("pattern")) + return + if node.type in {"assignment_pattern", "object_assignment_pattern"}: + bindings(node.child_by_field_name("left")) + return + if node.type == "pair_pattern": + bindings(node.child_by_field_name("value")) + return + for child in node.named_children: + bindings(child) + + bindings(fn_node.child_by_field_name("parameters") + or fn_node.child_by_field_name("parameter")) + body = fn_node.child_by_field_name("body") + if body is not None and body.type == "statement_block": + for statement in body.named_children: + if statement.type in {"lexical_declaration", "variable_declaration"}: + for declaration in statement.named_children: + if declaration.type == "variable_declarator": + bindings(declaration.child_by_field_name("name")) + elif statement.type in {"function_declaration", "class_declaration"}: + bindings(statement.child_by_field_name("name")) + return names + + +def _node_risk(node, ntext) -> str: + stack = [node] + while stack: + child = stack.pop() + if child.type == "regex": + return "regex" + if child.type in {"call_expression", "new_expression"}: + callee = (child.child_by_field_name("function") + or child.child_by_field_name("constructor")) + name = ntext(callee, 100) + if re.search(r"(?:^|\.)(?:createHash|createHmac|subtle|randomBytes)$", name): + return "crypto" + if re.search(r"(?:^|\.)(?:Worker|Atomics|SharedArrayBuffer)$", name): + return "concurrency" + if re.search(r"(?:^|\.)(?:RegExp|match|replace|search|split)$", name): + return "regex" + if child.type == "binary_expression": + if any(token.type in {"**", "<<", ">>", ">>>"} for token in child.children): + return "math" + stack.extend(child.named_children) + return "" + + +def _expression_flow(node, ntext, flow_fn=None) -> list: + node = _unwrap_parens(node) + if node is None: + return [] + if node.type in {"arrow_function", "function_expression"}: + body = node.child_by_field_name("body") + if body is None: + return [] + if body.type == "statement_block" and flow_fn is not None: + return flow_fn(body) + return _expression_flow(body, ntext, flow_fn) + if node.type in {"call_expression", "new_expression", "await_expression"}: + return [FlowOp("CALL", ntext(node, 120), line=node.start_point[0] + 1, + risk=_node_risk(node, ntext))] + return [FlowOp("RET", ntext(node, 120), line=node.start_point[0] + 1, + risk=_node_risk(node, ntext))] + + +def _extract_hooks(body, react, ntext, flow_fn, aliases=None, shadowed=None): + aliases = aliases or {} + shadowed = shadowed or set() hooks, setters, faults = react["hooks"], {}, react["faults"] for stmt in body.named_children: line = stmt.start_point[0] + 1 @@ -161,15 +471,18 @@ def _extract_hooks(body, react, ntext, flow_fn): continue name_node = d.child_by_field_name("name") value = d.child_by_field_name("value") + _fault_nested_conditional_hooks( + value, faults, ntext, aliases, shadowed) callee, args = _call_parts(value, ntext) if callee is None: continue - tail = callee.split(".")[-1] - if not HOOK_RE.match(tail): + tail, is_alias, is_hook = _hook_info(callee, aliases, shadowed) + if not is_hook: continue - risk = "aliased-hook" if "." in callee else "" + risk = "aliased-hook" if is_alias else "" + inherited_risk = _node_risk(value, ntext) if risk: - faults.append((risk, line)) + _append_fault(faults, risk, line) name_txt = ntext(name_node, 60) def bound(value_txt, name=None): @@ -184,61 +497,145 @@ def bound(value_txt, name=None): init = ntext(args[0], 80) if args else "undefined" if len(elems) > 1: setters[elems[1]] = state - hooks.append(HookUse("STATE", f"{state}={bound(init, state)}", line, risk)) + hooks.append(HookUse("STATE", f"{state}={bound(init, state)}", + line, risk, inherited_risk)) elif tail == "useReducer": - hooks.append(HookUse("STATE", f"{name_txt}={bound(ntext(value, 120))}", line, risk)) + hooks.append(HookUse("STATE", f"{name_txt}={bound(ntext(value, 120))}", + line, risk, inherited_risk)) elif tail == "useContext": - hooks.append(HookUse("CTX", f"{name_txt}={bound(ntext(value, 80))}", line, risk)) + hooks.append(HookUse("CTX", f"{name_txt}={bound(ntext(value, 80))}", + line, risk, inherited_risk)) elif tail == "useRef": - hooks.append(HookUse("REF", name_txt, line, risk)) + hooks.append(HookUse("REF", name_txt, line, risk, inherited_risk)) else: - hooks.append(HookUse("HOOK", f"{name_txt}={bound(ntext(value, 120))}", line, risk)) + hooks.append(HookUse("HOOK", f"{name_txt}={bound(ntext(value, 120))}", + line, risk, inherited_risk)) elif stmt.type == "expression_statement" and stmt.named_children: value = _unwrap_parens(stmt.named_children[0]) if value is None: continue - if value.type in COND_EXPR_TYPES: - _fault_conditional_hooks(value, faults, ntext) + _fault_nested_conditional_hooks( + value, faults, ntext, aliases, shadowed) + if _is_conditional_expression(value): continue callee, args = _call_parts(value, ntext) if callee is None: continue - tail = callee.split(".")[-1] + tail, is_alias, is_hook = _hook_info(callee, aliases, shadowed) + if not is_hook: + continue if tail in EFFECT_HOOKS: - risk = "aliased-hook" if "." in callee else "" + risk = "aliased-hook" if is_alias else "" + inherited_risk = _node_risk(value, ntext) if risk: - faults.append((risk, line)) + _append_fault(faults, risk, line) deps = f"deps={ntext(args[1], 100)}" if len(args) >= 2 else "deps=EVERY-RENDER" ops = [] - if args and args[0].type in ("arrow_function", "function_expression"): - cb_body = args[0].child_by_field_name("body") - if cb_body is not None and cb_body.type == "statement_block": - ops = flow_fn(cb_body) - hooks.append(HookUse("EFFECT", deps, line, risk, flow=ops)) - elif HOOK_RE.match(tail): - risk = "aliased-hook" if "." in callee else "" + callback = _unwrap_parens(args[0]) if args else None + if callback is not None and callback.type in ("arrow_function", "function_expression"): + cb_body = callback.child_by_field_name("body") + if cb_body is not None: + ops = (flow_fn(cb_body) if cb_body.type == "statement_block" + else _expression_flow(cb_body, ntext, flow_fn)) + hooks.append(HookUse("EFFECT", deps, line, risk, + inherited_risk, flow=ops)) + else: + risk = "aliased-hook" if is_alias else "" + inherited_risk = _node_risk(value, ntext) if risk: - faults.append((risk, line)) - hooks.append(HookUse("HOOK", ntext(value, 120), line, risk)) + _append_fault(faults, risk, line) + hooks.append(HookUse("HOOK", ntext(value, 120), line, + risk, inherited_risk)) else: - _fault_conditional_hooks(stmt, faults, ntext) + if stmt.type in { + "if_statement", "for_statement", "for_in_statement", + "while_statement", "do_statement", "switch_statement", + "try_statement", + }: + _fault_conditional_hooks(stmt, faults, ntext, aliases, shadowed) + _fault_nested_conditional_hooks( + stmt, faults, ntext, aliases, shadowed) react["setters"] = setters -def _fault_conditional_hooks(node, faults, ntext): +def _fault_conditional_hooks(node, faults, ntext, aliases=None, shadowed=None): + if node is None: + return + aliases = aliases or {} + shadowed = shadowed or set() stack = [node] while stack: n = stack.pop() - if n.type in FN_TYPES or n.type in JSX_TYPES: + if n.type in NESTED_SCOPE_TYPES: continue if n.type == "call_expression": - callee = n.child_by_field_name("function") - txt = ntext(callee, 80) if callee is not None else "" - if HOOK_RE.match(txt.split(".")[-1] or ""): - faults.append(("conditional-hook", n.start_point[0] + 1)) + _fault_hook_call(n, faults, ntext, aliases, shadowed) stack.extend(n.named_children) +def _fault_hook_call(node, faults, ntext, aliases=None, shadowed=None): + """Fault *node* itself when it is a hook call, without scanning children.""" + if node is None or node.type != "call_expression": + return + aliases = aliases or {} + shadowed = shadowed or set() + callee = node.child_by_field_name("function") + txt = ntext(callee, 80) if callee is not None else "" + _, is_alias, is_hook = _hook_info(txt, aliases, shadowed) + if is_hook: + line = node.start_point[0] + 1 + _append_fault(faults, "conditional-hook", line) + if is_alias: + _append_fault(faults, "aliased-hook", line) + + +def _fault_nested_conditional_hooks(node, faults, ntext, aliases=None, shadowed=None): + """Find guarded hook calls even when the conditional is nested in a call, + assignment, or JSX expression, while leaving unguarded operands alone.""" + if node is None: + return + stack = [node] + while stack: + current = stack.pop() + if current.type in NESTED_SCOPE_TYPES: + continue + if current.type == "ternary_expression": + condition = current.child_by_field_name("condition") + consequence = current.child_by_field_name("consequence") + alternative = current.child_by_field_name("alternative") + _fault_conditional_hooks( + consequence, faults, ntext, aliases, shadowed) + _fault_conditional_hooks( + alternative, faults, ntext, aliases, shadowed) + if condition is not None: + stack.append(condition) + continue + if current.type in {"binary_expression", "augmented_assignment_expression"}: + operator = current.child_by_field_name("operator") + if operator is not None and operator.type in { + "&&", "||", "??", "&&=", "||=", "??=", + }: + left = current.child_by_field_name("left") + right = current.child_by_field_name("right") + _fault_conditional_hooks( + right, faults, ntext, aliases, shadowed) + if left is not None: + stack.append(left) + continue + if _is_optional_call(current): + # The optional invocation and its arguments are guarded, while + # evaluating the callee expression itself is not. + _fault_hook_call(current, faults, ntext, aliases, shadowed) + arguments = current.child_by_field_name("arguments") + _fault_conditional_hooks( + arguments, faults, ntext, aliases, shadowed) + callee = current.child_by_field_name("function") + if callee is not None: + stack.append(callee) + continue + stack.extend(current.named_children) + + RENDER_BUDGET = 200 @@ -259,20 +656,29 @@ def _jsx_name(el, ntext): def _jsx_attrs(el, ntext): - """(attrs_text, spread_only, event_attrs) from an element's opening tag. - event_attrs: list of (attr_name, value_node) pairs.""" + """(attrs_text, spread_only, event_attrs, risk) from an opening tag. + event_attrs: list of (attr_name, value_node, secret_name_hint) triples.""" opening = el if el.type == "jsx_element": opening = el.named_children[0] if el.named_children else el - parts, named, spread, events = [], 0, 0, [] + parts, named, spread, events, risk = [], 0, 0, [], "" for a in opening.named_children: if a.type == "jsx_attribute": aname_node = a.named_children[0] if a.named_children else None aname = ntext(aname_node, 40) if aname_node is not None else "" value = a.named_children[1] if len(a.named_children) > 1 else None if re.match(r"^on[A-Z]", aname) and value is not None: - events.append((aname, value)) + events.append((aname, value, bool(SECRET_NAME.search(aname)))) continue + callback = value + if callback is not None and callback.type == "jsx_expression": + callback = callback.named_children[0] if callback.named_children else None + callback = _unwrap_parens(callback) + if (callback is not None + and callback.type in {"arrow_function", "function_expression"} + and (aname == "children" or aname.startswith("render") + or _returns_jsx(callback))): + risk = "render-prop" named += 1 if SECRET_NAME.search(aname) and value is not None: parts.append(f"{aname}={sanitize_string(ntext(value, 60), secret_hint=True)}") @@ -281,7 +687,7 @@ def _jsx_attrs(el, ntext): elif a.type == "jsx_expression": # {...spread} spread += 1 parts.append(ntext(a, 40).strip("{}")) - return " ".join(parts), (spread > 0 and named == 0), events + return " ".join(parts), (spread > 0 and named == 0), events, risk def _lower_jsx(node, ntext, counter, events_out, element_name=""): @@ -303,14 +709,16 @@ def make(tag, **kw): if n.type in ("jsx_element", "jsx_self_closing_element"): name, is_comp, risk = _jsx_name(n, ntext) - attrs, spread_only, ev = _jsx_attrs(n, ntext) + attrs, spread_only, ev, attr_risk = _jsx_attrs(n, ntext) if spread_only: risk = f"{risk}+spread-props" if risk else "spread-props" + if attr_risk: + risk = f"{risk}+{attr_risk}" if risk else attr_risk rn = make(name, attrs=attrs, is_component=is_comp, risk=risk) if rn is None: return [] - for aname, value in ev: - events_out.append((name, aname, value)) + for aname, value, secret in ev: + events_out.append((name, aname, value, secret)) if n.type == "jsx_element": for ch in n.named_children[1:-1]: rn.children.extend(_lower_jsx(ch, ntext, counter, events_out, name)) @@ -333,6 +741,33 @@ def make(tag, **kw): return out +def _jsx_returns(body) -> list: + """Return ``(return_node, jsx_expression)`` pairs outside nested scopes.""" + found = [] + for node in _return_nodes(body): + for child in node.named_children: + value = _unwrap_parens(child) + if _jsx_bearing(value): + found.append((node, value)) + break + return found + + +def _return_nodes(body) -> list: + """Return all return statements outside nested function/class scopes.""" + found = [] + stack = list(body.named_children) + while stack: + node = stack.pop() + if node.type in NESTED_SCOPE_TYPES: + continue + if node.type == "return_statement": + found.append(node) + continue + stack.extend(node.named_children) + return sorted(found, key=lambda node: node.start_byte) + + def _lower_expr(n, ntext, counter, events_out, element_name): n = _unwrap_parens(n) if n is None: @@ -384,8 +819,7 @@ def make(tag, **kw): if prop is not None and ntext(prop, 20) == "map": receiver = callee.child_by_field_name("object") args = n.child_by_field_name("arguments") - cb = next((a for a in args.named_children - if a.type in ("arrow_function", "function_expression")), None) if args is not None else None + cb = _function_argument(args) if cb is not None: params = cb.child_by_field_name("parameters") single = cb.child_by_field_name("parameter") @@ -395,11 +829,28 @@ def make(tag, **kw): param_txt = ntext(params.named_children[0], 40) else: param_txt = "_" - rn = make(f"FOR {param_txt} in {ntext(receiver, 60)}", is_structure=True) + receiver_txt = ntext(receiver, 60) + if SECRET_NAME.search(param_txt): + receiver_txt = sanitize_string( + ntext(receiver, 10000), secret_hint=True) + rn = make(f"FOR {param_txt} in {receiver_txt}", is_structure=True) if rn is None: return [] body = cb.child_by_field_name("body") - rn.children = _lower_jsx(body, ntext, counter, events_out, element_name) if body is not None else [] + render_body = _unwrap_parens(body) + if render_body is not None and render_body.type == "statement_block": + returns = _jsx_returns(render_body) + all_returns = _return_nodes(render_body) + if returns: + _, render_body = returns[-1] + if (len(all_returns) > 1 + or any(ret.parent != body for ret, _ in returns)): + rn.risk = "render-control-flow" + else: + render_body = None + rn.children = (_lower_expr(render_body, ntext, counter, + events_out, element_name) + if render_body is not None else []) return [rn] elif n.type in ("arrow_function", "function_expression"): rn = make("{" + ntext(n, 60) + "}", risk="render-prop") @@ -432,8 +883,14 @@ def _event_action(value_node, setters, ntext): if callee_txt in setters: args = body.child_by_field_name("arguments") arg = args.named_children[0] if args is not None and args.named_children else None - arg_txt = ntext(arg, 60) if arg is not None else "undefined" - return f"set {setters[callee_txt]}={arg_txt}" + state = setters[callee_txt] + if arg is None: + arg_txt = "undefined" + elif SECRET_NAME.search(state): + arg_txt = sanitize_string(ntext(arg, 10000), secret_hint=True) + else: + arg_txt = ntext(arg, 60) + return f"set {state}={arg_txt}" return callee_txt return ntext(expr, 80) return ntext(expr, 60) @@ -441,12 +898,95 @@ def _event_action(value_node, setters, ntext): def _extract_events(react, ntext): setters = react.get("setters", {}) - for element, attr, value in react.pop("_raw_events", []): - action = _event_action(value, setters, ntext) + for element, attr, value, secret in react.pop("_raw_events", []): + action = (sanitize_string(ntext(value, 10000), secret_hint=True) + if secret else _event_action(value, setters, ntext)) react["events"].append(EventUse(f"{element}.{attr}", action, value.start_point[0] + 1)) +def _secret_fragments(node, setters, ntext) -> list[str]: + """Collect syntax-selected secret values that can reappear in raw RET ops.""" + selected = [] + + def add(value): + if value is None: + return + selected.append(value) + # A capped flow detail can contain only part of a compound expression; + # retain literal descendants as additional safe replacement units. + stack = list(value.named_children) + while stack: + child = stack.pop() + if child.type in {"string", "template_string"}: + selected.append(child) + continue + stack.extend(child.named_children) + + stack = [node] + while stack: + current = stack.pop() + if current.type == "jsx_attribute": + name = current.named_children[0] if current.named_children else None + value = current.named_children[1] if len(current.named_children) > 1 else None + if name is not None and SECRET_NAME.search(ntext(name, 60)): + add(value) + elif current.type == "variable_declarator": + name = current.child_by_field_name("name") + value = current.child_by_field_name("value") + if name is not None and SECRET_NAME.search(ntext(name, 100)): + add(value) + elif current.type in {"assignment_expression", "assignment_pattern", + "object_assignment_pattern"}: + left = current.child_by_field_name("left") + right = current.child_by_field_name("right") + if left is not None and SECRET_NAME.search(ntext(left, 100)): + add(right) + elif current.type == "call_expression": + callee = current.child_by_field_name("function") + callee_txt = ntext(callee, 80) if callee is not None else "" + state = setters.get(callee_txt) + if state and SECRET_NAME.search(state): + arguments = current.child_by_field_name("arguments") + if arguments is not None: + for argument in arguments.named_children: + add(argument) + if callee is not None and callee.type == "member_expression": + prop = callee.child_by_field_name("property") + if prop is not None and ntext(prop, 20) == "map": + arguments = current.child_by_field_name("arguments") + callback = _function_argument(arguments) + if callback is not None: + params = (callback.child_by_field_name("parameter") + or callback.child_by_field_name("parameters")) + if params is not None and SECRET_NAME.search(ntext(params, 100)): + add(callee.child_by_field_name("object")) + stack.extend(current.named_children) + + fragments = [] + for value in selected: + raw = one_line_text(_raw_text(value)) + if raw and raw not in fragments: + fragments.append(raw) + return sorted(fragments, key=lambda value: (-len(value), value)) + + +def _scrub_text(text: str, fragments=()) -> str: + for raw in fragments: + if raw in text: + text = text.replace(raw, sanitize_string(raw, secret_hint=True)) + return _scrub_named_secrets(text) + + +def _scrub_flow_op(op, fragments=()): + detail = op.detail + if op.binds and SECRET_NAME.search(op.binds): + detail = sanitize_string(detail, secret_hint=True) + else: + detail = _scrub_text(detail, fragments) + return replace(op, detail=detail) + + def _extract_render(node, react, ntext, events_out): body = node.child_by_field_name("body") if body is None: @@ -458,20 +998,14 @@ def _extract_render(node, react, ntext, events_out): if not _jsx_bearing(jsx_root): jsx_root = None else: - best_line = -1 - stack = list(body.named_children) - while stack: - s = stack.pop() - if s.type in FN_TYPES: - continue - if s.type == "return_statement": - for ch in s.named_children: - u = _unwrap_parens(ch) - if _jsx_bearing(u) and s.start_point[0] > best_line: - best_line = s.start_point[0] - jsx_root = u # textually last JSX-bearing return wins - else: - stack.extend(s.named_children) + returns = _jsx_returns(body) + if returns: + jsx_root = returns[-1][1] # textually last JSX-bearing return wins + all_returns = _return_nodes(body) + if (len(all_returns) > 1 + or any(ret.parent != body for ret, _ in returns)): + _append_fault(react["faults"], "render-control-flow", + returns[0][0].start_point[0] + 1) if jsx_root is None: return if jsx_root.type in JSX_TYPES: @@ -532,44 +1066,89 @@ def _kept(nodes, level): return out -def _hook_call_op(op) -> bool: +def _hook_call_op(op, aliases=None, shadowed=None) -> bool: """True for CALL flow ops whose callee is a hook: already surfaced as STATE/CTX/REF/HOOK/EFFECT lines, so they must not duplicate as flow ops.""" if op.op != "CALL": return False - callee = op.detail.split("(")[0].strip() - return bool(HOOK_RE.match(callee.split(".")[-1])) + match = re.match( + r"^\s*([A-Za-z_$][\w$]*(?:\s*\.\s*[A-Za-z_$][\w$]*)*)", + op.detail, + ) + if match is None: + return False + callee = re.sub(r"\s+", "", match.group(1)) + _, _, is_hook = _hook_info(callee, aliases or {}, shadowed or ()) + return is_hook + + +def _react_risk_entries(react, symbol=None) -> list[tuple[str, int]]: + entries = list(react.get("faults", [])) + for hook in react.get("hooks", []): + if hook.risk: + entries.append((hook.risk, hook.line)) + if hook.inherited_risk: + entries.append((hook.inherited_risk, hook.line)) + for op in hook.flow: + if op.risk: + entries.append((op.risk, op.line)) + if symbol is not None: + for op in symbol.flow: + if op.risk: + entries.append((op.risk, op.line)) + stack = list(react.get("render", [])) + while stack: + node = stack.pop() + if node.risk: + entries.append((node.risk, node.line)) + stack.extend(node.children) + return list(dict.fromkeys(entries)) def component_lines(s, level, tier, faults): handle = s.semantic8 or s.slice8 - lines = [f"COMPONENT {s.name}({s.signature}) @L{s.span[0]}-{s.span[1]} ^{handle} ~{tier}"] - if level == 1: - return lines + head = f"COMPONENT {s.name}({s.signature}) @L{s.span[0]}-{s.span[1]} ^{handle} ~{tier}" r = s.react + if level == 1: + head_risks = set() + for risk, line in _react_risk_entries(r, s): + if risk not in head_risks: + head += f" !FAULT({risk})" + head_risks.add(risk) + faults.append(f"{risk}(L{line})") + return [head] + lines = [head] + secret_fragments = r.get("secret_fragments", ()) if r.get("wrapper"): lines.append(f" WRAP {r['wrapper']}") if r.get("props"): lines.append(" PROPS " + ", ".join(r["props"])) for h in r.get("hooks", []): - tag = "" - if h.risk: - tag = f" !FAULT({h.risk})" - faults.append(f"{h.risk}(L{h.line})") + hook_risks = [risk for risk in (h.risk, h.inherited_risk) if risk] + tag = "".join(f" !FAULT({risk})" for risk in dict.fromkeys(hook_risks)) + for risk in dict.fromkeys(hook_risks): + faults.append(f"{risk}(L{h.line})") lines.append(f" {h.kind} {h.detail}{tag}") - if h.kind == "EFFECT" and level >= 3: - effect_ops = [replace(op, detail=_scrub_named_secrets(op.detail)) - for op in h.flow] - lines.extend(flow_lines(s, level, tier, faults, ops=effect_ops)) + if h.kind == "EFFECT" and h.flow: + effect_ops = [_scrub_flow_op(op, secret_fragments) for op in h.flow] + visible_effect_ops = (effect_ops if level >= 3 + else [op for op in effect_ops if op.risk]) + if visible_effect_ops: + lines.extend(flow_lines(s, level, tier, faults, ops=visible_effect_ops)) for e in r.get("events", []): - lines.append(f" EVENT {e.target} -> {_scrub_named_secrets(e.action)}") + lines.append(f" EVENT {e.target} -> {_scrub_text(e.action, secret_fragments)}") effects = _render_provenanced(s.effects, s.unknown_calls) if effects: lines.append(" EFFECTS " + effects) - if level >= 3 and s.flow: - body_ops = [replace(op, detail=_scrub_named_secrets(op.detail)) - for op in s.flow if not _hook_call_op(op)] - lines.extend(flow_lines(s, level, tier, faults, ops=body_ops)) + if level >= 2 and s.flow: + body_ops = [_scrub_flow_op(op, secret_fragments) + for op in s.flow + if not _hook_call_op( + op, r.get("hook_aliases"), r.get("shadowed_hook_aliases"))] + visible_body_ops = (body_ops if level >= 3 + else [op for op in body_ops if op.risk]) + if visible_body_ops: + lines.extend(flow_lines(s, level, tier, faults, ops=visible_body_ops)) for risk, line in r.get("faults", []): lines.append(f" !FAULT({risk}) @L{line}") faults.append(f"{risk}(L{line})") @@ -579,16 +1158,26 @@ def component_lines(s, level, tier, faults): return lines +def _inside_class(node) -> bool: + parent = node.parent + while parent is not None: + if parent.type in {"class", "class_declaration", "abstract_class_declaration"}: + return True + parent = parent.parent + return False + + def lower_components(fn_nodes, ntext, flow_fn) -> bool: upgraded = False + hook_aliases = _react_hook_aliases(fn_nodes, ntext) for sym, node in fn_nodes: short = sym.name.split(".")[-1] - if not COMPONENT_NAME_RE.match(short) or not _returns_jsx(node): + if (node.type not in COMPONENT_FN_TYPES or _inside_class(node) + or not COMPONENT_NAME_RE.match(short) + or not _returns_jsx(node)): continue sym.kind = "component" - # The head line renders the raw signature text; name-hinted string - # defaults (apiToken = "…") must not leak through it. - sym.signature = _scrub_named_secrets(sym.signature) + shadowed_hook_aliases = _shadowed_hook_aliases(node, hook_aliases) sym.react = { "wrapper": sym.decorators[0] if sym.decorators else "", "props": _extract_props(node, ntext), @@ -596,15 +1185,20 @@ def lower_components(fn_nodes, ntext, flow_fn) -> bool: "events": [], "render": [], "faults": [], + "hook_aliases": hook_aliases, + "shadowed_hook_aliases": tuple(sorted(shadowed_hook_aliases)), } body = node.child_by_field_name("body") if body is not None and body.type == "statement_block": - _extract_hooks(body, sym.react, ntext, flow_fn) + _extract_hooks(body, sym.react, ntext, flow_fn, + hook_aliases, shadowed_hook_aliases) else: sym.react["setters"] = {} raw_events = [] _extract_render(node, sym.react, ntext, raw_events) sym.react["_raw_events"] = raw_events _extract_events(sym.react, ntext) + sym.react["secret_fragments"] = _secret_fragments( + node, sym.react.get("setters", {}), ntext) upgraded = True return upgraded diff --git a/tests/test_cache_integration.py b/tests/test_cache_integration.py index d777ff6..4256df8 100644 --- a/tests/test_cache_integration.py +++ b/tests/test_cache_integration.py @@ -12,6 +12,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) import kern_cache # noqa: E402 import kern_compile # noqa: E402 +import kern_react # noqa: E402 BIG_PY = '"""Doc."""\n\nimport json\n\n' + "\n\n".join( f'def fn_{i}(path, n):\n' @@ -181,6 +182,58 @@ def test_parser_capability_version_change_invalidates_cache(self): manifest["files"]["mod.js"]["ir_compiler_fingerprint"], ) + def test_react_adapter_content_change_invalidates_tsjs_cache(self): + if not kern_compile.tsjs_available(tsx=True): + self.skipTest("tsx grammar not installed") + self.config["min_ir_tokens"] = 1 + source = "export const Card = ({ title }) =>
    {title}
    ;\n" + (self.root / "card.tsx").write_text(source) + adapter = self.root / "kern_react.py" + adapter.write_text("# adapter revision one\n") + + with mock.patch.object(kern_react, "__file__", str(adapter)): + first = self.ensure("card.tsx") + manifest = json.loads(self.paths["manifest"].read_text()) + first_fingerprint = manifest["files"]["card.tsx"]["ir_compiler_fingerprint"] + self.assertFalse(first["cache_hit"]) + self.assertEqual(first["mode"], "structured:tree-sitter+react") + self.assertTrue(self.ensure("card.tsx")["cache_hit"]) + + adapter.write_text("# adapter revision two\n") + changed = self.ensure("card.tsx") + manifest = json.loads(self.paths["manifest"].read_text()) + + self.assertFalse(changed["cache_hit"]) + self.assertNotEqual( + first_fingerprint, + manifest["files"]["card.tsx"]["ir_compiler_fingerprint"], + ) + + def test_react_adapter_fingerprint_tolerates_missing_and_unavailable_file(self): + source = self.root / "card.tsx" + source.write_text("export const Card = () =>
    ;\n") + adapter = self.root / "missing-kern-react.py" + + with mock.patch.object(kern_react, "__file__", str(adapter)): + missing = kern_cache.compiler_fingerprint( + source, self.config, "L2", "structured:tree-sitter+react" + ) + + adapter.write_text("# unreadable adapter\n") + original_sha256_file = kern_cache.sha256_file + + def reject_adapter(path): + if Path(path) == adapter: + raise PermissionError("adapter source is unreadable") + return original_sha256_file(path) + + with mock.patch.object(kern_cache, "sha256_file", side_effect=reject_adapter): + unavailable = kern_cache.compiler_fingerprint( + source, self.config, "L2", "structured:tree-sitter+react" + ) + + self.assertNotEqual(missing, unavailable) + def test_pre_fingerprint_manifest_is_proactively_invalidated(self): self.ensure("big.py") manifest = json.loads(self.paths["manifest"].read_text()) diff --git a/tests/test_react_adversarial.py b/tests/test_react_adversarial.py new file mode 100644 index 0000000..0cd4ec0 --- /dev/null +++ b/tests/test_react_adversarial.py @@ -0,0 +1,393 @@ +import sys +import unittest +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_compile # noqa: E402 + + +@unittest.skipUnless(kern_compile.tsjs_available(tsx=True), "tsx grammar not installed") +class TestReactAdversarial(unittest.TestCase): + def parse(self, source): + return kern_compile.parse_tsjs(source, tsx=True) + + def il(self, source, tier="L3"): + module = self.parse(source) + return kern_compile.emit_il( + module, "app/T.tsx", "c" * 64, "none", tier, + ) + + def test_compound_and_nested_prop_defaults_are_structurally_redacted(self): + source = ( + 'function Login({ config: { apiToken = String("prop-secret"), ' + 'label = "visible" }, password: local = make("nested-secret") }) {' + ' return
    ; }' + ) + for tier in ("L1", "L2", "L3"): + rendered = self.il(source, tier) + self.assertNotIn("prop-secret", rendered) + self.assertNotIn("nested-secret", rendered) + self.assertIn('label = "visible"', self.il(source)) + + def test_compound_default_does_not_reintroduce_secret_literal_type(self): + source = ( + 'function Login({ apiToken = String("default-secret"), password }:' + ' { apiToken?: string; password: "type-secret" }) {' + ' return
    ; }' + ) + rendered = self.il(source, "L1") + self.assertNotIn("default-secret", rendered) + self.assertNotIn("type-secret", rendered) + + def test_aliased_prop_preserves_generic_secret_redaction(self): + source = ( + 'function Login({ label: local = "sk-abcdefghijklmnop", ' + 'value: other = getSecret("plain-secret") }) {' + ' return
    ; }' + ) + rendered = self.il(source) + self.assertNotIn("sk-abcdefghijklmnop", rendered) + self.assertNotIn("plain-secret", rendered) + + def test_secret_event_attr_and_nested_setter_argument_do_not_leak(self): + source = ( + 'function T(){ const [password, setPassword] = useState("");' + ' return setPassword(make("setter-secret"))}/>; }' + ) + rendered = self.il(source) + self.assertNotIn("event-secret", rendered) + self.assertNotIn("setter-secret", rendered) + self.assertIn("EVENT Input.onToken -> set password= { const password = identity("effect-secret");' + ' save(password); }, []); return ; }' + ) + rendered = self.il(source) + self.assertNotIn("component-secret", rendered) + self.assertNotIn("effect-secret", rendered) + self.assertIn("-> apiToken", rendered) + self.assertIn("-> password", rendered) + + def test_l1_keeps_react_fault_in_head_and_footer(self): + rendered = self.il("function T(){ return ; }", "L1") + self.assertIn("COMPONENT T()", rendered) + self.assertIn("!FAULT(dynamic-component)", rendered) + self.assertIn("dynamic-component(L1)", rendered) + + def test_l2_keeps_inherited_flow_risk(self): + source = "function T({pattern}){ const re = RegExp(pattern); return ; }" + rendered = self.il(source, "L2") + self.assertIn("CALL !FAULT(regex)", rendered) + self.assertIn("regex(L1)", rendered) + + def test_l2_keeps_effect_risk_without_emitting_plain_body_flow(self): + source = ( + "function T({value}){ useEffect(() => value.match(/a+/), []);" + " return ; }" + ) + rendered = self.il(source, "L2") + self.assertIn("CALL !FAULT(regex)", rendered) + self.assertIn("regex(L1)", rendered) + self.assertNotIn("\n RET", rendered) + + def test_hook_argument_risks_survive_all_tiers(self): + source = ( + "function T({pattern}){ const [re] = useState(RegExp(pattern));" + " return ; }" + ) + for tier in ("L1", "L2", "L3"): + rendered = self.il(source, tier) + self.assertIn("!FAULT(regex)", rendered) + self.assertIn("regex(L1)", rendered) + + def test_l1_keeps_effect_body_risk(self): + source = ( + "function T({pattern}){ useEffect(() => RegExp(pattern), []);" + " return ; }" + ) + rendered = self.il(source, "L1") + self.assertIn("!FAULT(regex)", rendered) + self.assertIn("regex(L1)", rendered) + + def test_conditional_hook_in_declaration_initializer_is_faulted(self): + source = ( + "function T({ready}){ const state = ready ? useState(0) : null;" + " return ; }" + ) + rendered = self.il(source, "L1") + self.assertIn("!FAULT(conditional-hook)", rendered) + self.assertIn("conditional-hook(L1)", rendered) + + def test_nested_conditional_hook_shapes_are_faulted(self): + cases = ( + "let state; state = ready ? useState(0) : null;", + "const state = identity(ready ? useState(0) : null);", + "const view =
    {ready && useState(0)}
    ;", + ) + for body in cases: + with self.subTest(body=body): + source = f"function T({{ready}}){{ {body} return ; }}" + rendered = self.il(source, "L1") + self.assertIn("!FAULT(conditional-hook)", rendered) + + def test_renamed_react_hook_imports_are_lowered_and_faulted(self): + source = ( + 'import { useState as useS, useEffect as useE } from "react";' + " function T(){ const [value, setValue] = useS(0);" + " useE(() => tick(), []); return ; }" + ) + rendered = self.il(source) + self.assertIn("STATE value=0 !FAULT(aliased-hook)", rendered) + self.assertIn("EFFECT deps=[] !FAULT(aliased-hook)", rendered) + self.assertIn("CALL tick()", rendered) + self.assertEqual(rendered.count("CALL useS"), 0) + + def test_renamed_hook_alias_does_not_capture_member_or_shadowed_binding(self): + source = ( + 'import { useState as useS } from "react";' + " function T({useS}){ const a = obj.useS(0);" + " const b = useS(1); return ; }" + ) + rendered = self.il(source) + self.assertNotIn("STATE a=", rendered) + self.assertNotIn("STATE b=", rendered) + self.assertIn("CALL obj.useS(0)", rendered) + self.assertIn("CALL useS(1)", rendered) + + def test_generic_renamed_hook_is_not_duplicated_as_flow(self): + source = ( + 'import { useState as useS } from "react";' + " function T(){ const [value] = useS(0); return ; }" + ) + rendered = self.il(source) + self.assertIn("STATE value=0", rendered) + self.assertNotIn("CALL useS", rendered) + + def test_function_typed_generic_hook_is_not_duplicated_as_flow(self): + rendered = self.il( + "function T(){ const cb = useCallback<() => void>(() => {});" + " return ; }" + ) + self.assertIn("HOOK cb=useCallback<() => void>", rendered) + self.assertNotIn("CALL useCallback<() => void>", rendered) + + def test_logical_assignment_and_optional_hooks_are_conditional(self): + for operator in ("&&=", "||=", "??="): + with self.subTest(operator=operator): + rendered = self.il( + f"function T({{ready}}){{ ready {operator} useState(0);" + " return ; }", + "L1", + ) + self.assertIn("!FAULT(conditional-hook)", rendered) + rendered = self.il("function T(){ useMaybe?.(0); return ; }", "L1") + self.assertIn("!FAULT(conditional-hook)", rendered) + + def test_awaited_hook_call_is_not_silently_suppressed(self): + rendered = self.il( + "async function T(){ await useData(); return ; }", + "L3", + ) + self.assertIn("CALL await useData()", rendered) + + def test_unguarded_conditional_operands_are_not_faulted(self): + sources = ( + "function T(){ const value = useReady() && true; return ; }", + "function T(){ const value = useReady() ? 1 : 2; return ; }", + ("function T({registry, maybe}){ const value = " + "registry[maybe?.name].useState(0); return ; }"), + ) + for source in sources: + with self.subTest(source=source): + self.assertNotIn("conditional-hook", self.il(source, "L1")) + + def test_parentheses_terminate_optional_chain_short_circuiting(self): + source = "function T(){ (React?.hooks).useState(0); return ; }" + self.assertNotIn("conditional-hook", self.il(source, "L1")) + + def test_expression_bodied_effect_callback_is_lowered(self): + source = ( + 'function T(){ useEffect(() => analytics.track("view"), []);' + " return ; }" + ) + self.assertIn('CALL analytics.track("view")', self.il(source)) + + def test_expression_bodied_cleanup_callback_is_lowered(self): + source = ( + "function T(){ useEffect(() => () => cleanup(), []);" + " return ; }" + ) + self.assertIn("CALL cleanup()", self.il(source)) + + def test_wrapped_effect_callback_is_lowered(self): + source = ( + "function T(){ useEffect((() => tick()) as EffectCallback, []);" + " return ; }" + ) + self.assertIn("CALL tick()", self.il(source)) + + def test_block_bodied_map_callback_return_is_lowered(self): + source = ( + "function T({items}){ return
    {items.map(item => {" + " return ; })}
    ; }" + ) + rendered = self.il(source) + self.assertIn("FOR item in items", rendered) + self.assertIn("Row key={item.id}", rendered) + + def test_guarded_block_map_is_faulted(self): + source = ( + "function T({items}){ return
    {items.map(item => {" + " if (!item) return null; return ; })}
    ; }" + ) + rendered = self.il(source, "L2") + self.assertIn("FOR item in items", rendered) + self.assertIn("> Row", rendered) + self.assertIn("!FAULT(render-control-flow)", rendered) + + def test_secret_named_map_binding_redacts_receiver_and_raw_flow(self): + source = ( + 'function T(){ return
    {["map-secret"].map(' + "password => )}
    ; }" + ) + for tier in ("L2", "L3"): + rendered = self.il(source, tier) + self.assertNotIn("map-secret", rendered) + self.assertIn("FOR password in {["quartz-velvet-7319"].map(' + "((password => ) as Mapper))}
    ; }" + ) + rendered = self.il(source, "L3") + self.assertNotIn("quartz-velvet-7319", rendered) + self.assertIn("FOR password in {items.map(" + "renderItem, () => )}
    ; }" + ) + rendered = self.il(source, "L3") + self.assertNotIn("FOR _ in items", rendered) + self.assertNotIn("\n Fake", rendered) + + def test_long_structural_secret_values_are_redacted_before_text_cap(self): + secret = "quartzvelvet" * 20 + sources = ( + f'function T(){{ return ; }}', + (f'function T(){{ return
    {{["{secret}"].map(' + "apiToken => )}
    ; }"), + ) + for source in sources: + with self.subTest(source=source[:40]): + self.assertNotIn(secret[:64], self.il(source, "L3")) + + def test_ts_as_and_satisfies_wrappers_are_components(self): + module = self.parse( + "const A = () => ( as JSX.Element);" + " const B = () => satisfies ReactNode;" + ) + kinds = {symbol.name: symbol.kind for symbol in module.symbols} + self.assertEqual(kinds["A"], "component") + self.assertEqual(kinds["B"], "component") + + def test_ts_non_null_wrapper_is_component(self): + module = self.parse("const A = () => (!);") + self.assertEqual(next(symbol for symbol in module.symbols if symbol.name == "A").kind, + "component") + + def test_class_method_and_nested_generator_do_not_upgrade(self): + source = ( + "class View { Card(){ return ; } }" + " function Outer(){ const Inner = function*(){ return ; };" + " return 2; }" + ) + module = self.parse(source) + kinds = {symbol.name: symbol.kind for symbol in module.symbols} + self.assertEqual(kinds["View.Card"], "function") + self.assertEqual(kinds["Outer"], "function") + self.assertNotIn("component", kinds.values()) + + def test_generator_function_is_not_a_component(self): + module = self.parse("function* Feed(){ return ; }") + self.assertEqual(next(symbol for symbol in module.symbols if symbol.name == "Feed").kind, + "function") + + def test_secret_named_object_method_body_is_redacted(self): + source = ( + 'function T(){ return ; }' + ) + rendered = self.il(source) + self.assertNotIn("method-secret", rendered) + + def test_secret_named_method_signature_literals_are_redacted(self): + methods = { + "param-secret": 'password(value="param-secret") { return value; }', + "type-secret": 'password(): "type-secret" { return value; }', + "generic-secret": 'password() { return value; }', + } + for secret, method in methods.items(): + with self.subTest(secret=secret): + source = f"function T(){{ return ; }}" + self.assertNotIn(secret, self.il(source, "L3")) + + def test_secret_named_class_method_metadata_and_flow_are_redacted(self): + source = ( + 'class C { password(value="param-secret"): "type-secret" {' + ' throw "raise-secret"; return "body-secret"; } }' + ) + module = kern_compile.parse_tsjs(source, typescript=True) + rendered = kern_compile.emit_il( + module, "app/C.ts", "c" * 64, "none", "L3", + ) + for secret in ("param-secret", "type-secret", "raise-secret", "body-secret"): + self.assertNotIn(secret, rendered) + + def test_secret_named_type_method_signature_is_redacted(self): + secret = "quartz-velvet-7319" + source = f'type Config = {{ password(): "{secret}" }};' + module = kern_compile.parse_tsjs(source, typescript=True) + rendered = kern_compile.emit_il( + module, "app/Config.ts", "c" * 64, "none", "L3", + ) + self.assertNotIn(secret, rendered) + + def test_secret_named_method_call_metadata_is_redacted(self): + secret = "quartz-velvet-7319" + source = f'class C {{ password() {{ client["{secret}"](); }} }}' + module = kern_compile.parse_tsjs(source, typescript=True) + for tier in ("L1", "L2", "L3"): + with self.subTest(tier=tier): + rendered = kern_compile.emit_il( + module, "app/C.ts", "c" * 64, "none", tier, + ) + self.assertNotIn(secret, rendered) + + def test_render_prop_attribute_is_faulted(self): + source = "function T(){ return }/>; }" + for tier in ("L1", "L2", "L3"): + rendered = self.il(source, tier) + self.assertIn("!FAULT(render-prop)", rendered) + self.assertIn("render-prop(L1)", rendered) + + def test_guard_returns_are_explicitly_faulted(self): + source = "function T({ready}){ if (ready) return ; return ; }" + for tier in ("L1", "L2", "L3"): + rendered = self.il(source, tier) + self.assertIn("render-control-flow", rendered) + + def test_output_is_deterministic(self): + source = "function T({ok}){ return ok ? : ; }" + self.assertEqual(self.il(source), self.il(source)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_react_wrapper_regressions.py b/tests/test_react_wrapper_regressions.py new file mode 100644 index 0000000..07f36e5 --- /dev/null +++ b/tests/test_react_wrapper_regressions.py @@ -0,0 +1,131 @@ +import shutil +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "kern" / "scripts")) +import kern_cache # noqa: E402 +import kern_compile # noqa: E402 + + +@unittest.skipUnless(kern_compile.tsjs_available(tsx=True), "tsx grammar not installed") +class TestReactWrapperRegressions(unittest.TestCase): + def parse(self, source): + return kern_compile.parse_tsjs(source, tsx=True) + + def symbol(self, module, name="Row"): + return next(symbol for symbol in module.symbols if symbol.name == name) + + def test_non_jsx_wrapper_probe_preserves_const_symbol(self): + cases = { + "memo": "const Row = memo((value) => value + 1);\n", + "forwardRef": "const Row = forwardRef((props, ref) => props.value);\n", + "React.memo": "const Row = React.memo((value) => value + 1);\n", + "React.forwardRef": ( + "const Row = React.forwardRef((props, ref) => props.value);\n" + ), + } + for wrapper, source in cases.items(): + with self.subTest(wrapper=wrapper): + module = self.parse(source) + symbol = self.symbol(module) + self.assertEqual(module.frontend, "tree-sitter") + self.assertEqual(symbol.kind, "const") + self.assertEqual(symbol.detail, "=" + source.split("=", 1)[1].strip().rstrip(";")) + self.assertEqual(symbol.react, {}) + self.assertEqual(len([item for item in module.symbols if item.name == "Row"]), 1) + + def test_inline_comparator_is_part_of_component_source_identity(self): + original = ( + "const Row = memo(\n" + " ({ id }) =>
  • {id}
    • ,\n" + " (previous, next) => previous.id === next.id\n" + ");\n" + ) + changed = original.replace("previous.id === next.id", "previous.key === next.key") + + before_module = kern_compile.apply_semantic_handles(self.parse(original)) + after_module = kern_compile.apply_semantic_handles(self.parse(changed)) + before = self.symbol(before_module) + after = self.symbol(after_module) + + self.assertEqual(before.kind, "component") + self.assertEqual(before.react["wrapper"], "memo") + self.assertEqual(before.span, (1, 4)) + self.assertEqual(after.span, before.span) + self.assertNotEqual(after.slice8, before.slice8) + self.assertNotEqual(after.semantic8, before.semantic8) + + def test_declaration_keyword_is_part_of_component_source_identity(self): + original = "const\nRow = memo(() => );\n" + changed = original.replace("const", "var") + + before = self.symbol(kern_compile.apply_semantic_handles(self.parse(original))) + after = self.symbol(kern_compile.apply_semantic_handles(self.parse(changed))) + + self.assertEqual(before.span, (1, 2)) + self.assertEqual(after.span, before.span) + self.assertNotEqual(after.slice8, before.slice8) + self.assertNotEqual(after.semantic8, before.semantic8) + + def test_typed_wrapper_callbacks_are_unwrapped(self): + callbacks = { + "parenthesized": "(({ id }: P) =>
  • {id}
    • )", + "as": "((({ id }: P) =>
  • {id}
    • ) as React.FC

    )", + "satisfies": "((({ id }: P) =>

  • {id}
    • ) satisfies React.FC

    )", + "non-null": "((({ id }: P) =>

  • {id}
    • )!)", + } + for label, callback in callbacks.items(): + with self.subTest(label=label): + module = self.parse(f"const Row = memo({callback});\n") + symbol = self.symbol(module) + self.assertEqual(module.frontend, "tree-sitter+react") + self.assertEqual(symbol.kind, "component") + self.assertEqual(symbol.react["wrapper"], "memo") + + def test_typed_wrapper_result_is_unwrapped(self): + module = self.parse( + "const Row = (memo(({id}: P) =>
  • {id}
    • ) as React.FC

    );\n" + ) + symbol = self.symbol(module) + self.assertEqual(module.frontend, "tree-sitter+react") + self.assertEqual(symbol.kind, "component") + self.assertEqual(symbol.react["wrapper"], "memo") + + def test_verify_reports_stale_after_inline_comparator_change(self): + original = ( + "const Row = memo(\n" + " ({ id }) =>

  • {id}
    • ,\n" + " (previous, next) => previous.id === next.id\n" + ");\n" + ) + changed = original.replace("previous.id === next.id", "previous.key === next.key") + module = kern_compile.apply_semantic_handles(self.parse(original)) + symbol = self.symbol(module) + + root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, root, ignore_errors=True) + source = root / "component.tsx" + source.write_text(changed) + paths, _ = kern_cache.initialize(root) + relative, normalized = kern_cache.normalize_rel(root, "component.tsx") + + result = kern_cache.verify_symbol( + root, + paths, + relative, + normalized, + "Row", + symbol.semantic8, + f"L{symbol.span[0]}-{symbol.span[1]}", + ) + + self.assertEqual(result["result"], "stale") + self.assertIs(result["ok"], False) + self.assertEqual(result["reason"], "source-handle-changed") + self.assertEqual(result["current_span"], "L1-4") + + +if __name__ == "__main__": + unittest.main()