diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32fd920..62a64a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,17 @@ jobs: strategy: fail-fast: false matrix: + # Floor and ceiling, deliberately. 3.10 is the oldest CPython still + # receiving security fixes and what older LTS distros ship, so it is + # the realistic worst case for someone running this harness on the + # box they already have; 3.14 is what the reference runs use. The + # suite is stdlib-only and uses no 3.11+ syntax, so the floor costs + # nothing to hold. + # + # Keep both. The one failure this matrix has caught was a race in + # the proxy-attribution selftest that is wrong on every version -- + # 3.10's http.client keep-alive behaviour just made it likely enough + # to see. A narrower matrix would have hidden it. python-version: ["3.10", "3.14"] steps: diff --git a/.gitignore b/.gitignore index 460054b..3854b70 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,4 @@ proxy*.jsonl scoreboard.md *.bak +_private/ diff --git a/Makefile b/Makefile index a1a3cc9..162ce98 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ RUNNER = $(PYTHON) -m adherence.runner --adapter $(ADAPTER) --model $(MODEL) \ --trials $(TRIALS) $(ARMFLAGS) --out $(OUT) .PHONY: help check ci-local compile selftest schema lint table matrix report \ - analyze floors live calibrate screen mkpr mkscenarios trees probe run all clean + analyze floors live tasks design stop clean-runs calibrate screen mkpr mkscenarios trees probe run all clean help: @printf '%s\n' \ @@ -28,7 +28,14 @@ help: ' make ci-local [JOB=lint] run CI'"'"'s own steps locally, from the workflow file' \ '' \ 'viewing — read-only, never spends a GPU-second:' \ + ' make tasks what each scenario asks for and how' \ + ' it is judged (no run data)' \ + ' make design what each arm is + the ground rules' \ ' make live what is running right now, one-shot' \ + ' make stop [YES=1] stop a run, kill orphans, sweep temp' \ + ' (prints a plan unless YES=1; never' \ + ' deletes results -- rm those yourself)' \ + ' make clean-runs delete sandboxes/out-dirs from dead runs' \ ' make table [FILTER=a3/*] one-shot snapshot (NOCOLOR=1 to strip ANSI)' \ ' make matrix [FILTER=a3/*] interactive results matrix' \ ' FILES= scopes to one run (default: all of' \ @@ -84,9 +91,24 @@ lint: # --- viewing --- +## Stop a run and reclaim what it left. Never deletes results. +stop: + @$(PY) -m adherence.stoprun --out $(or $(OUT),runs/probe.jsonl) $(if $(YES),--yes,) + +tasks: + @$(PY) -m adherence.tasks + +design: + @$(PY) -m adherence.design + live: @$(PY) -m adherence.live +## Delete sandboxes and out-dirs left by finished or killed runs. +## Refuses while anything is running. +clean-runs: + @$(PY) -m adherence.live --clean + table: @FILTER="$(FILTER)" REF="$(REF)" PROXY="$(PROXY)" FILES="$(FILES)" \ $(PY) -m adherence.table $(FILTER) diff --git a/adapters/opencode.sh b/adapters/opencode.sh index 9427c59..13f9573 100755 --- a/adapters/opencode.sh +++ b/adapters/opencode.sh @@ -40,10 +40,20 @@ fi # step_finish per inference call, with per-call tokens and cache fields # (design §3.1). Verified on opencode 1.18.10. The human-readable format # carries none of that. +# Salvage mode. When a trial is killed at its deadline the harness loses +# the adapter, and with it the conversion step -- so a run that produced +# 27 MB of events and worked for 45 minutes was recorded as calls=0, +# tok=0. The runner re-invokes this script with ADH_SALVAGE=1 to convert +# whatever the stream captured, without generating anything new. +if [ "${ADH_SALVAGE:-0}" = "1" ]; then + RUN_EXIT=0 +else RUN_EXIT=0 opencode run --format json -m "$MODEL" "${AGENT_ARGS[@]}" "$(cat "$PROMPT_FILE")" \ > "$OUT/stdout.txt" 2> "$OUT/stderr.txt" || RUN_EXIT=$? +fi + if [ "$RUN_EXIT" -ne 0 ]; then echo "opencode-adapter: opencode run exited $RUN_EXIT" >&2 echo "--- stdout ---" >&2; tail -n 40 "$OUT/stdout.txt" >&2 || true diff --git a/bench/isolate.sh b/bench/isolate.sh index 197ff10..6af6d02 100755 --- a/bench/isolate.sh +++ b/bench/isolate.sh @@ -27,6 +27,32 @@ ROOT="$(cd "$HERE/.." && pwd)" BENCH_XDG="${ADH_BENCH_XDG:-$HERE/.xdg}" PORT="${ADH_PROXY_PORT:-8010}" +# The proxy records the tokens and round trips this suite exists to +# measure, and design §3.2 makes it AUTHORITATIVE over the adapter -- so +# it runs by default. It used to be opt-in behind ADH_PROXY_LOG, which +# meant the ordinary path produced numbers nothing had verified and the +# H4 agreement gate could not be checked at all for the run you were +# actually doing. +# +# The log is paired to the results file rather than shared, so a proxy log +# always belongs to exactly one run: runs/probe.jsonl -> runs/probe.proxy.jsonl. +# ADH_NO_PROXY=1 opts out (no endpoint, or deliberately measuring the +# adapter alone). +if [ -z "${ADH_PROXY_LOG:-}" ] && [ "${ADH_NO_PROXY:-0}" != "1" ]; then + _out="" + _next=0 + for _a in "$@"; do + if [ "$_next" = "1" ]; then _out="$_a"; _next=0; fi + if [ "$_a" = "--out" ]; then _next=1; fi + done + if [ -n "$_out" ]; then + ADH_PROXY_LOG="${_out%.jsonl}.proxy.jsonl" + else + ADH_PROXY_LOG="$ROOT/runs/proxy.jsonl" + fi + export ADH_PROXY_LOG +fi + mkdir -p "$BENCH_XDG/opencode" cp "$HERE/opencode-bench.json" "$BENCH_XDG/opencode/opencode.json" diff --git a/docs/EVAL.md b/docs/EVAL.md index f6e2958..dadb80b 100644 --- a/docs/EVAL.md +++ b/docs/EVAL.md @@ -429,18 +429,25 @@ absent entirely on some runs. Real spend, but not attributable to the instruction surface, so they are excluded from arm comparisons and reported separately. Stating the rule beats quietly picking a total. -## Two costs of parallelism +## The cost of parallelism -`--jobs > 1` buys throughput and spends two things: +`--jobs > 1` buys throughput and spends one thing: - **Wall-clock stops being comparable across arms.** Latency becomes a function of GPU scheduling. Contended runs are stamped `contended: true` rather than left to be compared against serial ones later. -- **Per-trial proxy attribution becomes impossible.** The proxy mark is one - piece of state and *nothing in an inference request identifies the trial* — - measured: the sandbox path appears nowhere in the request body, not even in - the system prompt. The runner skips marks and warns rather than writing an - attribution that is wrong. **Calibration runs serially.** + +~~**Per-trial proxy attribution becomes impossible.**~~ This was recorded as the +second cost and it was not a cost, it was a missing key. The premise was right — +*nothing in an inference request body identifies its trial*, measured: the +sandbox path appears nowhere in it, not even in the system prompt — but the +request **path** is ours. Each trial now routes through +`/__run//v1`, the proxy strips the prefix before forwarding, and +attribution arrives with the call instead of being read off one piece of shared +state that two concurrent trials would fight over. The H4 gate is measurable at +any `--jobs`. Verified in the selftest by driving two trials at one proxy +concurrently and asserting no call lands in the wrong bucket, and that upstream +never sees the prefix. Concurrent `opencode run` against one `XDG_DATA_HOME` also dies with `database is locked`, so each parallel trial gets its own data home. @@ -539,6 +546,96 @@ dropped with the reason logged. This was underspecified in v1. Pinning it now, before data, is the point of saying so here rather than deciding it when the first results look wrong. +## Amendment 2 — grading a feature PR without handing over its shape + +Recorded 2026-08-04, against the frozen v1 tag, **after the validation grid +and before any experiment data**. It changes how a subset of tasks is graded. +No registered claim, threshold, arm, or analysis procedure changes. + +### What the validation grid found + +Amendment 1 registered SWE-bench-style grading: the agent works from the PR's +parent, never sees the tests, and afterwards the harness applies the PR's own +`_test.go` files and runs the affected packages. + +That protocol is sound for a **bug fix**, whose tests call API that already +exists. It is not sound for a **feature PR**, whose tests call API the PR is +adding. Measured on the 210-run validation grid: + +| failure mode | scenarios | pass rates | in the [0.25, 0.80] band | +|---|---|---|---| +| tests name a symbol the fix introduces | 4 | 0%, 5%, 0%, 0% | **0 / 4** | +| tests compile; assertions disagree | 6 | 5%, 19%, 71%, 76%, 90%, 95% | 2 / 6 | + +A perfect separation, and the dominant cause of the floor risk this document +names as its biggest schedule risk. On `cli-cli-13057` the failure was +`unknown field IssueType in struct literal of type CreateOptions` — a +**compile error in the test file** — on a trial that had already passed +`diff_coverage` with 13/13 pre-edit probes inside the real diff's +directories. The agent located the work, implemented it, and failed because +it did not independently choose the maintainers' internal field name. + +That is not model capability and it is not a grader bug. It is a protocol +that asks a question the agent was never given the means to answer. + +### The repair that was rejected, and why + +The obvious fix is to state the required API surface in the prompt. **It was +rejected**, and the reason is recorded here because it is the more tempting +option: + +A symbol list *is* routing information. `CreateOptions.IssueType` names its +subsystem; `skillSearchFunc` names its module. Supplying it hands every arm a +piece of exactly what the treatment under test is supposed to supply — inside +the treatment's own channel. It would bias the primary outcome toward the +null and make a null result uninterpretable, because "the pattern did not +help" and "the control was given the answer" would be the same measurement. + +Grading at the public interface was preferred but has **no supply here**: of +34 tasks, 30 are unit-test-only and none are acceptance-test-only. + +### What is registered instead + +Tasks whose tests cannot compile against a differently-named implementation +are graded at the **command-line boundary**, against the PR's own binary: + + reference = build(merge commit) what the PR actually shipped + candidate = build(the agent's tree) what the agent produced + compare flag-for-flag at the CLI + +`gh issue create --type` is a user-facing contract. It already appears in the +PR body the agent receives, so **nothing is added to the prompt** — the +grader stops demanding an identifier it never disclosed. The oracle is the +merge commit's own compiled binary, so the expectations remain the +maintainers' and not the experimenter's, and the command path is derived +mechanically from the PR's file paths rather than chosen. + +**Assignment is by compiler, not by judgement.** `adherence.classify` checks +the PR's tests out onto the parent tree and builds them. A build error naming +an undefined identifier routes the task to the CLI grader; anything else +stays with the unit grader. The choice and the identifiers that forced it are +written into each task record. A declaration-scan heuristic was tried first +and rejected for false negatives: it missed `13675` +(`field.onSearchDone.Store undefined` — a field on an existing struct) and +`13624`, neither of which adds a top-level declaration. + +### Limits, and how results are reported + +`cli.surface` verifies that the flags the PR shipped are present and +accepted. It does **not** verify they behave correctly, so it is a weaker +signal than a passing unit test. + +Therefore: **unit-graded tasks are the primary evidence and CLI-graded tasks +are reported as a separate tier.** The two are never pooled into one pass +rate. A cost comparison conditioned on success uses the unit-graded set +unless a result is stated explicitly as spanning both. + +Verified before registration on a real trial from the validation run — one +that scored `pr.task_pass = fail` purely for naming a struct field +differently: `cli.builds` pass, `cli.surface` pass (all 17 flags the PR's own +binary advertises), `cli.extra_surface` pass. + + ## Known gaps - **E5 is not testable locally.** vLLM returns `prompt_tokens_details: null`, so diff --git a/docs/SESSION-LOG.md b/docs/SESSION-LOG.md new file mode 100644 index 0000000..a49fee7 --- /dev/null +++ b/docs/SESSION-LOG.md @@ -0,0 +1,483 @@ +# Building adherence-suite: a session log + +A narrative record of how this harness was built, what it is for, and — mostly +— what it got wrong along the way and how each error surfaced. Written while +the work was happening, so the defect accounts are first-hand rather than +reconstructed. + +The short version: **a benchmark's hardest problem is not measuring the thing. +It is noticing when you are measuring something else.** Almost every defect +below was silent, and several were directional — they would have produced a +clean, publishable, wrong answer rather than an obvious failure. + +--- + +## 1. What this is and why it exists + +`adherence-suite` is a deterministic harness for measuring agent behaviour +against a repository. No LLM sits anywhere in the scoring path; every verdict +comes from a grader that a person can read and re-run. + +It was built to answer one question, which arrived from an internal +platform-engineering discussion about +[directed-contexts](https://github.com/TGPSKI/directed-contexts) — a pattern +where a small always-loaded `AGENTS.md` "router" points an agent at bounded +context modules loaded on demand. + +The question put to the tool's author was blunt: **"do you have evals?"** + +Correctness was already enforced in Go (`contextctl`). Comparative evidence on +token spend and adherence did not exist. A second participant asked how the +pattern compares to a minimal `AGENTS.md` plus LSPs and MCPs — that question +became arm A5. + +So the primary question became: + +> Does routing an agent to bounded contexts cost less than injecting a +> monolith — in tokens, in round trips, and in subagent overhead — **at equal +> work delivered**? + +The last clause is the whole difficulty. Cost without an adjacent success rate +is not a result; it is a way of being cheap by doing less. + +--- + +## 2. The design, and what it refuses to do + +### Arms + +Six instruction surfaces, materialized from one source so they cannot drift: + +| arm | always-loaded | vs a1 | on demand | what it is for | +|---|---|---|---|---| +| a0 | 0 B | −6,518 | 0 | the floor: what does *any* instruction cost or buy | +| a1 | 6,518 B | — | 0 | the practical control — the repo's own `AGENTS.md`, recovered verbatim | +| a2 | 18,060 B | +11,542 | 0 | the scientific control — same content as a3, one file | +| a3 | 3,474 B | −3,044 | 13,850 | the pattern as shipped | +| a4 | 3,289 B | −3,229 | 13,850 | the same set delivered by spawning a subagent (E3) | +| a5 | 417 B | −6,101 | 0 | minimal `AGENTS.md` + tools | + +A2 exists because without it, a win for A3 could just mean "the generated text +is better written." A2 is content-matched to A3 by construction and the +equality is asserted byte-for-byte. + +### The rules that make the numbers mean something + +- **Falsifiers F1–F6 fixed before data**, with thresholds, and each one carries + what its death would name next. +- **`ungradeable` is not `fail`.** A harness capability gap must never be + scored as a model failure. +- **`purpose` on every record.** The registered analysis reads only rows marked + `experiment`; anything unlabelled is treated as validation, because the safe + reading of no label is not "this is real data." +- **The proxy is authoritative.** On disagreement about tokens or round trips, + the recording proxy wins over the adapter. It counts calls by construction: a + round trip is a request it handled. +- **Cost is meaningless unconditional.** No cost number is reportable without + the pass rate beside it. +- **Deviations are disclosed, not forbidden.** Undisclosed deviation is the only + thing actually prohibited. + +Registered at tag `prereg-directed-contexts-v1` on a repository whose ruleset +makes `prereg-*` tags immutable and signature-required. The first attempt at +that tag was **not** actually immutable — the ruleset pattern did not match a +nested tag path, and a tamper test overwrote the registration message. Flattened +and re-verified: both delete and force-move are now refused. + +--- + +## 3. The fixture, and the supply problem + +`cli/cli` (the GitHub CLI), pinned at `e83adbc0`. Chosen against criteria fixed +before screening: MIT, post-cutoff PR supply, multiple subsystems, ships its own +`AGENTS.md` (so A1 is *recovered*, not authored) and a `CODEOWNERS`. + +Three things about the fixture were stated up front as reasons the eval could +come out uninformative: it is one fixture so E4 is out of reach; it is a +heavily-starred Go repo so the codebase is familiar to the model even where a +specific PR is not; and the floor risk is real. + +**The supply collapsed twice under inspection.** + +The E1 screen counted 174 post-cutoff merges. Classifying them by what they +actually change: + +| bucket | n | share | +|---|---|---| +| CI / agent-config (`.github/`, workflows, skill files) | 83 | 47.7% | +| dependency bumps | 42 | 24.1% | +| **Go change with its own tests** | **32** | **18.4%** | +| Go change, no tests | 13 | 7.5% | +| docs / other | 4 | 2.3% | + +Nearly half of this repository's recent history is agentic-workflow +configuration, which is a fact about `cli/cli` in 2026 rather than a defect. + +Then extraction reported **3 usable tasks**. The cause was a harness bug, not +the fixture: the Go module cache was warmed at `HEAD` while tasks run at their +PR's parent commit, and 125 lines of `go.mod`/`go.sum` drift meant almost every +task failed to build offline. Warming per-commit took it to **34**. + +That number was one command away from being published as the fixture's supply. + +--- + +## 4. Validation: the runs that were never meant to be results + +### The n=7 single-task run + +42 runs across all six arms on one scenario. Useful mainly for shaking out +plumbing, and for one exchange worth recording: a token ratio was narrated as a +verdict without the pass rate beside it — violating the suite's own rule — and +the pass figure quoted alongside it was wrong as well. The correction is why +`tok_won` and the Pareto plane exist on every surface now. + +### The 210-run validation grid + +10 tasks × 3 arms × 7 trials. Explicitly **not** experimental data — the framing +was stated at the time: + +> *"these aren't experimental runs, this is validating the method, the process, +> the code, and the results, before we actually run the experiment."* + +That sentence produced the `purpose` field. Nothing on disk distinguished a +shakedown from real data — same shape, same directory, separated only by +someone's memory of which file was which. + +**What the grid actually found:** + +Only **2 of 10 scenarios** landed in the registered calibration band +[0.25, 0.80]. Six floored, two hit the ceiling. `MIN_PAIRED=4` was not met, so +the registered analysis correctly refused to produce verdicts. + +The diagnosis mattered more than the count: **agents located the right files +95–100% of the time and then failed the maintainers' tests.** Routing was not +the bottleneck. That has a direct implication for the eval's premise — with this +model on this fixture, `task_pass` is dominated by fix quality, so a routing +treatment has little room to move correctness and the differences must appear in +cost. + +--- + +## 5. The defect catalogue + +Grouped by what they would have done to a published result. This is the part of +the log worth reading. + +### 5.1 Defects that would have produced a wrong answer, quietly + +**Orphaned processes → a compounding, invisible slowdown.** +`subprocess.run(timeout=)` kills the process it started. The adapter is a shell +script, so the thing holding the GPU is a *grandchild*, and it survived every +timeout — reparented to systemd, running to completion, output read by nobody. +Measured live: two orphans still going twenty minutes later against three live +trials. Five processes contending for three slots. Not a leak but a spiral — +each timeout permanently shrinks the pool, so more trials time out. Fixed with +process-group kills, `ADH_RUN_ID` attribution via `/proc//environ`, atexit +and signal reapers, and a startup refusal when unowned harness processes exist. + +**A wall-clock deadline that could not tell a hung run from a working one.** +Trials were being killed at 900s while 110 calls deep and still advancing, and +recorded as `calls=0` because the transcript is written at the end. The results +file said those runs did nothing; they had done more than any run that finished. +Split into `--idle-timeout` (no stream activity: stuck) and `--timeout` (a +generous ceiling). + +**A delegating trial read as a stalled one — twice.** +A subagent runs in its own opencode session and the root stream carries none of +its events, so a trial that dispatches one and waits is silent by construction. +Observed: parent 0 calls, subagent 5 calls and 206,689 tokens, forty tool calls +deep, stream untouched for 3m27s — reported "stalled", and `--idle-timeout 300` +would have killed it. **It would have done that more to the arms that route to +subagents, which are the arms under test.** + +The first fix added opencode's session store to the activity check. That was the +right idea and the wrong file: the store runs SQLite in **WAL mode**, so live +writes land in `opencode.db-wal` and the main `.db` mtime only moves on +checkpoint. Measured 282s stale on the main file while `-wal` was 29s fresh. The +half-fix made it a coin flip against checkpoint timing — and it was caught only +because the TUI flickered. + +**Subagent cost read as zero.** +The live view counted distinct session ids in the root stream, which is always +1. A trial that had dispatched two `explore` agents showed `sub=0`. Reading +totals from the root session under-reports by 3.1×, measured on s13 — and **E3 +is the claim that subagent handoff is "0-cost"**, so a root-only total would +have confirmed it by construction. Child sessions and their tokens are now read +live from the store; totals are parent + children, and the split is displayed +because E3 is a claim about the child half specifically. + +**Results buffered and lost.** +`ThreadPoolExecutor.map` yields in *submission* order, so a single slow trial +buffered every later result inside the executor. Caught live: a trial completed, +its out-dir was cleaned up, and its row sat unwritten behind two trials still 6+ +minutes from finishing — invisible to the live view, missing from the count, and +**lost outright if the runner were killed.** + +**A liveness check that killed the process it checked.** +`os.kill(pid, 0)` is the standard existence probe on POSIX. On Windows +`signal.CTRL_C_EVENT == 0`, so the call delivers a real Ctrl-C. The selftest +wrote a marker carrying its own pid, checked it, and raised `KeyboardInterrupt` +inside an unrelated subprocess several tests later — sixteen seconds into the +job, with a traceback pointing at code that had nothing to do with it. CI was +red on exactly one platform. + +### 5.2 Verdicts from tests that never ran + +**F3 and F6 reported TRIPPED off a `nan` CI** with k=1. A verdict from a test +that never ran reads as evidence, which is worse than no result. + +**F3 reported `TRIPPED, k=4` while silently discarding 6 of 10 scenarios** — +every one dropped because neither arm ever succeeded on it, so the survivors +were exactly the tasks the model could already do. `paired_log_ratios`' own +docstring says callers must surface `dropped`; F1 did, F2/F3/F6 bound the +variable and never read it. Six of ten is also past F1's own refusal threshold. + +**Exclusion criteria 1 and 2 were registered and unimplemented.** An adapter +fault was graded `fail` — indistinguishable downstream from a model that failed +the task — and schema errors went to stderr and never reached the record, so +criterion 2 could not read its own evidence. Writing an exclusion rule down is +not the same as having one. + +### 5.3 Metrics that overclaimed + +**`redundant_reads` counted re-read-after-edit.** The naive version scored as +waste the single most correct thing an agent does: re-reading a file it just +changed. + +**`probes_to_first_edit` was gameable** and counted steps rather than bytes. +Replaced with a family — `probes_total`, `probes_after_first_edit`, +`probe_bytes`, `probe_trail` — and route correctness became measurable rather +than asserted. + +**Per-arm floors were never measured.** `tok_in_marginal = tok_in_billed − +floor × calls`, and exclusion criterion 4 refuses the whole comparison when the +floor is wrong — but nothing computed one. The number was going to be typed in +by hand. `make floors` derives it from `first_call_input`, where the task prompt +is byte-identical across arms so the call-1 difference is the instruction +surface and nothing else. The cross-check is the point: a measured floor must +track the on-disk surface at one constant bytes-per-token rate. A2 and A3 +independently imply 3.72 and 3.73 B/tok — agreement to 0.4%. + +### 5.4 Arms and fixtures that were not what they claimed + +**A1 was the router I generated, not the maintainers' file.** Generation +overwrites `AGENTS.md`, so the "practical control" had silently become the +treatment's own text. Fixed with `--a1-ref` recovery and a refusal that inspects +for generated markers. + +**A4 was byte-identical to A3.** The spawn arm — the one E3 depends on — was a +copy of the inline arm, so E3's question would have been answered by an arm that +never spawns. + +**`.adh-task.json` was reported as an agent edit.** The runner writes it, then +rewrites it after the agent stops to hand metrics to the grader — which turned +it into a modified tracked file. Observed as *"agent touched +['.adh-task.json']"* on a run that touched nothing. + +### 5.5 The two findings that changed the experiment + +**Feature PRs are unpassable by construction.** + +SWE-bench-style grading applies the PR's own tests to the agent's independent +implementation. That is sound for a bug fix, whose tests call API that already +exists. It is not sound for a feature PR, whose tests call API the PR is adding: + +``` +create_test.go:695:5: unknown field IssueType in struct literal of type CreateOptions +``` + +A **compile error in the test file** — on a trial that had already passed +`diff_coverage` with 13/13 pre-edit probes inside the real diff's directories. +The agent located the work, implemented it, and failed because it did not +independently choose the maintainers' internal field name. + +The separation on the validation grid was perfect: + +| class | scenarios | pass rates | in band | +|---|---|---|---| +| tests name a symbol the fix introduces | 4 | 0%, 5%, 0%, 0% | **0/4** | +| tests compile; assertions disagree | 6 | 5%, 19%, 71%, 76%, 90%, 95% | 2/6 | + +The tempting repair — state the required API surface in the prompt — **was +rejected**, and the reasoning is the most important paragraph in this log: a +symbol list *is* routing information. `CreateOptions.IssueType` names its +subsystem. Supplying it hands every arm a piece of exactly what the treatment is +supposed to supply, biasing the primary outcome toward the null *inside the +treatment's own channel*, and making a null result uninterpretable. + +Grading at the public interface was preferred but has no supply here: of 34 +tasks, 30 are unit-test-only and none acceptance-only. + +**Amendment 2** registers CLI-boundary grading instead. The oracle is the PR's +own binary: + +``` +reference = build(merge commit) what the PR actually shipped +candidate = build(the agent's tree) what the agent produced +compare flag-for-flag at the CLI +``` + +`gh issue create --type` is a user-facing contract already present in the PR +body the agent receives, so **nothing is added to the prompt** — the grader +stops demanding an identifier it never disclosed. Assignment is by compiler: +`adherence.classify` checks each PR's tests onto its parent tree and builds +them. Result across 34 tasks: **16 unit-graded, 18 cli-graded.** + +A declaration-scan heuristic was tried first and rejected for false negatives — +it missed `13675` (a field on an existing struct) and `13624`, neither of which +adds a top-level declaration. + +**The prompt never asked for anything.** + +The deepest defect, found last. `mkpr` uses the PR body as the prompt, and a PR +body is not reliably a task. Bodies describing the *solution* read as changelog +entries. The model replied, verbatim: + +> *"I see you've shared a PR description about migrating GitHub database IDs +> from `int` to `int64`. **What would you like me to do with this?**"* + +One inference call, zero tool calls, six seconds. That accounted for **8 of the +14 abandoned trials** in the validation grid. Bodies describing a *problem* +("`gh pr checks` prints a blank summary when…") were attempted normally. + +It was arm-dependent — **a1 12%, a2 1%** — so the instruction surface was being +credited for whether the agent guessed the prompt was a task at all. A confound +sitting directly on the primary outcome, flattering the quitting arm twice over: +a refusal costs one call, so a1's median token count read **23% below** the work +it actually did, and **467× below** on `a1/cli-cli-13403`, where four of seven +trials refused: + +``` + 12,547 ABANDONED + 12,554 ABANDONED + 12,554 ABANDONED + 12,556 ABANDONED +2,725,240 worked +5,869,686 worked +6,393,551 worked +``` + +The fix is a fixed imperative on every prompt, naming no file, package, symbol +or API, identical for every arm. Verified by reproducing the failure and then +removing it — same scenario, same arm, same model: + +| | calls | tools | tokens | behaviour | +|---|---|---|---|---| +| before | 1 | 0 | 12,553 | asks what to do | +| after | 21 | 45 | 1,180,052 | editing files | + +--- + +## 6. The TUI, and why it was the best debugging tool built + +A results matrix was adapted from `leather`'s sig-triage eval. It grew into a +live dashboard, and the pattern that emerged is the single most useful lesson of +the session: + +> **Every display feature requested surfaced a real defect in the measurement +> path.** Not coincidence — putting a number on screen forces the question "is +> that true?", and each time the answer was no. + +| what was asked for | what it exposed | +|---|---| +| see runs in progress | orphaned processes compounding into a death spiral | +| show subagent counts | root-only totals that would have confirmed E3 by construction | +| why does this say stalled | an idle timeout that would preferentially kill the treatment | +| why is this task 0/3 | grading that demanded identifiers never given to the agent | +| why is this already graded | completed results buffered, invisible, and lost on kill | +| where do these numbers come from | a WAL-mode staleness check watching the wrong file | + +Three of those are *directional* — they would have biased toward a specific +conclusion rather than adding noise. Noise you notice. + +The dashboard now carries: a live section (per-trial state, calls, tools, +subagent sessions, tokens, elapsed, budget-against-deadline, current tool), a +graded section with full check evidence, a per-(arm, scenario) rollup with +median/p90 and abandonment, an activity feed reading tool *output* from the +session store, and two reference tabs — `tasks` (what each scenario asks and how +it is judged) and `design` (what each arm is, measured off disk, plus the ground +rules) — separated by a divider from the run-data views. + +Its own crop of bugs, all found by use: a cursor that walked off the end of the +list and opened a detail pane for a row never on screen; a positional cursor +that let new events replace the one being read; a phantom nested level that took +two backs to leave; sections silently truncated to what fit; `q` meaning both +"back" and "quit"; sorts that belonged to a different tab; a progress bar that +rendered as an ellipsis; JSON collapsed into one unreadable line. + +--- + +## 7. Current status + +**Branch** `harden/live-tui`, 26 commits, PR #1 open and deliberately unlabelled +so the cross-platform matrix stays out until the stack is stable. + +| | | +|---|---| +| selftest | **24/24**, several checks encoding specific past failures | +| lint / `ci-local` | green | +| proxy-vs-adapter agreement | **0.000%**, live, under `--jobs 3` | +| per-trial proxy attribution in parallel | working — previously declared impossible | +| task supply | 34 verified, 16 unit-graded / 18 cli-graded | +| per-arm floors | measured, arms agree on 3.73 B/tok to 0.4% | +| registration | `prereg-directed-contexts-v1`, immutable, + Amendments 1 and 2 | +| validation run | 120 trials in flight — the first end-to-end test of the whole stack | + +### Known gaps, unfixed and disclosed + +- **A0, A4 and A5 have never run on a PR task.** A4 is the arm E3 depends on and + has never been observed spawning anything. +- **`cli.surface` verifies flags exist and are accepted, not that they work.** + Weaker than a passing unit test, which is why Amendment 2 reports CLI-graded + tasks as a separate tier, never pooled. +- **E5 / F2 (prompt caching) is dead locally.** vLLM returns + `prompt_tokens_details: null`, so `tok_effective` degenerates. Needs a metered + API or the claim leaves the set. +- **Cost profile may not be affordable.** `cli-cli-13057` burned 18M tokens and + 23 minutes per trial. The registered grid at 34 tasks × 6 arms × 7 trials + needs computing from real numbers before it is committed to. +- **A2 section ordering** is fixed across trials; the registration wants + per-trial re-seeding. +- **Compaction events** are parsed but never observed. + +--- + +## 8. Meta-highlights + +**The suite tests itself more rigorously than it tests anything else.** 24 +selftest checks, several written to encode a specific past failure so it cannot +return: the harness-fault-is-not-a-model-failure check, the live cursor +anchoring check, the proxy attribution-under-concurrency check, the +process-hygiene check that kills a grandchild and verifies it died. + +**It caught its author twice.** Changing the budget semantics broke a selftest +assertion that had been correct against the old contract — the suite noticed +before CI did. And a stray-process guard now refuses to start a run when unowned +harness processes are alive, which is a rule written after watching those +orphans distort a run. + +**Refusing beats guessing, everywhere.** `analyze` reports NOT TESTABLE rather +than "not tripped". `floors` exits non-zero rather than print a number it cannot +defend. The live view shows arm `?` rather than infer one. `make probe` refuses +when the arm cannot be applied. `mkscenarios` refuses unverified tasks. The +registered analysis refuses rows not marked `experiment`. Every one of those +refusals exists because the alternative once produced a confident wrong answer. + +**The most dangerous defects were all silent and most were directional.** A +harness that kills the treatment for delegating, a metric that credits it for +not spawning, a grader that fails it for naming things differently, a prompt +that makes the control quit and look cheap — none of these announce themselves. +They produce a clean result with a plausible story. + +**Destructive convenience was deliberately declined.** A one-command +reset-and-clear was built and then split: `make stop` reclaims processes and +sweeps temp directories but never deletes results, and prints the `rm` for you +to type. Bundling "kill the processes" with "remove the data" is how a habit +formed for the safe purpose performs the destructive one, and a `--force` flag +exists to be used. + +**What the whole exercise is really about.** The eval has not run. What exists +is a harness that has been argued with, caught out, and corrected roughly thirty +times — and the argument is the artifact. A number produced by an instrument +nobody tried to break is not evidence. diff --git a/docs/VALIDATION-120-make-probe-sh.txt b/docs/VALIDATION-120-make-probe-sh.txt new file mode 100644 index 0000000..7b1de3a --- /dev/null +++ b/docs/VALIDATION-120-make-probe-sh.txt @@ -0,0 +1,163 @@ +tyler@ego-killer ~/git/TGPSKI/adherence-suite $ make probe ARMSDIR=fixtures/cli-cli.arms JOBS=3 TIMEOUT=2700 IDLE=300 \ + ONLY=cli-cli-13057,cli-cli-13068,cli-cli-13362,cli-cli-13412,cli-cli-13413,cli-cli-13418,cli-cli-13421,cli-cli-13430,cli-cli-13449,cli-cli-13459,cli-cli-13471,cli-cli-13523,cli-cli-13541,cli-cli-13548,cli-cli-13580,cli-cli-13620,cli-cli-13681,cli-cli-13722,cli-cli-13723,cli-cli-13807,cli-cli-13823,cli-cli-13864,cli-cli-13987,cli-cli-13988 +mkdir -p runs/ +bench/isolate.sh python3 -m adherence.runner --suite suite-pr.yaml \ + --adapter adapters/opencode.sh --model local/qwen36-35b-a3b-nvfp4 --trials 5 \ + --arm a1 --out runs/probe.jsonl \ + --arms-dir fixtures/cli-cli.arms \ + --jobs 3 --timeout 2700 \ + --idle-timeout 300 \ + --only cli-cli-13057,cli-cli-13068,cli-cli-13362,cli-cli-13412,cli-cli-13413,cli-cli-13418,cli-cli-13421,cli-cli-13430,cli-cli-13449,cli-cli-13459,cli-cli-13471,cli-cli-13523,cli-cli-13541,cli-cli-13548,cli-cli-13580,cli-cli-13620,cli-cli-13681,cli-cli-13722,cli-cli-13723,cli-cli-13807,cli-cli-13823,cli-cli-13864,cli-cli-13987,cli-cli-13988 +proxy: 127.0.0.1:8010 -> http://127.0.0.1:8000 log=runs/probe.proxy.jsonl +isolate: recording every inference call to runs/probe.proxy.jsonl +note: --jobs>1 with a recording proxy. Trial marks are skipped (they are one piece of shared state and two trials would interleave), but each trial routes through its own /__run/ path, so per-trial attribution is exact and the H4 gate is measurable in parallel. +cli-cli-13057 arm=a1 trial=1 all_pass=False dur=2701.0s calls=0 tok_in=0 +cli-cli-13057 arm=a1 trial=2 all_pass=False dur=2701.0s calls=0 tok_in=0 +cli-cli-13057 arm=a1 trial=0 all_pass=False dur=2701.0s calls=0 tok_in=0 +cli-cli-13068 arm=a1 trial=0 all_pass=True dur=404.3s calls=26 tok_in=894535 +cli-cli-13068 arm=a1 trial=1 all_pass=True dur=303.0s calls=62 tok_in=2898750 +cli-cli-13068 arm=a1 trial=2 all_pass=True dur=244.0s calls=35 tok_in=1383795 +cli-cli-13057 arm=a1 trial=4 all_pass=True dur=1166.8s calls=30 tok_in=2657861 +cli-cli-13068 arm=a1 trial=3 all_pass=True dur=273.4s calls=47 tok_in=2420639 +cli-cli-13362 arm=a1 trial=0 all_pass=True dur=97.1s calls=29 tok_in=1049417 +cli-cli-13068 arm=a1 trial=4 all_pass=True dur=189.8s calls=35 tok_in=1272343 +cli-cli-13362 arm=a1 trial=1 all_pass=True dur=105.8s calls=30 tok_in=1293304 +cli-cli-13362 arm=a1 trial=2 all_pass=True dur=108.0s calls=30 tok_in=1160735 +cli-cli-13362 arm=a1 trial=3 all_pass=False dur=37.2s calls=11 tok_in=237090 FAILED:['pr.diff_coverage', 'pr.task_pass'] +cli-cli-13057 arm=a1 trial=3 all_pass=True dur=1482.4s calls=85 tok_in=6982703 +cli-cli-13362 arm=a1 trial=4 all_pass=False dur=25.6s calls=8 tok_in=184949 FAILED:['pr.diff_coverage', 'pr.task_pass'] +cli-cli-13412 arm=a1 trial=0 all_pass=True dur=71.7s calls=20 tok_in=550371 +cli-cli-13412 arm=a1 trial=2 all_pass=True dur=60.0s calls=20 tok_in=512865 +cli-cli-13412 arm=a1 trial=1 all_pass=True dur=91.5s calls=26 tok_in=842463 +cli-cli-13413 arm=a1 trial=0 all_pass=True dur=45.6s calls=12 tok_in=245068 +cli-cli-13412 arm=a1 trial=4 all_pass=True dur=82.0s calls=24 tok_in=612347 +cli-cli-13412 arm=a1 trial=3 all_pass=True dur=89.2s calls=28 tok_in=767078 +cli-cli-13413 arm=a1 trial=1 all_pass=True dur=29.7s calls=6 tok_in=172849 +cli-cli-13413 arm=a1 trial=4 all_pass=True dur=29.4s calls=7 tok_in=156997 +cli-cli-13413 arm=a1 trial=3 all_pass=True dur=44.7s calls=11 tok_in=322022 +cli-cli-13413 arm=a1 trial=2 all_pass=True dur=82.0s calls=20 tok_in=419035 +cli-cli-13418 arm=a1 trial=0 all_pass=True dur=320.2s calls=60 tok_in=5670343 +cli-cli-13418 arm=a1 trial=1 all_pass=True dur=390.8s calls=75 tok_in=5475968 +cli-cli-13418 arm=a1 trial=2 all_pass=True dur=529.4s calls=68 tok_in=6032990 +cli-cli-13421 arm=a1 trial=0 all_pass=True dur=91.0s calls=22 tok_in=1214383 +cli-cli-13421 arm=a1 trial=1 all_pass=True dur=44.0s calls=12 tok_in=315729 +cli-cli-13418 arm=a1 trial=3 all_pass=True dur=441.3s calls=73 tok_in=4941788 +cli-cli-13421 arm=a1 trial=2 all_pass=True dur=103.6s calls=20 tok_in=496254 +cli-cli-13418 arm=a1 trial=4 all_pass=True dur=440.7s calls=101 tok_in=8838701 +cli-cli-13421 arm=a1 trial=3 all_pass=True dur=104.7s calls=24 tok_in=1099840 +cli-cli-13430 arm=a1 trial=0 all_pass=True dur=72.7s calls=15 tok_in=393772 +cli-cli-13421 arm=a1 trial=4 all_pass=True dur=96.0s calls=16 tok_in=655945 +cli-cli-13430 arm=a1 trial=1 all_pass=True dur=117.8s calls=30 tok_in=992842 +cli-cli-13430 arm=a1 trial=2 all_pass=True dur=88.2s calls=21 tok_in=508089 +cli-cli-13430 arm=a1 trial=4 all_pass=True dur=34.7s calls=8 tok_in=215432 +cli-cli-13430 arm=a1 trial=3 all_pass=True dur=119.9s calls=16 tok_in=472160 +cli-cli-13449 arm=a1 trial=2 all_pass=True dur=175.9s calls=15 tok_in=634758 +cli-cli-13449 arm=a1 trial=0 all_pass=True dur=648.4s calls=88 tok_in=6988228 +cli-cli-13449 arm=a1 trial=1 all_pass=True dur=752.8s calls=96 tok_in=7704365 +cli-cli-13449 arm=a1 trial=3 all_pass=True dur=855.2s calls=96 tok_in=9068772 +cli-cli-13459 arm=a1 trial=0 all_pass=False dur=386.6s calls=51 tok_in=2973607 FAILED:['cli.surface'] +cli-cli-13459 arm=a1 trial=1 all_pass=False dur=166.0s calls=25 tok_in=3162088 FAILED:['cli.surface'] +cli-cli-13459 arm=a1 trial=3 all_pass=False dur=319.5s calls=56 tok_in=2478276 FAILED:['cli.surface'] +cli-cli-13449 arm=a1 trial=4 all_pass=True dur=1083.5s calls=98 tok_in=10081404 +cli-cli-13459 arm=a1 trial=2 all_pass=False dur=632.8s calls=84 tok_in=5241689 FAILED:['cli.surface'] +cli-cli-13471 arm=a1 trial=0 all_pass=True dur=109.1s calls=22 tok_in=1621232 +cli-cli-13471 arm=a1 trial=1 all_pass=True dur=94.6s calls=24 tok_in=1780156 +cli-cli-13471 arm=a1 trial=2 all_pass=True dur=119.1s calls=25 tok_in=1856871 +cli-cli-13471 arm=a1 trial=3 all_pass=True dur=101.8s calls=19 tok_in=1368389 +cli-cli-13471 arm=a1 trial=4 all_pass=True dur=206.9s calls=43 tok_in=3798524 +cli-cli-13523 arm=a1 trial=0 all_pass=True dur=201.0s calls=32 tok_in=1494979 +cli-cli-13459 arm=a1 trial=4 all_pass=False dur=687.1s calls=72 tok_in=6690945 FAILED:['cli.surface'] +cli-cli-13523 arm=a1 trial=1 all_pass=True dur=116.8s calls=14 tok_in=1136913 +cli-cli-13523 arm=a1 trial=2 all_pass=True dur=137.3s calls=29 tok_in=2441263 +cli-cli-13523 arm=a1 trial=3 all_pass=True dur=156.8s calls=22 tok_in=993191 +cli-cli-13523 arm=a1 trial=4 all_pass=True dur=125.8s calls=21 tok_in=618300 +cli-cli-13541 arm=a1 trial=0 all_pass=True dur=741.8s calls=136 tok_in=14587499 +cli-cli-13541 arm=a1 trial=2 all_pass=True dur=1057.4s calls=171 tok_in=21189386 +cli-cli-13541 arm=a1 trial=4 all_pass=True dur=143.2s calls=46 tok_in=2878231 +cli-cli-13541 arm=a1 trial=1 all_pass=True dur=1257.2s calls=149 tok_in=18833119 +cli-cli-13548 arm=a1 trial=1 all_pass=False dur=297.1s calls=70 tok_in=4367467 FAILED:['pr.task_pass'] +cli-cli-13548 arm=a1 trial=0 all_pass=True dur=341.4s calls=71 tok_in=6027213 +cli-cli-13541 arm=a1 trial=3 all_pass=True dur=1030.5s calls=166 tok_in=22243133 +cli-cli-13548 arm=a1 trial=3 all_pass=True dur=349.1s calls=81 tok_in=4759703 +cli-cli-13548 arm=a1 trial=4 all_pass=True dur=242.4s calls=48 tok_in=3784065 +cli-cli-13548 arm=a1 trial=2 all_pass=True dur=408.4s calls=82 tok_in=5319810 +cli-cli-13580 arm=a1 trial=0 all_pass=True dur=506.7s calls=94 tok_in=7448490 +cli-cli-13580 arm=a1 trial=1 all_pass=True dur=536.9s calls=92 tok_in=6829563 +cli-cli-13580 arm=a1 trial=2 all_pass=True dur=644.2s calls=122 tok_in=9196758 +cli-cli-13580 arm=a1 trial=4 all_pass=True dur=461.0s calls=94 tok_in=8526204 +cli-cli-13580 arm=a1 trial=3 all_pass=True dur=541.4s calls=93 tok_in=5145390 +cli-cli-13620 arm=a1 trial=0 all_pass=True dur=582.4s calls=106 tok_in=10671217 +cli-cli-13620 arm=a1 trial=2 all_pass=True dur=671.7s calls=122 tok_in=9581448 +cli-cli-13620 arm=a1 trial=3 all_pass=True dur=519.3s calls=77 tok_in=6326413 +cli-cli-13681 arm=a1 trial=0 all_pass=True dur=35.0s calls=7 tok_in=140826 +cli-cli-13620 arm=a1 trial=1 all_pass=True dur=831.7s calls=114 tok_in=14318005 +cli-cli-13681 arm=a1 trial=2 all_pass=True dur=27.8s calls=9 tok_in=192142 +cli-cli-13681 arm=a1 trial=1 all_pass=True dur=88.9s calls=14 tok_in=215045 +cli-cli-13681 arm=a1 trial=3 all_pass=True dur=83.9s calls=12 tok_in=293425 +cli-cli-13681 arm=a1 trial=4 all_pass=True dur=67.9s calls=14 tok_in=317490 +cli-cli-13722 arm=a1 trial=1 all_pass=True dur=81.5s calls=14 tok_in=282432 +cli-cli-13722 arm=a1 trial=0 all_pass=True dur=145.5s calls=34 tok_in=1179386 +cli-cli-13722 arm=a1 trial=2 all_pass=True dur=146.2s calls=20 tok_in=514474 +cli-cli-13620 arm=a1 trial=4 all_pass=True dur=517.3s calls=113 tok_in=12521195 +cli-cli-13722 arm=a1 trial=4 all_pass=True dur=81.0s calls=12 tok_in=245366 +cli-cli-13723 arm=a1 trial=0 all_pass=False dur=79.2s calls=21 tok_in=906852 FAILED:['pr.task_pass'] +cli-cli-13722 arm=a1 trial=3 all_pass=False dur=236.4s calls=20 tok_in=699433 FAILED:['pr.task_pass'] +cli-cli-13723 arm=a1 trial=1 all_pass=False dur=114.8s calls=34 tok_in=1541929 FAILED:['pr.task_pass'] +cli-cli-13723 arm=a1 trial=2 all_pass=True dur=427.2s calls=55 tok_in=3828896 +cli-cli-13723 arm=a1 trial=4 all_pass=False dur=331.2s calls=68 tok_in=4348326 FAILED:['pr.task_pass'] +cli-cli-13723 arm=a1 trial=3 all_pass=True dur=388.7s calls=68 tok_in=4560597 +cli-cli-13807 arm=a1 trial=0 all_pass=False dur=441.8s calls=45 tok_in=2428552 FAILED:['cli.builds'] +cli-cli-13807 arm=a1 trial=2 all_pass=True dur=484.8s calls=41 tok_in=2933541 +cli-cli-13807 arm=a1 trial=1 all_pass=True dur=683.6s calls=71 tok_in=6177076 +cli-cli-13823 arm=a1 trial=0 all_pass=True dur=210.0s calls=52 tok_in=2487856 +cli-cli-13807 arm=a1 trial=3 all_pass=True dur=560.9s calls=70 tok_in=5923553 +cli-cli-13807 arm=a1 trial=4 all_pass=True dur=611.7s calls=112 tok_in=8274050 +cli-cli-13823 arm=a1 trial=1 all_pass=True dur=299.4s calls=72 tok_in=3582348 +cli-cli-13823 arm=a1 trial=2 all_pass=True dur=206.0s calls=44 tok_in=2023766 +cli-cli-13864 arm=a1 trial=0 all_pass=True dur=115.4s calls=15 tok_in=506492 +cli-cli-13864 arm=a1 trial=1 all_pass=True dur=77.9s calls=12 tok_in=375720 +cli-cli-13823 arm=a1 trial=4 all_pass=True dur=238.9s calls=26 tok_in=1479459 +cli-cli-13823 arm=a1 trial=3 all_pass=True dur=351.9s calls=62 tok_in=3722595 +cli-cli-13864 arm=a1 trial=2 all_pass=True dur=78.5s calls=13 tok_in=421433 +cli-cli-13987 arm=a1 trial=0 all_pass=False dur=51.1s calls=9 tok_in=189640 FAILED:['pr.task_pass'] +cli-cli-13864 arm=a1 trial=4 all_pass=True dur=100.7s calls=15 tok_in=476778 +cli-cli-13864 arm=a1 trial=3 all_pass=True dur=128.2s calls=19 tok_in=533772 +cli-cli-13987 arm=a1 trial=2 all_pass=True dur=53.6s calls=12 tok_in=612767 +cli-cli-13987 arm=a1 trial=3 all_pass=True dur=49.1s calls=9 tok_in=327236 +cli-cli-13987 arm=a1 trial=1 all_pass=True dur=107.0s calls=14 tok_in=559753 +cli-cli-13987 arm=a1 trial=4 all_pass=True dur=39.8s calls=11 tok_in=348235 +cli-cli-13988 arm=a1 trial=0 all_pass=True dur=70.2s calls=13 tok_in=312838 +cli-cli-13988 arm=a1 trial=1 all_pass=True dur=48.3s calls=9 tok_in=185792 +cli-cli-13988 arm=a1 trial=2 all_pass=True dur=41.0s calls=8 tok_in=135470 +cli-cli-13988 arm=a1 trial=3 all_pass=True dur=41.3s calls=10 tok_in=201787 +cli-cli-13988 arm=a1 trial=4 all_pass=True dur=45.3s calls=11 tok_in=238263 +scenario pass@1 trials verdict +cli-cli-13057 40% 5 KEEP +cli-cli-13362 60% 5 KEEP +cli-cli-13548 80% 5 KEEP +cli-cli-13722 80% 5 KEEP +cli-cli-13723 40% 5 KEEP +cli-cli-13807 80% 5 KEEP +cli-cli-13987 80% 5 KEEP +cli-cli-13068 100% 5 ceiling — drop +cli-cli-13412 100% 5 ceiling — drop +cli-cli-13413 100% 5 ceiling — drop +cli-cli-13418 100% 5 ceiling — drop +cli-cli-13421 100% 5 ceiling — drop +cli-cli-13430 100% 5 ceiling — drop +cli-cli-13449 100% 5 ceiling — drop +cli-cli-13471 100% 5 ceiling — drop +cli-cli-13523 100% 5 ceiling — drop +cli-cli-13541 100% 5 ceiling — drop +cli-cli-13580 100% 5 ceiling — drop +cli-cli-13620 100% 5 ceiling — drop +cli-cli-13681 100% 5 ceiling — drop +cli-cli-13823 100% 5 ceiling — drop +cli-cli-13864 100% 5 ceiling — drop +cli-cli-13988 100% 5 ceiling — drop +cli-cli-13459 0% 5 floor — drop + +7 of 24 tasks land in [0.25, 0.80] + 16 ceiling · 1 floor · 0 harness + +VERDICT: proceed with thin margin (7 usable). Consider a second fixture before generating contexts — it also unlocks the N-scaling claim a single fixture cannot reach. diff --git a/docs/VALIDATION-120.md b/docs/VALIDATION-120.md new file mode 100644 index 0000000..41781ad --- /dev/null +++ b/docs/VALIDATION-120.md @@ -0,0 +1,386 @@ +# Validation grid — 120 runs, single arm + +**This is method validation. It is not experiment data and must never be +reported as a result.** + +Every row carries `purpose: validation`. The registered analysis reads only +rows marked `experiment`, so nothing here can leak into a finding by accident. +The question this grid answers is not *"does the directed-contexts pattern +work"* — only one arm ran. It is *"can this fixture supply tasks that +discriminate between arms at all, and does the instrument measure them +honestly."* + +![The design tab: arm a1 described as the practical control, its always-loaded surface measured at 6,518 bytes with surface sha 5caa66e9, followed by the eight ground rules the run is bound by.](media/validation/validation-a1-design-tab.png) + +*The rules this grid is bound by, read off disk rather than off a document — +`[design]`. Four of them decide what follows: cost is meaningless without the +pass rate beside it, the calibration band is `[0.25, 0.80]`, harness faults are +never model failures, and validation never becomes evidence.* + +--- + +## 1. What ran + +| | | +|---|---| +| Batch | `runs/probe.jsonl`, 120 rows, complete | +| Suite commit | `1c573d7`, clean tree (`suite_dirty: false`) | +| Arm | `a1` only — the practical control, the repo's own `AGENTS.md` | +| Fixture | `cli-cli` at base `3c162a78` | +| Scenarios | 24, five trials each | +| Model | `local/qwen36-35b-a3b-nvfp4` | +| Harness | `opencode 1.18.10` via `adapters/opencode.sh`, Python 3.14.6 | +| Concurrency | `--jobs 3`, `--timeout 2700`, `--idle-timeout 300` | +| Window | 2026-08-04 20:27:11Z → 23:40:16Z (last start) | +| Integrity | `arm_sha df9a4467`, `scenario_sha eae60fa7` | + +Cost: 370,649,394 prompt tokens billed, 1,768,053 completion, 11.8 h of summed +trial duration across three workers. Median trial 161 s; longest 2,701 s — the +hard ceiling, which matters below. + +![The live view mid-run at 73 of 120: three running trials with their current shell commands, a per-scenario summary with pass rates and token medians, and a scrolling table of graded results newest-first.](media/validation/validation.png) + +*The grid in flight at 73/120. Three workers, one row each, with the command +each is executing; the per-cell rollup beneath; graded results newest-first. +`0 stalled` is a rendered state, not an absence — a hung worker would appear +here rather than silently vanish.* + +--- + +## 2. Headline + +**6 scenarios land in the calibration band `[0.25, 0.80]`, against +`MIN_PAIRED = 4`. The band requirement is met.** + +Split by grading tier, which Amendment 2 requires be reported separately and +never pooled: + +| Tier | In band | Scenarios | Rate | +|---|---|---|---| +| **unit** — primary evidence | **5** | 9 | 56% | +| **cli** — Amendment 2 tier | **1** | 15 | 7% | +| total | **6** | 24 | 25% | + +**The unit tier clears the threshold on its own** (5 ≥ 4). That is the result +that matters, because a passing unit test is a stronger signal than a flag +comparison and the registration treats the tiers as different evidence. + +**The CLI tier does not discriminate.** Eleven of fifteen cli-graded scenarios +sit at 100%. A grader that everything passes cannot separate arms, whatever its +other virtues. + +![The cost view: a pass-rate-versus-tokens scatter in which sixteen starred cells all sit on the 100% line, spread across the full width of the x-axis from 202k to 18,833k tokens, with only a handful of unstarred points below.](media/validation/validation-a1-cost-Pareto.png) + +*The ceiling, rendered. Sixteen cells on the Pareto frontier and **every one of +them at 100%**, spread across a 93× range of input tokens — 202k to 18,833k. +When the quality axis is pinned, the frontier degenerates into "whichever was +cheapest," and cost stops being a trade-off against anything. This is the +single clearest argument for treating the CLI tier as secondary.* + +--- + +## 3. Per-scenario results + +Pass rates are over **gradeable** trials — rows excluded as harness faults are +not counted as failures. See §5, where that was not always true. + +| Scenario | Grader | Graded | Pass | Band | Abandons | +|---|---|---|---|---|---| +| cli-cli-13057 | cli | 2 | 100% | | 4 | +| cli-cli-13068 | cli | 5 | 100% | | 1 | +| cli-cli-13362 | unit | 5 | 60% | **IN** | 0 | +| cli-cli-13412 | cli | 5 | 100% | | 0 | +| cli-cli-13413 | unit | 5 | 100% | | 0 | +| cli-cli-13418 | cli | 5 | 100% | | 0 | +| cli-cli-13421 | cli | 5 | 100% | | 0 | +| cli-cli-13430 | cli | 5 | 100% | | 0 | +| cli-cli-13449 | cli | 5 | 100% | | 1 | +| cli-cli-13459 | cli | 5 | 0% | | 0 | +| cli-cli-13471 | cli | 5 | 100% | | 0 | +| cli-cli-13523 | cli | 5 | 100% | | 0 | +| cli-cli-13541 | cli | 5 | 100% | | 1 | +| cli-cli-13548 | unit | 5 | 80% | **IN** | 0 | +| cli-cli-13580 | cli | 5 | 100% | | 0 | +| cli-cli-13620 | cli | 5 | 100% | | 0 | +| cli-cli-13681 | unit | 5 | 100% | | 0 | +| cli-cli-13722 | unit | 5 | 80% | **IN** | 0 | +| cli-cli-13723 | unit | 5 | 40% | **IN** | 0 | +| cli-cli-13807 | cli | 5 | 80% | **IN** | 0 | +| cli-cli-13823 | cli | 5 | 100% | | 0 | +| cli-cli-13864 | unit | 5 | 100% | | 0 | +| cli-cli-13987 | unit | 5 | 80% | **IN** | 0 | +| cli-cli-13988 | unit | 5 | 100% | | 0 | + +Exclusions applied per `docs/EVAL.md` criteria 1–2: **3 rows** for adapter +failure, 0 for schema violations. 117 of 120 rows are gradeable. + +Failing checks across the grid: `pr.task_pass` ×8, `cli.surface` ×5, +`pr.diff_coverage` ×2, `cli.builds` ×1. Seven rows abandoned. +Fifty-nine of 120 rows spawned subagents, consuming 24,125,708 input tokens +that the parent stream does not carry. + +![A cell detail card for a1/cli-cli-13068: median, mean, p90 and p90-over-median for tokens, calls, tool calls, probes to first edit and duration; then trials 5, pass@1 100%, two medians for tokens, ungradeable 0 of 5, abandoned 1 of 5, subagents dispatched, and a per-trial breakdown.](media/validation/validation-a1-live-summary-cli-cli-13068.png) + +*One cell opened with `[space]`. Two things to notice. The p90/median column +flags a 2.1× token tail in red — a median that looks cheap while the tail runs +away is not cheap. And there are **two** token medians: 1,383,795 over passing +trials against 1,902,217 over trials that actually worked, because the one +abandoned trial spends a fraction of a real attempt and drags the headline +number down. Reporting a single median here would understate cost by 27%.* + +--- + +## 4. H4 — proxy vs adapter calibration + +**The gate FAILS as specified, and the failure is entirely localized.** + +| Set | Aggregate delta | Worst run | Calls a/p | Gate | +|---|---|---|---|---| +| All 120 rows | 19.571% | 100.00% | 5319 / 6090 | **FAIL** | +| 117 gradeable | 9.484% | 87.48% | 5319 / 5646 | **FAIL** | + +Tolerance is 2% on aggregate, 2% on every run, and exact equality on call +counts. But the per-scenario view is the one that explains it: + +| | | +|---|---| +| Scenarios agreeing **exactly** — 0.0% tokens *and* identical call counts | **22 of 24** | +| `cli-cli-13057` | 90.7% delta; 115 adapter calls vs **846** proxy calls | +| `cli-cli-13068` | 14.3% delta; 205 vs 245 calls | + +Twenty-two scenarios do not merely agree within tolerance — they agree to the +token and to the call. The adapter's instrumentation is sound. + +**The mechanism is truncation, not accounting.** Three `cli-cli-13057` trials +hit the 2,700 s hard ceiling; the adapter reported **0 tokens** for each while +the proxy recorded 134–172 calls and 16–20 M tokens apiece. When a trial is +killed mid-flight its transcript is incomplete even after salvage, while the +proxy counts a round trip by construction — a request it handled. That is +exactly the asymmetry the registration anticipated when it made the proxy +authoritative. + +**All six in-band scenarios show 0.0% delta and matching call counts.** The +calibration failure does not touch a single scenario the experiment would use. + +**Consequences, stated plainly:** + +1. Per the registered rule, cost figures come from `runs/probe.proxy.jsonl`. + Adapter token totals are dropped from any report. +2. The gate's aggregate form is dominated by trials that are *already excluded* + as harness faults. Computing it over the analysis set rather than every row + is a defensible refinement, but it is **a deviation and would have to be + disclosed**, and it does not rescue the gate here (9.484%) — so nothing is + gained by making it. +3. `cli-cli-13057` is unusable on both axes: 3 of 5 trials ungradeable, 4 of 5 + abandoned, and a 90.7% accounting divergence. It should be dropped from the + fixture rather than carried. + +![The tasks tab for cli-cli-13057: PR 13057, judged by the PR's own binary flag-for-flag, forced by symbols IssueRelationshipsSupported and IssueType, answer spanning six directories, thirteen files touched, a 1800 second timeout, and a 173-line prompt beginning 'Add Issues 2.0 support: issue types, sub-issues, and relationships' that closes five separate issues.](media/validation/validation-a1-tasks-cli-cli-13057.png) + +*Why `13057` blew the ceiling, from the `[tasks]` tab — which carries no run +data at all, only what the scenario asks and how it is judged. It is a +**173-line prompt** implementing GitHub Issues 2.0 across `create`, `edit`, +`view` and `list`, closing five separate issues, touching 13 files across six +directories. This is not a task that a five-trial cell can characterize; it is +a project. The 2,700 s ceiling was not the problem — the scenario was.* + +--- + +## 5. Defects found by this grid + +The grid's other job is to break the instrument before the experiment does. +Four defects surfaced, all fixed. + +**Three of them were the same defect in three different surfaces**: the viewer +(§5.1), the runner's verdict (§5.4), and — already correct — the registered +analysis. Each computed a pass rate by dividing over every trial, counting +harness faults as model failures. The registered analysis had always applied +the exclusion, so the disagreement was silent until someone compared them. + +There is now one `is_ungradeable()` predicate in `suitedata`, imported by the +others. One definition, so the surfaces cannot drift apart again. + +### 5.1 The viewer scored harness faults as model failures — and it moved a scenario in-band + +`suitedata.pass_rate` averaged over every row. An ungradeable row carries +`all_pass=False` because nothing graded it, so an adapter that hit its ceiling +counted as a model that got the task wrong — which the registration forbids in +as many words ("Harness faults are not model failures"). The registered +analysis has always applied the exclusion, so the two surfaces disagreed: + +``` +cli-cli-13057 viewer 40% registered analysis 100% +``` + +40% is inside the calibration band. 100% is outside it. **The viewer was +inflating the count that decides whether the experiment can run.** Reported as +7 in band before the fix; 6 after. + +One `_ungradeable()` predicate now feeds both the `ung` column and the rate, so +a row cannot be displayed as ungradeable and counted as a failure beside it. +Verified: the viewer and the registered analysis now agree on every cell. +Fixed in `ffa6044`. + +### 5.2 The `left` column counted cells that had reported, not cells that exist + +`per_cell = expect / len(cells)`, where `len(cells)` is cells that have +*produced a row*. At 86/120 that read 120/18 = 7, so every finished five-trial +cell showed two trials remaining with an ETA attached to completed work. The +estimate converges only once the last scenario starts, which is when nobody +needs it. + +Now read from the batch's own recorded `argv` — the runner had already written +`--trials 5` down. The `[s] left` sort had the matching defect, ranking by time +*spent*. Fixed in `b2d1db4`. + +### 5.3 The grader hole was live for this run + +The suite commit (`1c573d7`) predates the fix (`df1ba6f`), so `cli-cli-13523` +was graded with no CLI surface and no unit grader — `all_pass` reduced to +"touched the right file." It reports **5/5 pass**. + +**Those five rows must be discarded.** The scenario is out of band either way, +so §2's count is unaffected; dropping it makes the cli tier 1 of 14. The fix is +in the tree and `mkscenarios` now refuses such tasks at extraction, so a +re-run would not produce them. + +### 5.4 The go/no-go verdict itself was wrong — a third surface, same defect + +`probe.py` prints the fixture's verdict at the end of every run. The raw +transcript of this grid is preserved at +[`VALIDATION-120-make-probe-sh.txt`](VALIDATION-120-make-probe-sh.txt), and it +ends: + +``` +cli-cli-13057 40% 5 KEEP +... +7 of 24 tasks land in [0.25, 0.80] + 16 ceiling · 1 floor · 0 harness + +VERDICT: proceed with thin margin (7 usable). +``` + +Two defects, compounding: + +1. The adapter check reports a ceiling hit as `ungradeable`, not `fail`, and + the harness counter matched on `"fail"`. It printed **`0 harness`** for a + run in which three trials were killed at 2,700 s. +2. The rate divided by all five trials, so those three counted as model + failures — `cli-cli-13057` read **40%** and was marked **KEEP**, landing + inside the band, when its gradeable trials were 2 of 2 and it belongs + outside. + +Corrected, the same command now prints: + +``` +cli-cli-13057 100% 2 ceiling — drop +6 of 24 tasks land in [0.25, 0.80] +``` + +This was the worst of the three places to hold the defect, because it is the +one that renders a decision rather than a number. The headline in §2 is **6**; +the transcript's **7** is preserved unedited as the record of what the +instrument said before it was fixed. + +### 5.5 A sharp edge left in place + +`status: "ungradeable"` is used both for *"this check could not run"* and for +informational skips such as `cli.extra_surface`. Across the grid there are 174 +checks with that status against only 3 genuinely ungradeable **rows**. Any +predicate that scans for "a check with status ungradeable" will classify all +120 rows as ungradeable — as one did while preparing this document. + +The row-level definition correctly keys on the `adapter` check by name, so no +reported number is affected. Recorded because the next person to write an +ad-hoc query will hit it. + +![A single trial's check list for cli-cli-13068 trial 0, verdict pass: pr.diff_coverage pass, pr.scope pass, pr.route ungradeable with routing evidence, cli.builds pass, cli.surface pass, cli.extra_surface pass.](media/validation/validation-a1-0-live-grading-cli-cli-13068.png) + +*The edge, on screen. This trial's verdict is **pass**, and one of its checks is +**ungradeable** — `pr.route`, which could not judge routing because the trail +shows no first edit to anchor on. A check that cannot run is not a row that +cannot be graded, and conflating the two classifies all 120 rows as harness +faults. Note also `purpose: validation` on the card: the guard is visible on +every single trial, not just in the file header.* + +--- + +## 6. Instrument floors + +Only `a1` ran, so the cross-arm agreement check could not execute — it needs +two arms to compare bytes-per-token rates. + +| | | +|---|---| +| `a1` always-loaded surface | 6,518 B on disk | +| Per-scenario first-call input | 11,413 – 12,295 tokens | + +The measured floor is stable across scenarios, which is what the check exists +to confirm. The rate cross-check must be re-run once a second arm is +materialized. + +--- + +## 7. What this grid does and does not license + +**Licensed:** + +- The fixture can supply discriminating tasks. Six scenarios land in band, five + of them unit-graded, against a threshold of four. +- The harness measures honestly on the scenarios that matter. Twenty-two of 24 + scenarios show exact adapter/proxy agreement, including all six in-band. +- The grading tiers behave as Amendment 2 predicted: unit grading + discriminates, CLI grading ceilings. + +**Not licensed:** + +- **Any statement about the directed-contexts pattern.** One arm ran. There is + no comparison here, and there is no primary outcome. + +![The arms rollup showing exactly one row: a1, monolith-realistic, 24 scenarios, 120 trials, 88% pass@1, 1,260,354 median tokens, 26 calls, marked 'reference' in the ratio columns, which are otherwise empty.](media/validation/validation-a1-arms-rollup.png) + +*The most honest image in this document. The arms rollup exists to show paired +geometric ratios against the reference, and here it shows **one row** — a1 +compared to itself, with the ratio columns empty because there is nothing to +divide by. The 88% is a1's unweighted mean across 24 scenarios, and it is not a +finding about anything. This is what "no experiment ran" looks like when the +instrument is honest about it.* +- **Any cost figure from the adapter.** H4 failed; the proxy is authoritative. +- **Any claim that 24 scenarios is the working set.** After dropping + `cli-cli-13523` (grader hole) and `cli-cli-13057` (unusable on two axes), the + fixture carries 22 scenarios, 6 of them discriminating. + +**Open before the experiment can run:** + +1. Drop `cli-cli-13523` (grader hole) and `cli-cli-13057` (unusable on two + axes); re-extract with the current `mkscenarios`. `probe.py` now agrees — + it marks both for dropping without being told. +2. Materialize arms `a0`, `a2`–`a5` and re-run `floors` for the bytes-per-token + agreement check. +3. Decide whether to run on the unit-graded subset only. The registration + already permits reporting the tiers separately; the cli tier's 1-of-15 rate + is an argument for treating it as secondary rather than as evidence. +4. Re-run calibration once ceiling-hit trials are removed from the fixture, and + confirm H4 passes on a clean grid. + +--- + +## 8. Reproduction + +```bash +# the grid, as it ran +PYTHONPATH=src python3 -m adherence.runner \ + --suite suite-pr.yaml --adapter adapters/opencode.sh \ + --model local/qwen36-35b-a3b-nvfp4 --trials 5 --arm a1 \ + --arms-dir fixtures/cli-cli.arms --out runs/probe.jsonl \ + --jobs 3 --timeout 2700 --idle-timeout 300 + +# the numbers in this document +PYTHONPATH=src python3 -m adherence.calibrate runs/probe.jsonl runs/probe.proxy.jsonl +PYTHONPATH=src python3 -m adherence.floors runs/probe.jsonl --arms-dir fixtures/cli-cli.arms --ref a1 +make table FILES=runs/probe.jsonl +make watch FILES=runs/probe.jsonl EXPECT=120 +``` + +Rows are `purpose: validation`. `make analyze` will exclude all 120 by design — +that is the guard working, not a failure. diff --git a/docs/media/validation/validation-a1-0-live-grading-cli-cli-13068.png b/docs/media/validation/validation-a1-0-live-grading-cli-cli-13068.png new file mode 100644 index 0000000..851bec5 Binary files /dev/null and b/docs/media/validation/validation-a1-0-live-grading-cli-cli-13068.png differ diff --git a/docs/media/validation/validation-a1-arms-rollup.png b/docs/media/validation/validation-a1-arms-rollup.png new file mode 100644 index 0000000..d9c6028 Binary files /dev/null and b/docs/media/validation/validation-a1-arms-rollup.png differ diff --git a/docs/media/validation/validation-a1-cost-Pareto.png b/docs/media/validation/validation-a1-cost-Pareto.png new file mode 100644 index 0000000..c5bcea9 Binary files /dev/null and b/docs/media/validation/validation-a1-cost-Pareto.png differ diff --git a/docs/media/validation/validation-a1-design-tab.png b/docs/media/validation/validation-a1-design-tab.png new file mode 100644 index 0000000..fd587cd Binary files /dev/null and b/docs/media/validation/validation-a1-design-tab.png differ diff --git a/docs/media/validation/validation-a1-live-summary-cli-cli-13068.png b/docs/media/validation/validation-a1-live-summary-cli-cli-13068.png new file mode 100644 index 0000000..06744b8 Binary files /dev/null and b/docs/media/validation/validation-a1-live-summary-cli-cli-13068.png differ diff --git a/docs/media/validation/validation-a1-tasks-cli-cli-13057.png b/docs/media/validation/validation-a1-tasks-cli-cli-13057.png new file mode 100644 index 0000000..4f6083d Binary files /dev/null and b/docs/media/validation/validation-a1-tasks-cli-cli-13057.png differ diff --git a/docs/media/validation/validation.png b/docs/media/validation/validation.png new file mode 100644 index 0000000..15ed20e Binary files /dev/null and b/docs/media/validation/validation.png differ diff --git a/scenarios-pr/cli-cli/ground-truth.jsonl b/scenarios-pr/cli-cli/ground-truth.jsonl index 20afd52..ef38290 100644 --- a/scenarios-pr/cli-cli/ground-truth.jsonl +++ b/scenarios-pr/cli-cli/ground-truth.jsonl @@ -1,34 +1,34 @@ -{"pr": 13988, "title": "Fix `RESTWithNext` error type, repairing `gh status` and attestation retries", "base_commit": "55dbb4dc6b7edb10b48e3d7fc5bccd32318d1b55", "merge_commit": "e83adbc0642994fae7c39a9a012eb34b8c81f4f1", "prompt": "Fix `RESTWithNext` error type, repairing `gh status` and attestation retries\n\n## Description\r\n\r\nThe `api.Client.RESTWithNext` method did not handle errors in the same way as `api.Client.REST`. The latter wrapped the `go-gh` error into a new `api` error and the former simply returned it. From an error message point of view, there was no visible difference but programmatic checks for `errors.As(err, &api.HTTPError{})` would be true for `REST` and false for `RESTWithNext`.\r\n\r\nI cannot find any reason for this to have been intentional. I suspect what happened was that `RESTWithNext` previously used to handle this case via the following removed code:\r\n\r\n```go\r\n\tsuccess := resp.StatusCode >= 200 && resp.StatusCode < 300\r\n\tif !success {\r\n\t\treturn \"\", HandleHTTPError(resp)\r\n\t}\r\n```\r\n\r\nBut at some point `go-gh` was created, and the `go-gh` api client `Request`/`RequestWithContext` methods began returning errors for non success http status codes.\r\n\r\nThis broke two behaviours:\r\n * `gh status` is supposed to gracefully degrade if notifications or events are inaccessible.\r\n * `gh attestation` commands is supposed to retry attestation fetching for 500 status codes.\r\n\r\n## Notes\r\n\r\nThis is a prerequisite for consolidating GitHub API URL derivation onto `api.Client` (part of the groundwork for per-host API routing, #13717 / #13760), but it is a user-facing bugfix in its own right and is worth reviewing as such.", "test_files": ["api/client_test.go", "pkg/cmd/attestation/api/client_test.go", "pkg/cmd/status/status_test.go"], "test_cmd": "go test ./api/... ./pkg/cmd/attestation/api/... ./pkg/cmd/status/...", "code_files": ["api/client.go"], "additions": 156, "verified": "fails at parent, passes with the fix"} -{"pr": 13987, "title": "Replace Windsurf with Devin in skill agents", "base_commit": "b1a9f7b183c12ae111ea63048d49aad2a53e9101", "merge_commit": "89f3699581659f514c4276bef0820d8caadbb67d", "prompt": "Replace Windsurf with Devin in skill agents\n\nFixes #13986\n\n- Replace the `windsurf` agent entry with `devin`\n- Install Devin skills in `.devin/skills` at project and user scope\n- Update install help text and registry tests\n\n## Testing\n\n- `go test ./internal/skills/... ./pkg/cmd/skills/...`", "test_files": ["internal/skills/registry/registry_test.go"], "test_cmd": "go test ./internal/skills/registry/...", "code_files": ["internal/skills/registry/registry.go", "pkg/cmd/skills/install/install.go"], "additions": 25, "verified": "fails at parent, passes with the fix"} -{"pr": 13967, "title": "Fix skill picker label wrapping", "base_commit": "01b79dd983af0859e4e3d7454961ad3f08cf88b4", "merge_commit": "c2ad3b0eb7ead66eec3a8a61239e10a765332ec7", "prompt": "Fix skill picker label wrapping\n\nTruncates skill picker labels to the available terminal width so descriptions do not wrap onto a separate line.\r\n\r\nAdds coverage for multiple terminal widths.\r\n\r\n**Before**\r\n\"Screenshot\r\n\r\n**After (Examples with varying terminal width)**\r\n\"Screenshot\r\n\"Screenshot\r\n\"Screenshot", "test_files": ["pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./pkg/cmd/skills/install/...", "code_files": ["pkg/cmd/skills/install/install.go"], "additions": 50, "verified": "fails at parent, passes with the fix"} -{"pr": 13864, "title": "Add Grok skill host support", "base_commit": "2b970995a3c63b7a7600f242d55d15185a82ed8b", "merge_commit": "9d95dd7344abe6388b59e25b8db54a3c7371cd6a", "prompt": "Add Grok skill host support\n\n## Summary\n- add Grok as a supported `gh skill` agent host\n- install project and user skills in `.grok/skills`\n\nFixes #13846", "test_files": ["internal/skills/registry/registry_test.go"], "test_cmd": "go test ./internal/skills/registry/...", "code_files": ["internal/skills/registry/registry.go", "pkg/cmd/skills/install/install.go", "pkg/cmd/skills/update/update.go"], "additions": 26, "verified": "fails at parent, passes with the fix"} -{"pr": 13823, "title": "Add named field columns to gh project item-list", "base_commit": "bba0816a3808c7f7ea0a7e75be9d365b3410c524", "merge_commit": "efe3f165dd297c85fff11473dbf586f2d39fbf86", "prompt": "Add named field columns to gh project item-list\n\nProposal: cli/cli#13816\nTracking issue: github/planning-tracking#3431\n\n## What this does\n\nAdds named field columns to `gh project item-list`:\n\n```\ngh project item-list --owner --field \"Status\"\n```\n\n- `--field ` (repeatable) resolves the field by name and adds a column showing each item's value for it, with no separate field-ID preflight lookup.\n- `--field-id ` does the same by node ID. The two are mutually exclusive, and an unknown `--field-id` lists candidate fields as `name (id)` for symmetry with the name-based errors.\n- Name matching is case-insensitive, mirroring the Projects v2 platform API.\n\n## Relationship to cli/cli#13807\n\nThis builds on the same shared name-resolution layer as the `item-edit` PR cli/cli#13807. To keep the two PRs independently mergeable, each one adds its own copy of the small, self-contained field-resolver file `` (~90 lines: `ResolveFieldByName`, `ResolveFieldByID`, and their error types), with identical contents in both. Because the copies match exactly, Git merges them into a single file automatically, so the two PRs can land in either order without conflicts or duplicate-definition build errors. The item-list-only `FieldValueNodes.DisplayValue` helper lives in its own ``, so this PR leaves `` untouched.", "test_files": ["pkg/cmd/project/item-list/item_list_test.go", "pkg/cmd/project/shared/queries/queries_test.go", "pkg/cmd/project/shared/queries/resolve_fields_test.go"], "test_cmd": "go test ./pkg/cmd/project/item-list/... ./pkg/cmd/project/shared/queries/...", "code_files": ["pkg/cmd/project/item-list/item_list.go", "pkg/cmd/project/shared/queries/queries.go", "pkg/cmd/project/shared/queries/resolve_fields.go"], "additions": 813, "verified": "fails at parent, passes with the fix"} -{"pr": 13807, "title": "Add name-based resolution to gh project item-edit", "base_commit": "9d95dd7344abe6388b59e25b8db54a3c7371cd6a", "merge_commit": "d1897f89b294fc172e5b2993325ad1a0184f711d", "prompt": "Add name-based resolution to gh project item-edit\n\nProposal: #13816\nTracking issue: github/planning-tracking#3431\n\n> **Companion PR:** The `item-list` read path ships as a separate, independently-mergeable PR \u2014 #13823. Both PRs add their own copy of the small `` file, with identical contents. Because the two copies match exactly, Git merges them into one automatically, so the PRs can land in either order without conflicts.\n\n## What this does\n\nLets `gh project item-edit` address ProjectV2 fields, single-select options, and items by **name** instead of node IDs. Existing ID-based flags are fully backward compatible; the name flags are additive and optional.\n\n### `item-edit` (new name-based mode)\nMirrors `item-add` addressing:\n\n```\ngh project item-edit --owner --url --field \"Status\" --value \"In Progress\"\n```\n\n- `[]` positional + `--owner` resolve the project.\n- `--url` resolves the item within that project.\n- `--field ` resolves the field; `--value ` is dispatched by the field's data type (text/number/date, and for single-select the value is resolved as an option **name**).\n\nhttps://github.com/user-attachments/assets/4e7dceb1-0e4f-4706-9d95-37570a56cdcd\n\n### Mutual exclusivity (enforced in code)\nEvery name/ID pair fails clearly when both are set, never silently merged: `--field` xor `--field-id`, `--url` xor `--id`, and `--value` xor the typed value flags (`--text`/`--number`/`--date`/`--single-select-option-id`/`--iteration-id`). Name-based flags also require the project number argument. Invalid combos are rejected before any GraphQL request fires.\n\n### Errors are self-correctable\nName resolution happens before the ID-typed GraphQL mutation, so a bad name fails loudly with candidate names rather than `400 []`. Six failure modes mirror the sibling MCP PR (github/github-mcp-server#2760): field-not-found, field-ambiguous, option-not-found, option-ambiguous, wrong-field-type, item-not-in-project \u2014 surfaced as human-readable stderr messages listing candidates (CLI convention; not JSON).\n\n### Name matching is case-insensitive\nField and option names are matched with `strings.EqualFold` to mirror the Projects v2 platform API, which normalizes field names via a `name_slug` and downcases single-select option names before matching. Field names are case-insensitively unique on the platform; option names are not, so a case-only option collision still surfaces the `option_ambiguous` error.\n\n## Backwards compatibility\nThe command previously marked `--id` as a required cobra flag; this PR removes that requirement so an item can also be addressed by `--url`, but `--id` remains required in effect via a runtime guard (one of `--id` or `--url` must be set). Existing `--id`-based invocations are unchanged.\n\n## Out of scope / follow-ups\n- **`item-list` named columns** \u2014 ships as a separate, independently-mergeable PR (#13823).\n- MultiSelect and issue-field-value node types: the shared resolver dispatches option resolution by field data type so adding these later is additive, not a rewrite.\n- Bounding/paginating large read payloads (github/planning-tracking#3332).\n\n## Coordination with the sibling MCP PR (github/github-mcp-server#2760)\n- **Case-insensitive matching:** this PR matches names case-insensitively (see above). MCP #2760's resolver currently does case-sensitive matching and should be aligned so both surfaces behave identically.\n- **Read-path mutual exclusivity:** the companion `item-list` PR (#13823) enforces reject-both for its name/ID field selector. MCP #2760's read path currently silently merges `fields`+`field_names`; that should be aligned to the same reject-both behavior.", "test_files": ["pkg/cmd/project/item-edit/item_edit_test.go", "pkg/cmd/project/shared/queries/resolve_fields_test.go", "pkg/cmd/project/shared/queries/resolve_test.go"], "test_cmd": "go test ./pkg/cmd/project/item-edit/... ./pkg/cmd/project/shared/queries/...", "code_files": ["pkg/cmd/project/item-edit/item_edit.go", "pkg/cmd/project/shared/queries/queries.go", "pkg/cmd/project/shared/queries/resolve.go", "pkg/cmd/project/shared/queries/resolve_fields.go"], "additions": 1432, "verified": "fails at parent, passes with the fix"} -{"pr": 13784, "title": "Support antigravity-cli and antigravity2.0 in gh skill", "base_commit": "397876d3dc2828f1c932c651b31ab78e72506da4", "merge_commit": "326faaac8b5c3b736ddc6bfa573cc97dc6452a24", "prompt": "Support antigravity-cli and antigravity2.0 in gh skill\n\n### Description\n\n`gh skill` recognized only one Antigravity surface. This registers `antigravity-cli` and `antigravity2.0` as skill agent hosts across every `gh skill` subcommand, at both project and user scope.\n\nFixes #13767\n\n### Key Points\n\nAntigravity ships several surfaces that share one project skills dir but read user-scope skills from different dirs. The two new [registry entries](https://github.com/cli/cli/blob/8c6bac8c9/internal/skills/registry/registry.go#L91-L104) map each surface to its documented path:\n\n```go\n// Both share the .agents/skills project dir; user dirs differ per surface.\nUserDir: \".gemini/antigravity-cli/skills\" // antigravity-cli\nUserDir: \".gemini/config/skills\" // antigravity2.0\n```\n\nBecause both reuse `.agents/skills` for the project dir, selecting them next to other `.agents/skills` agents still writes each skill once. Nothing else needs wiring: `--agent`, `--scope`, `InstallDir`, help text, and enum suggestions all derive from the registry slice.\n\n### Notes for reviewers\n\nStart in the [`registry` package](https://github.com/cli/cli/blob/8c6bac8c9/internal/skills/registry/registry.go), where the `AgentHost` entries live. The [registry tests](https://github.com/cli/cli/blob/8c6bac8c9/internal/skills/registry/registry_test.go) extend the existing `TestFindByID` and `TestInstallDir` tables with project- and user-scope cases for both agents.\n\nThe user-scope paths come from Antigravity's own docs, confirmed by hands-on testing (see Additional Context). `.gemini/config/skills` is the one global dir every surface reads, so `antigravity2.0` uses it; the CLI reads its own `.gemini/antigravity-cli/skills`.\n\n### Additional Context\n\n- [Antigravity skills docs](https://antigravity.google/docs/skills) give the official per-surface skill directories.\n- [Where does Antigravity look for Agent Skills?](https://atamel.dev/posts/2026/07-01_where_agy_agent_skills/) verifies by experiment which dir each surface actually reads.\n- [Choosing your surface](https://cloud.google.com/blog/topics/developers-practitioners/choosing-your-surface-antigravity-20-antigravity-cli-antigravity-ide-or-antigravity-sdk) explains the 2.0 / CLI / IDE split.", "test_files": ["internal/skills/registry/registry_test.go"], "test_cmd": "go test ./internal/skills/registry/...", "code_files": ["internal/skills/registry/registry.go"], "additions": 53, "verified": "fails at parent, passes with the fix"} -{"pr": 13766, "title": "fix(skills): honor --dir without agent prompt", "base_commit": "ff4e4a255006ca98e3036bd0f8d98374a405835f", "merge_commit": "f1d112104821b055bb0c5656f2989f9213db71f6", "prompt": "fix(skills): honor --dir without agent prompt\n\n## Summary\n- skip the target-agent prompt when `gh skill install` is given `--dir`\n- keep using the default agent host internally only for shared post-install messaging / telemetry\n- add regression coverage for the interactive `--dir` path\n\nFixes #13763.\n\n## Testing\n- not run locally (avoided local Go test/build in this environment)", "test_files": ["pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./pkg/cmd/skills/install/...", "code_files": ["pkg/cmd/skills/install/install.go"], "additions": 34, "verified": "fails at parent, passes with the fix"} -{"pr": 13723, "title": "Allow downloading release assets without authentication", "base_commit": "954ffc37e5931a9621868a50f05c8fb17e288f19", "merge_commit": "71fb4f5f4358c44c9328637752f0c15a47d84139", "prompt": "Allow downloading release assets without authentication\n\nFixes https://github.com/cli/cli/issues/2680\n\n### Description\n\n`gh release download` now works without authentication against public repositories, matching `gh extension install`. A token is still used when one is present.\n\n### Key Points\n\nThis change has three parts, from simplest to subtlest:\n\n- **Why drop the auth gate** - the login requirement is unnecessary for public downloads.\n- **Fixing the by-tag race to be success oriented** - an anonymous draft-lookup 403 no longer masks a release the REST lookup found.\n- **Honoring the gate under repo override** - the cobra plumbing fix that makes the dropped gate apply to `download`.\n\n#### Why drop the auth gate\n\nRelease endpoints for public repositories are readable anonymously over REST, so the login gate is unnecessary. Removing it is the same one-line `cmdutil.DisableAuthCheck` change that #13176 made for `gh extension install`.\n\nThe same logic here applies as with `gh extension install`:\n\n> What this says is that extension installation is going to happen unauthenticated whether we make it easier or not. We might as well make it easier and keep people installing through our core commands.\n\nIn `gh release download` - people are going to just `curl` the endpoint directly, and that degrades their experience since we provide a lot of abstractions over the API that make `gh` much more than a `curl`/`gh api` call.\n\nBut I'll broaden the conversation - we got feedback that folks are using `gh release download` more in CI and they want to secure their CI by not providing `gh` a token unless it's necessary. If you're downloading a public asset, that's not necessary and we force you to use a larger than necessary token scope.\n\n#### Fixing the by-tag race to be success oriented\n\nThe by-tag path creates problems that make this deviate a bit from `gh ext install` - that's one reason why this diff is a bit bigger. [`FetchRelease`](https://github.com/cli/cli/blob/f0a128ad1c33a41b7d1bdc93b5d507c842713d8a/pkg/cmd/release/shared/fetch.go) races a published-release REST lookup against a draft-release GraphQL lookup. GraphQL rejects anonymous requests, so the draft lookup returns a 403. The old selector returned the first result unless it was exactly `ErrReleaseNotFound`, so that 403 could win the race and mask a release the REST lookup found.\n\nThe selector now prefers a found release and only errors when both lookups fail. We used to return whichever result arrived first, treating only a not-found error as a reason to wait for the second:\n\n```go\n// Before - a non-not-found error from either lookup wins\nres := <-results\nif errors.Is(res.error, ErrReleaseNotFound) {\n res = <-results\n cancel()\n} else {\n cancel()\n <-results // drain the channel\n}\nreturn res.release, res.error\n```\n\nNow we take a success from either lookup, and only surface an error when both fail:\n\n```go\n// Now - a found release wins; an error needs both lookups to fail\nfirst := <-results\nif first.error == nil {\n cancel()\n <-results // drain the channel\n return first.release, nil\n}\n\nsecond := <-results\ncancel()\nif second.error == nil {\n return second.release, nil\n}\nif errors.Is(second.error, ErrReleaseNotFound) {\n return nil, second.error\n}\nreturn nil, first.error\n```\n\nThis also stops a transient draft-lookup failure from masking a real release for authenticated users.\n\n#### Honoring the gate under repo override\n\nRemoving the gate did not take effect on its own. `release` enables the `-R/--repo` override, and that installs a `PersistentPreRunE` on the `release` command. cobra runs only the nearest such hook walking up from the command you invoked, so the override hook shadows the [root auth gate](https://github.com/cli/cli/blob/7b681a4e8b67d203ccdaab5ff54570bdf6ca3669/pkg/cmd/root/root.go). The override re-runs the nearest ancestor hook to make up for that, but it handed *the ancestor* to the hook as the command, so the auth gate judged `release` instead of `download` and never saw download's `DisableAuthCheck`.\n\nThe fix keeps the invoked command and passes that leaf up, the node cobra would have judged if the repo override didn't muck with it:\n\n```go\n// Before - climbs by reassigning cmd, so the gate is judged against the ancestor\nfor cmd.HasParent() {\n cmd = cmd.Parent()\n if cmd.PersistentPreRunE != nil {\n return cmd.PersistentPreRunE(cmd, args)\n }\n}\n\n// Now - keep the invoked leaf and pass it up\nfor p := overrideCmd.Parent(); p != nil; p = p.Parent() {\n if p.PersistentPreRunE != nil {\n return p.PersistentPreRunE(cmd, args)\n }\n}\n```\n\nThis mirrors [cobra's own `execute` loop](https://github.com/spf13/cobra/blob/v1.10.2/command.go#L984-L986), which walks the parents but always hands them the leaf command. cobra's `EnableTraverseRunHooks` global would run the whole chain for us and maybe is a better long term fix, but it is process-wide and would change pre-run behavior for every command, so I'm not touching it here.\n\nI guarded this with an integration test, since the bug only surfaces with the pieces wired together. [`Test_EnableRepoOverride_authCheckIntegration`](https://github.com/cli/cli/blob/7b681a4e8b67d203ccdaab5ff54570bdf6ca3669/pkg/cmdutil/repo_override_test.go) builds a root gate, a repo-override parent, and a leaf, then asserts the gate judged the leaf: opting out with `DisableAuthCheck` skips the check, otherwise the gate still runs. It lives in `cmdutil` beside the helper, since the coupling is the helper's, not any one command's.\n\n### Notes for reviewers\n\nThree atomic commits:\n\n- `fix(release): don't let a failed draft lookup mask a found release` changes the selector in [`FetchRelease`](https://github.com/cli/cli/blob/f0a128ad1c33a41b7d1bdc93b5d507c842713d8a/pkg/cmd/release/shared/fetch.go) and adds a regression case to the existing `Test_downloadRun` table.\n- `feat(release): allow download without authentication` removes the gate in the [download command](https://github.com/cli/cli/blob/f0a128ad1c33a41b7d1bdc93b5d507c842713d8a/pkg/cmd/release/download/download.go).\n- `fix(cmdutil): honor DisableAuthCheck under repo-override parents` fixes [the repo-override helper](https://github.com/cli/cli/blob/7b681a4e8b67d203ccdaab5ff54570bdf6ca3669/pkg/cmdutil/repo_override.go) so the gate removal actually takes effect, and adds an integration test that pins the coupling between repo override and the root auth gate.\n\n### Additional Context\n- #2680 is the standing request to allow unauthenticated public requests, and its motivating example is `gh release download` on a public repo.\n- #13176 removed the same gate for `gh extension install` and is the precedent this follows.", "test_files": ["pkg/cmd/release/download/download_test.go", "pkg/cmdutil/repo_override_test.go"], "test_cmd": "go test ./pkg/cmd/release/download/... ./pkg/cmdutil/...", "code_files": ["pkg/cmd/release/download/download.go", "pkg/cmd/release/shared/fetch.go", "pkg/cmdutil/repo_override.go"], "additions": 142, "verified": "fails at parent, passes with the fix"} -{"pr": 13722, "title": "Detect additional third-party coding agents", "base_commit": "dd26eb39b04db5bd282509787d2b8fbff11ba694", "merge_commit": "561d9f919cbdadc22f886a3eadb260ba0b4f49c5", "prompt": "Detect additional third-party coding agents\n\n### Description\n\nExtends agent detection so the User-Agent header can identify more third-party coding agents driving `gh`. Adds nine new agents and an extra environment-variable signal for one existing one.\n\n### Key Points\n\nNew agents detected: antigravity, augment-cli, replit, goose, cowork, cursor, cursor-cli, kiro, and pi. One existing agent gains an extra signal: claude-code also matches `CLAUDE_CODE`.\n\nDetection order encodes confidence. `AI_AGENT` still wins, followed by agent-specific variables. Two signals are deliberately low-confidence and ranked accordingly:\n\n- `REPL_ID` is present throughout any Replit environment, not only when a Replit agent drives the CLI, so it is annotated as broad in the code.\n- `GOOSE_PROVIDER` only means Goose is configured as a model provider, not that it is running, so Goose is checked absolutely last and any stronger signal wins.\n\nThe single-source block now matches precisely instead of loosely: `kiro` is an exact `TERM_PROGRAM` match, and `pi` is anchored to a path boundary (`/.pi/agent`, or the Windows `\\.pi\\agent`) so an incidental substring cannot trigger it.\n\nWe considered a Devin signal and dropped it. The only reliable marker is the existence of a `/opt/.devin` directory, and this detector is intentionally environment-only with no filesystem access. The environment-variable alternatives (for example an `EDITOR` substring) could produce false positives, and Devin's developer-terminal footprint is small, so a noisy signal was not worth the telemetry pollution.\n\n### Notes for reviewers\n\nTwo files change. Start in the [`agents` detector](https://github.com/cli/cli/blob/1d4ca33b34646ce1165067377fd89f01148cbaf3/internal/agents/detect.go): the constant block lists the new identifiers, and `detectWith` runs the ordered checks top to bottom, returning on first match. Read it straight down to see the precedence. Then the [detector tests](https://github.com/cli/cli/blob/1d4ca33b34646ce1165067377fd89f01148cbaf3/internal/agents/detect_test.go) add table cases for every new agent and signal, plus the precedence and negative cases (exact `kiro`, path-boundary `pi`, Goose losing to stronger signals).\n\nFirst-party documentation for each signal lives inline next to its check, or is marked `// No first-party docs` where none exists.", "test_files": ["internal/agents/detect_test.go"], "test_cmd": "go test ./internal/agents/...", "code_files": ["internal/agents/detect.go"], "additions": 190, "verified": "fails at parent, passes with the fix"} -{"pr": 13681, "title": "fix(skills): install universal agent to ~/.agents/skills", "base_commit": "70bb306bd25eb407f90eabefd98824aed62cf519", "merge_commit": "cd68c69dbd5ea0793cece6355f0911ca6a68ffc1", "prompt": "fix(skills): install universal agent to ~/.agents/skills\n\nThe Universal agent host installed user-scoped skills to ~/.config/agents/skills, which is not the location scanned by the majority of compliant clients (Copilot CLI, Pi, OpenCode).\n\nPer the agentskills.io cross-client convention (https://agentskills.io/client-implementation/adding-skills-support), user-level skills for compliant clients live under ~/.agents/skills.\n\nThis PR switches UserDir for the universal agent to the shared project skills directory constant, matching the convention already used by Warp and Cline. Adds regression tests covering both project and user scope.\n\nFixes #13494", "test_files": ["internal/skills/registry/registry_test.go"], "test_cmd": "go test ./internal/skills/registry/...", "code_files": ["internal/skills/registry/registry.go"], "additions": 20, "verified": "fails at parent, passes with the fix"} -{"pr": 13679, "title": "fix: show checks summary when all checks were cancelled", "base_commit": "70bb306bd25eb407f90eabefd98824aed62cf519", "merge_commit": "23f83e6ae18998fcd6ac0497f57a1d9ebd95862a", "prompt": "fix: show checks summary when all checks were cancelled\n\n### Description\n\n`gh pr checks` prints a blank summary when a PR's only checks were **cancelled**.\n\n`printSummary` (``) gates the summary block on:\n\n```go\nif counts.Failed+counts.Passed+counts.Skipping+counts.Pending > 0 {\n```\n\n`counts.Canceled` is omitted, so a cancelled-only result sums to 0, the block is skipped, and the `else if counts.Canceled > 0 { \"Some checks were cancelled\" }` branch becomes unreachable in exactly the case it was written for. The existing \"some cancelled\" test masked this by pairing a cancelled check with passing ones.\n\n### Fix\n\nAdd `counts.Canceled` to the guard.\n\n### Testing\n\nAdded `TestPrintSummary` (``) with a cancelled-only case asserting the \"Some checks were cancelled\" summary renders. Fails before, passes after; `go test ./pkg/cmd/pr/checks/` green.\n\nReported in #13680.", "test_files": ["pkg/cmd/pr/checks/output_test.go"], "test_cmd": "go test ./pkg/cmd/pr/checks/...", "code_files": ["pkg/cmd/pr/checks/output.go"], "additions": 56, "verified": "fails at parent, passes with the fix"} -{"pr": 13675, "title": "Fix flaky TestHuhPrompterMultiSelectWithSearchPersistence on slow architectures", "base_commit": "015e6cf6ac39ad18138f868545f02359c3c8d3ef", "merge_commit": "954ffc37e5931a9621868a50f05c8fb17e288f19", "prompt": "Fix flaky TestHuhPrompterMultiSelectWithSearchPersistence on slow architectures\n\nFixes https://github.com/cli/cli/issues/13727\r\n\r\nThe test was using a fixed 50ms sleep (`waitForOptions()`) to wait for an async bubbletea search goroutine before sending the next keystroke. On slow architectures such as s390x under QEMU emulation, the goroutine scheduling and event loop round-trip can exceed 50ms, causing the `enter` key to arrive while `m.loading` is still `true`. In that state the field silently drops all keystrokes, the form never advances, and the 5s overall deadline fires with \"form.Run() did not complete in time\".\r\n\r\nThe fix replaces the blind sleep with proper channel-based synchronization. An `onSearchDone func()` hook is added to `multiSelectSearchField` (nil in production, set only in tests) and called at the end of `applySearchResult`. A new test helper `waitForSearch(field)` wires a one-shot `sync.Once`-guarded channel into that hook and blocks until it fires - guaranteeing the search is truly complete before the next step is sent, regardless of scheduler timing or machine speed.\r\n\r\n## Error log\r\n```\r\n[ 770s] --- FAIL: TestHuhPrompterMultiSelectWithSearchPersistence (5.42s)\r\n[ 770s] --- FAIL: TestHuhPrompterMultiSelectWithSearchPersistence/selections_persist_after_changing_search_query (5.31s) Context\r\n[ 770s] :478: form.Run() did not complete in time 54,494 tokens\r\n[ 770s] FAIL 5% used\r\n[ 770s] FAIL github.com/cli/cli/v2/internal/prompter 11.841s\r\n```", "test_files": ["internal/prompter/huh_prompter_test.go"], "test_cmd": "go test ./internal/prompter/...", "code_files": ["internal/prompter/multi_select_with_search.go"], "additions": 69, "verified": "fails at parent, passes with the fix"} -{"pr": 13624, "title": "Gracefully handle failed GitHub verifier initialization caused by missing trusted root", "base_commit": "bd81afda9ab653bd880298e6cfbf143541522a2f", "merge_commit": "d51b72243a01729cdc76a5b57a0866a2f1bfd2bc", "prompt": "Gracefully handle failed GitHub verifier initialization caused by missing trusted root\n\n\r\n\r\nGracefully handle the failure to initialize the GitHub verifier when the trusted root is missing.", "test_files": ["pkg/cmd/attestation/verification/sigstore_integration_test.go", "pkg/cmd/attestation/verification/sigstore_test.go"], "test_cmd": "go test ./pkg/cmd/attestation/verification/...", "code_files": ["pkg/cmd/attestation/verification/sigstore.go"], "additions": 39, "verified": "fails at parent, passes with the fix"} -{"pr": 13620, "title": "feat(discussion): add comment command", "base_commit": "4166ecf2cada209714af79387a5243b334af764b", "merge_commit": "42db02df0c55f8ae556b15ca353887ee4043cd88", "prompt": "feat(discussion): add comment command\n\n## Add `gh discussion comment` command\r\n\r\nThis PR adds a new `gh discussion comment` command for managing comments and replies on GitHub Discussions, along with UX improvements to the `view` command.\r\n\r\n### `gh discussion comment`\r\n\r\nThe command supports adding, editing, and deleting comments and replies. The positional argument determines the action context:\r\n\r\n**Adding a top-level comment:**\r\n```\r\ngh discussion comment 123 --body \"Thanks for raising this\"\r\ngh discussion comment 123 --body-file response.md\r\n```\r\n\r\n**Replying to a comment (positional arg is a comment ID or URL):**\r\n```\r\ngh discussion comment DC_abc123 --body \"Good point\"\r\ngh discussion comment \"https://github.com/OWNER/REPO/discussions/123#discussioncomment-456\" --body \"I agree\"\r\n```\r\n\r\n**Editing a comment or reply:**\r\n```\r\ngh discussion comment DC_abc123 --edit --body \"Updated text\"\r\ngh discussion comment \"https://github.com/OWNER/REPO/discussions/123#discussioncomment-456\" --edit --body-file updated.md\r\n```\r\n\r\n**Deleting a comment or reply:**\r\n```\r\ngh discussion comment DC_abc123 --delete --yes\r\ngh discussion comment \"https://github.com/OWNER/REPO/discussions/123#discussioncomment-456\" --delete --yes\r\n```\r\n\r\nWhen no `--body` or `--body-file` is provided in an interactive terminal, an editor is opened for input. For edits, the editor is pre-populated with the existing comment body.\r\n\r\nComment URLs are resolved to node IDs via msgpack encoding of the repository database ID and comment database ID extracted from the URL fragment.\r\n\r\n### Changes to `gh discussion view`\r\n\r\nThe `--replies` flag now accepts both comment node IDs and comment URLs:\r\n\r\n```\r\ngh discussion view 123 --replies DC_abc123\r\ngh discussion view 123 --replies \"https://github.com/OWNER/REPO/discussions/123#discussioncomment-456\"\r\n```\r\n\r\nWhen a URL is provided, the repository and discussion number from the URL override the positional argument.\r\n\r\n### Tests\r\n\r\n- Added acceptance tests for the `comment` command covering add, reply, edit (including from file), delete (with confirmation prompt check), and URL-based operations.\r\n- Updated existing acceptance tests (`discussion-view.txtar`, `discussion-list.txtar`) to use the new `comment` command instead of raw `gh api graphql` mutations.\r\n- Added a test case in `discussion-view.txtar` to verify `--replies` works with a comment URL.\r\n- Unit tests cover flag parsing, all run paths (add/reply/edit/delete), comment URL resolution, body-file/stdin input, editor pre-population, and error cases.", "test_files": ["pkg/cmd/discussion/client/client_impl_test.go", "pkg/cmd/discussion/comment/comment_test.go", "pkg/cmd/discussion/shared/lookup_test.go", "pkg/cmd/discussion/view/view_test.go"], "test_cmd": "go test ./pkg/cmd/discussion/client/... ./pkg/cmd/discussion/comment/... ./pkg/cmd/discussion/shared/... ./pkg/cmd/discussion/view/...", "code_files": ["pkg/cmd/discussion/client/client.go", "pkg/cmd/discussion/client/client_impl.go", "pkg/cmd/discussion/client/client_mock.go", "pkg/cmd/discussion/client/types.go", "pkg/cmd/discussion/comment/comment.go", "pkg/cmd/discussion/create/create.go", "pkg/cmd/discussion/discussion.go", "pkg/cmd/discussion/edit/edit.go", "pkg/cmd/discussion/shared/lookup.go", "pkg/cmd/discussion/view/view.go"], "additions": 2666, "verified": "fails at parent, passes with the fix"} -{"pr": 13581, "title": "Auto-install official extension stubs in CI", "base_commit": "46480afb125d852c3edd2b4850a97c09b65c64cf", "merge_commit": "7ef9c54bf87fb17516e93a7ddb932efb7c660a35", "prompt": "Auto-install official extension stubs in CI\n\n### Description\n\nSkip the install prompt when `CI` is detected so official extension stubs (like `gh aw`, `gh stack`) auto-install instead of blocking on `Y/n` in CI runners and agent sessions.\n\n### Key Points\n\n- The previous version of this diff mirrored [`runCopilot`](https://github.com/cli/cli/blob/trunk/pkg/cmd/copilot/copilot.go) exactly: TTY always wins, so a pseudo-TTY agent session with `CI=1` still hung on the prompt. This revision flips the order so `CI` wins over TTY.\n- Intentional divergence from `gh copilot`: a follow-up could apply the same change there for consistency, but extension stubs are the more pressing case (agents call `gh ` frequently).\n- `CI` is detected via [`ci.IsCI()`](https://github.com/cli/cli/blob/trunk/internal/ci/ci.go) (any of `CI`, `BUILD_NUMBER`, or `RUN_ID`).\n\n### Notes for reviewers\n\nTwo commits, reviewable in order:\n\n1. [`fix(extension): auto-install official extension stubs in CI`](https://github.com/cli/cli/pull/13581/commits/ef8c7d8ab5c5539a5279f0c05d2d83d6a39f22be) is the behavior change. The [`official_extension_stub`](https://github.com/cli/cli/blob/ef8c7d8ab5c5539a5279f0c05d2d83d6a39f22be/pkg/cmd/root/official_extension_stub.go) file holds the new branching; the [stub test](https://github.com/cli/cli/blob/ef8c7d8ab5c5539a5279f0c05d2d83d6a39f22be/pkg/cmd/root/official_extension_stub_test.go) extends the existing run-function table test and clears CI env vars per subtest; the [acceptance test](https://github.com/cli/cli/blob/ef8c7d8ab5c5539a5279f0c05d2d83d6a39f22be/acceptance/testdata/telemetry/telemetry-for-official-extension-stub.txtar) clears CI env vars and expects non-zero exit plus install instructions on stderr.\n2. [`docs(extension): explain CI bypass in stub install logic`](https://github.com/cli/cli/pull/13581/commits/61a752068e271b699b67bdb39b90a10549adf332) is a one-line comment addressing review feedback that the non-TTY+CI behavior wasn't obvious from the code.\n\n### Additional Context\n\nManually verified locally with `gh stack` uninstalled. The TTY+CI case is the one this revision actually fixes versus the previous version of the diff:\n\n```\n$ CI=1 ./bin/gh stack --help # TTY + CI\nSuccessfully installed github/gh-stack # exit 0\n\n$ CI=1 ./bin/gh stack --help < /dev/null # non-TTY + CI\nSuccessfully installed github/gh-stack # exit 0\n\n$ ./bin/gh stack --help < /dev/null # non-TTY, no CI\ngh stack is available as an official extension.\nTo install it, run:\n gh extension install github/gh-stack # exit 1\n```", "test_files": ["pkg/cmd/root/official_extension_stub_test.go"], "test_cmd": "go test ./pkg/cmd/root/...", "code_files": ["pkg/cmd/root/official_extension_stub.go"], "additions": 57, "verified": "fails at parent, passes with the fix"} -{"pr": 13580, "title": "feat: add `repo read-file` and `repo read-dir`", "base_commit": "d05c71a9da4bce4d57f622aacc6a8ee24e5e158e", "merge_commit": "733e35c04c4141468f8882d17b063414ffe36c57", "prompt": "feat: add `repo read-file` and `repo read-dir`\n\n## Summary\r\n\r\nThis PR adds two new (preview) commands for reading repository contents without cloning.\r\n\r\n### `gh repo read-file`\r\n\r\n```\r\ngh repo read-file [--ref ] [--output ] [--clobber] [--unsafe] [--json ] [--jq ] [--template ] [--repo ]\r\n```\r\n\r\nReads a single file from a repository and writes it to stdout (or to a file with `--output`).\r\n\r\n### `gh repo read-dir`\r\n\r\n```\r\ngh repo read-dir [] [--ref ] [--json ] [--jq ] [--template ] [--repo ]\r\n```\r\n\r\nLists the entries of a directory in a repository, showing each entry's type, name, and size.\r\n\r\nBoth commands are marked **preview**, default to the current repository's default branch (overridable with `--ref`), and support `--json`/`--jq`/`--template` for scripting.\r\n\r\n## Test plan\r\n\r\nYou can verify the command behaviour via acceptance tests and the following examples.\r\n\r\nThe commands below target public repositories so they can be run as-is.\r\n\r\n### `gh repo read-file`\r\n\r\n```sh\r\n# Read a file to stdout\r\ngh repo read-file README.md --repo cli/cli\r\n\r\n# Read from a specific tag/branch/commit\r\ngh repo read-file README.md --ref v2.0.0 --repo cli/cli\r\n\r\n# Write to a file, then confirm --clobber is required to overwrite\r\ngh repo read-file README.md --output --repo cli/cli\r\ngh repo read-file README.md --output --repo cli/cli # should error: already exists\r\ngh repo read-file README.md --output --clobber --repo cli/cli # should succeed\r\n\r\n# JSON output, with jq and template\r\ngh repo read-file README.md --json path,size,type --repo cli/cli\r\ngh repo read-file README.md --json path,size --jq '.size' --repo cli/cli\r\ngh repo read-file README.md --template '{{.path}} ({{.size}} bytes){{\"\\n\"}}' --repo cli/cli\r\n\r\n# Error: path does not exist\r\ngh repo read-file --repo cli/cli\r\n```\r\n\r\n### `gh repo read-dir`\r\n\r\n```sh\r\n# List the repository root\r\ngh repo read-dir --repo cli/cli\r\n\r\n# List a subdirectory (note executables render as `file*`)\r\ngh repo read-dir script --repo cli/cli\r\n\r\n# Mixed entry types in one listing: regular files, a symlink (RelNotes),\r\n# and a submodule (sha1collisiondetection)\r\ngh repo read-dir --repo git/git\r\n\r\n# List from a specific tag/branch/commit\r\ngh repo read-dir docs --ref v2.0.0 --repo cli/cli\r\n\r\n# Non-TTY / scriptable output (tab separated)\r\ngh repo read-dir script --repo cli/cli | cat\r\n\r\n# JSON output, with jq and template\r\ngh repo read-dir script --json name,type,mode,modeOctal --repo cli/cli\r\ngh repo read-dir script --json name,type --jq '.entries[] | select(.type==\"file\") | .name' --repo cli/cli\r\ngh repo read-dir --template '{{range .entries}}{{.type}}: {{.name}}{{\"\\n\"}}{{end}}' --repo cli/cli\r\n\r\n# Error: path points to a file, not a directory\r\ngh repo read-dir README.md --repo cli/cli\r\n\r\n# Error: path or ref does not exist\r\ngh repo read-dir nope/nope --repo cli/cli\r\n```", "test_files": ["internal/text/text_test.go", "pkg/cmd/repo/read-dir/read_dir_test.go", "pkg/cmd/repo/read-file/read_file_test.go"], "test_cmd": "go test ./internal/text/... ./pkg/cmd/repo/read-dir/... ./pkg/cmd/repo/read-file/...", "code_files": ["internal/text/text.go", "pkg/cmd/repo/read-dir/http.go", "pkg/cmd/repo/read-dir/read_dir.go", "pkg/cmd/repo/read-file/http.go", "pkg/cmd/repo/read-file/read_file.go", "pkg/cmd/repo/repo.go"], "additions": 2535, "verified": "fails at parent, passes with the fix"} -{"pr": 13548, "title": "feat(skills): list available skills when install runs non-interactively", "base_commit": "01bcd474447b5034da20686c838ae4b0cf2b23f5", "merge_commit": "70bb306bd25eb407f90eabefd98824aed62cf519", "prompt": "feat(skills): list available skills when install runs non-interactively\n\n### Summary\n\nRunning `gh skill install ` (or `--from-local `) without a skill name in a non-interactive context previously errored with `must specify a skill name when not running interactively`. That made discovery awkward when piping or scripting \u2014 users had to switch to an interactive shell just to see what skills a repo offered.\n\nThis PR changes that behavior so the discovered skills are printed instead, mirroring (approximately) what the interactive picker shows. The list is rendered through `tableprinter`, so it auto-switches between a friendly TTY table and tab-separated `SKILL\\tDESCRIPTION` rows when piped \u2014 making it easy to grep / fzf / awk.\n\n### Before\n\n```console\n$ gh skill install github/awesome-copilot | grep review\nmust specify a skill name when not running interactively\n```\n\n### After\n\n```console\n$ gh skill install github/awesome-copilot | grep review\ncode-review Reviews pull requests\n\u2026\n```\n\nRe-running with a specific skill name installs as before. Same behavior for `--from-local`.\n\n### Context\n\nCame out of a Slack thread where this was identified as a UX gap: interactive mode supports browsing-then-installing, and non-interactive mode should provide the same discoverability so it composes with standard Unix tooling.\n\n### Changes\n\n- `selectSkillsWithSelector` now prints discovered skills via a new `listAvailableSkills` helper instead of erroring when `!canPrompt`.\n- Sentinel `errSkillsListed` lets `installRun` / `runLocalInstall` bail cleanly without going into the install path.\n- Descriptions are pre-fetched (same `fetchDescriptions` callback the picker uses) so the listed output is useful, not bare.\n- Help text + an example for piping into `grep` updated.\n- Tests: replaced the old \"errors\" case with a remote listing assertion and added an equivalent local install case.\n\n### Verification\n\n- `go test ./pkg/cmd/skills/...` \u2705\n- `make lint` \u2705", "test_files": ["pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./pkg/cmd/skills/install/...", "code_files": ["pkg/cmd/skills/install/install.go"], "additions": 105, "verified": "fails at parent, passes with the fix"} -{"pr": 13541, "title": "feat: add `discussion` command set", "base_commit": "5c9ec1c5f51d682dbf65b0c16f856d8b9303adff", "merge_commit": "52e8e7427702e023d743318f748fceda4c0dcf37", "prompt": "feat: add `discussion` command set\n\nThis PR adds the `gh discussion` command group (list, view, create, edit) as a preview feature.\r\n\r\nHuge thanks to @maxbeizer for doing the heavy lifting on this. :copilot: was also instrumental in implementation, review, and polish throughout.\r\n\r\nI rewrote the commit history (81 commits squashed into 6) for easier review and a cleaner git log. Note that the commits are structured to split the work logically; intermediate commits may not compile independently.\r\n\r\n## Commands\r\n\r\n### `discussion list`\r\n\r\n```shell\r\ngh discussion list [--category ] [--state ] [--author ] [--label ] [--search ] [--limit ] [--json ] [--web]\r\n```\r\n\r\nList discussions in a repository with optional filters. ([8b73951](https://github.com/cli/cli/commit/8b73951ca75c6495a776e36da9fb48cc93000866))\r\n\r\n### `discussion view`\r\n\r\n```\r\ngh discussion view { | } [--comments [--limit ]] [--replies [--limit ]] [--json ] [--web]\r\n```\r\n\r\nView a discussion's details, comments, or replies to a specific comment. ([57008e7](https://github.com/cli/cli/commit/57008e797097e9b275ba7d1849ac750e762743e1))\r\n\r\n### `discussion create`\r\n\r\n```\r\ngh discussion create --title --body --category [--label ,...] [--body-file ]\r\n```\r\n\r\nCreate a new discussion, interactively or via flags. ([7bd67a8](https://github.com/cli/cli/commit/7bd67a840456436a1edc0f80dc85687f29b277af))\r\n\r\n### `discussion edit`\r\n\r\n```\r\ngh discussion edit { | } [--title ] [--body ] [--body-file ] [--category ] [--add-label ,...] [--remove-label ,...]\r\n```\r\n\r\nEdit a discussion's title, body, category, or labels, interactively or via flags. ([797effe](https://github.com/cli/cli/commit/797effe0295914e03f4b40cb873276da1d3b7e93))\r\n\r\n## Reusable components\r\n\r\n- **``**: GraphQL client for Discussions covering list, search, get, create, update, and label mutations with full pagination support. ([51b7653](https://github.com/cli/cli/commit/51b765381870dbf62390e0f7a297cc57ea771db7))\r\n- **``**: Shared utilities including argument parsing, category matching, label resolution, and client factory helpers. ([d3a1538](https://github.com/cli/cli/commit/d3a153872b9774009d620d0f3e12f993a21765eb))\r\n\r\n## Screenshots\r\n\r\n### `gh discussion list`\r\n\r\n![Terminal output showing a table of discussions with columns for number, state, title, category, author, and date](https://github.com/user-attachments/assets/094587eb-93f8-4b55-8bf9-5ed0965cc67d)\r\n\r\n### `gh discussion view 9094`\r\n\r\n![Terminal output showing discussion details including title, metadata, labels, and body text](https://github.com/user-attachments/assets/55dcf8c0-c4e0-4bc0-b073-8025c5490107)\r\n\r\n### `gh discussion view 9094 --comments --limit 1`\r\n\r\n![Terminal output showing a discussion with its threaded comments displayed below the body](https://github.com/user-attachments/assets/64ea48f7-2cd6-404e-b261-14eaf9a26eb1)\r\n\r\n### `gh discussion view 9094 --replies DC_kwDODKw3uc4AkHLS --limit 1`\r\n\r\n![Terminal output showing paginated replies to a specific discussion comment](https://github.com/user-attachments/assets/87bad4fe-82f3-4124-91c4-c581d9e2a0ef)\r\n\r\n### `gh discussion create` (non-interactive)\r\n\r\n![Terminal showing discussion creation via flags with title, body, category, and label options](https://github.com/user-attachments/assets/bf65e362-7983-4b97-af48-2840fa74c3b9)\r\n\r\n### `gh discussion create` (interactive, category selection)\r\n\r\n![Terminal showing an interactive prompt listing available discussion categories to choose from](https://github.com/user-attachments/assets/7fdcf50f-b74d-4e5d-87a3-9a05aa9bbc78)\r\n\r\n### `gh discussion create` (interactive)\r\n\r\n![Terminal showing interactive prompts for title and body input during discussion creation](https://github.com/user-attachments/assets/764ea1e8-2225-4c6f-9803-77e438e1d1fa)\r\n\r\n### `gh discussion edit` (interactive)\r\n\r\n1. Select fields to update\r\n![Terminal showing a multi-select prompt asking which fields to edit: title, body, category](https://github.com/user-attachments/assets/be05bef6-4dce-44bf-a78e-9b907ef70d46)\r\n\r\n2. Populate new values\r\n![Terminal showing input prompts pre-filled with current values for the selected fields](https://github.com/user-attachments/assets/b2bb8fa8-b0e0-475a-a7ff-a7d6fcf29b0b)\r\n\r\n3. Submit\r\n![Terminal showing the discussion URL printed after a successful edit](https://github.com/user-attachments/assets/9d8e9c31-020f-4961-8ce7-d43bf685ca0d)\r\n\r\n### `gh discussion edit` (non-interactive)\r\n\r\n![Terminal showing discussion edit via flags updating title, body, and labels in one command](https://github.com/user-attachments/assets/5fc60e1b-227a-4ee1-b5b7-2b6281561c5f)", "test_files": ["acceptance/acceptance_test.go", "pkg/cmd/discussion/client/client_test.go", "pkg/cmd/discussion/comment/comment_test.go", "pkg/cmd/discussion/create/create_test.go", "pkg/cmd/discussion/edit/edit_test.go", "pkg/cmd/discussion/list/list_test.go", "pkg/cmd/discussion/shared/labels_test.go", "pkg/cmd/discussion/shared/lookup_test.go", "pkg/cmd/discussion/view/view_test.go"], "test_cmd": "go test ./acceptance/... ./pkg/cmd/discussion/client/... ./pkg/cmd/discussion/comment/... ./pkg/cmd/discussion/create/... ./pkg/cmd/discussion/edit/... ./pkg/cmd/discussion/list/... ./pkg/cmd/discussion/shared/... ./pkg/cmd/discussion/view/...", "code_files": ["pkg/cmd/discussion/client/client.go", "pkg/cmd/discussion/client/client_mock.go", "pkg/cmd/discussion/client/types.go", "pkg/cmd/discussion/comment/comment.go", "pkg/cmd/discussion/create/create.go", "pkg/cmd/discussion/discussion.go", "pkg/cmd/discussion/edit/edit.go", "pkg/cmd/discussion/list/list.go", "pkg/cmd/discussion/shared/categories.go", "pkg/cmd/discussion/shared/client.go", "pkg/cmd/discussion/shared/labels.go", "pkg/cmd/discussion/shared/lookup.go", "pkg/cmd/discussion/view/view.go", "pkg/cmd/root/root.go"], "additions": 13075, "verified": "fails at parent, passes with the fix"} -{"pr": 13523, "title": "Support custom CLAUDE_CONFIG_DIR in install", "base_commit": "9a593ce81b593dee752cc11737d1a3ef768e52b3", "merge_commit": "412c5fae63239705c0120e76ec9508307d2709a0", "prompt": "Support custom CLAUDE_CONFIG_DIR in install\n\n\r\n\r\nFixes https://github.com/cli/cli/issues/13262\r\n\r\n## Summary\r\n\r\nThis updates Claude Code user-scope skill directory resolution to honor `CLAUDE_CONFIG_DIR` when it is set. User-scope installs for Claude Code now write to `$CLAUDE_CONFIG_DIR/skills`, falling back to the existing `~/.claude/skills` location when the environment variable is not present.\r\n\r\nThe change lives in the shared agent registry path resolver so `gh skill install`, `gh skill list`, and `gh skill update` all use the same location for Claude Code user-scope skills.\r\n\r\n## Tests\r\n\r\n- Added registry coverage for Claude Code's default user-scope path and `CLAUDE_CONFIG_DIR` override.\r\n- Added an install regression test that verifies `gh skill install --agent claude-code --scope user` writes under `$CLAUDE_CONFIG_DIR/skills` and not `~/.claude/skills`.\r\n- Ran `go test ./internal/skills/registry ./pkg/cmd/skills/...`.", "test_files": ["internal/skills/registry/registry_test.go", "pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./internal/skills/registry/... ./pkg/cmd/skills/install/...", "code_files": ["internal/skills/registry/registry.go"], "additions": 69, "verified": "fails at parent, passes with the fix"} -{"pr": 13471, "title": "add --all flag to install all skills in a repo", "base_commit": "8bd56966ac5e2fe7d3162ad424dd11d877f4815e", "merge_commit": "e6fa2faf9e3203c2caaa54dcb1526c5f093fa998", "prompt": "add --all flag to install all skills in a repo\n\nFixes https://github.com/cli/cli/issues/13453\r\n\r\n## Summary\r\n- Add `--all` to `gh skill install` so users can install every discovered skill from a repository without providing an individual skill name.\r\n- Reuse the existing all-skills selection path, including collision detection, so skills that would overwrite each other still fail with guidance.\r\n- Update help text, examples, and tests for flag parsing, validation, and multi-skill remote installs.\r\n\r\n## Testing\r\n- `go test ./pkg/cmd/skills/install`", "test_files": ["pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./pkg/cmd/skills/install/...", "code_files": ["pkg/cmd/skills/install/install.go"], "additions": 78, "verified": "fails at parent, passes with the fix"} -{"pr": 13469, "title": "Improve update --all command if some skills don't have metadata", "base_commit": "81ed6d4e355446dbb4c1a7b038534948f7c195b8", "merge_commit": "5cfb648ace667b394728332d492b3e1a017baafa", "prompt": "Improve update --all command if some skills don't have metadata\n\n\r\n\r\nFixes https://github.com/cli/cli/issues/13369\r\n\r\n## Summary\r\n\r\nFixes `gh skill update --all` so it no longer prompts for skills that are missing GitHub metadata.\r\n\r\nWhen `--all` is used, skills without GitHub metadata are skipped with a notice, matching the flag's \"without prompting\" behavior.\r\n\r\n## Changes\r\n\r\n- Skip missing-metadata skills when `--all` is used instead of prompting interactively.\r\n- Keep the warning summary so users know which skills were skipped.\r\n- Update the warning text to explain that users can run `gh skill update ` interactively to add metadata, or reinstall the skill.\r\n- Add a regression test to verify `--all` does not prompt for missing metadata.\r\n\r\n## Testing\r\n\r\n- `go test ./pkg/cmd/skills/update/... -run TestUpdateRun -count=1`\r\n- `go test ./pkg/cmd/skills/... -count=1`", "test_files": ["pkg/cmd/skills/update/update_test.go"], "test_cmd": "go test ./pkg/cmd/skills/update/...", "code_files": ["pkg/cmd/skills/update/update.go"], "additions": 39, "verified": "fails at parent, passes with the fix"} -{"pr": 13459, "title": "fix discovery support in nested dirs", "base_commit": "8bd56966ac5e2fe7d3162ad424dd11d877f4815e", "merge_commit": "7f885723a9cb72cd4318eef3c831f8a7bb9d9093", "prompt": "fix discovery support in nested dirs\n\n\r\n\r\nFixes https://github.com/cli/cli/issues/13372\r\n\r\n## Summary\r\n\r\nFix `gh skill install` and `gh skill preview` handling for nested skill paths.\r\n\r\n## What changed\r\n\r\n- Allow `--allow-hidden-dirs` to discover hidden skill directories at any repository depth, not only at the root.\r\n- Treat deeply nested skill arguments as exact repo-relative paths in both `install` and `preview`.\r\n - Example: `packages/agent-skills/my-skill`\r\n- Preserve one-slash selectors like `owner/skill` as namespaced skill names, not paths.\r\n- Share path-detection logic between `install` and `preview`.\r\n- Add regression tests for nested hidden directories, arbitrary nested path installs, and arbitrary nested path previews.\r\n\r\n## Product note\r\n\r\nThis does not change the recommended Agent Skills publishing conventions or broaden search/publish behavior. Non-standard layouts are only supported when the user explicitly provides an exact path.\r\n\r\n## Testing\r\n\r\n- `go test ./internal/skills/discovery ./pkg/cmd/skills/install ./pkg/cmd/skills/preview -run 'TestIsSkillPath|Test_isSkillPath|TestPreviewRun|TestMatchHiddenDirConventions|TestDiscoverSkillsWithOptions|TestDiscoverLocalSkillsWithOptions|TestInstallRun' -count=1`\r\n- `go test ./internal/skills/discovery ./pkg/cmd/skills/... -count=1`\r\n- `git diff --check`", "test_files": ["internal/skills/discovery/discovery_test.go", "pkg/cmd/skills/install/install_test.go", "pkg/cmd/skills/preview/preview_test.go"], "test_cmd": "go test ./internal/skills/discovery/... ./pkg/cmd/skills/install/... ./pkg/cmd/skills/preview/...", "code_files": ["internal/skills/discovery/discovery.go", "pkg/cmd/skills/install/install.go", "pkg/cmd/skills/preview/preview.go"], "additions": 271, "verified": "fails at parent, passes with the fix"} -{"pr": 13449, "title": "fix(skills): stage updates in a temp dir and swap in-place", "base_commit": "a6012aca41ffa2d943ea9f12094a7398ce2e4b7e", "merge_commit": "01bcd474447b5034da20686c838ae4b0cf2b23f5", "prompt": "fix(skills): stage updates in a temp dir and swap in-place\n\nCloses #13370.\n\n## Problem\n\n`gh skill update` could relocate skills and delete the original install directory:\n\n- With `--dir`, namespaced layouts (`{dir}/{namespace}/{name}/SKILL.md`) had their install base computed by walking `filepath.Dir` twice, putting the new install one level too shallow. The post-install migration step then `RemoveAll`ed the original namespaced directory, taking the user's files with it.\n- Without `--dir`, the same migration cleanup quietly relocated namespaced skills found under an agent install directory to a flat layout, even though users expect updates to land where the skill currently lives.\n\nEither path breaks symlinks, mounts, and any external references that point at the skill directory.\n\n## Fix\n\nEach update is staged into a sibling directory of the existing skill dir (same filesystem, so renames are atomic). On success the contents are swapped in via per-entry rename:\n\n1. Move every existing entry from the skill dir into a sibling backup dir.\n2. Move every staged entry into the skill dir.\n3. Remove the backup.\n\nIf any step fails, the rollback path removes any freshly installed entries and moves the originals back from the backup.\n\nThis guarantees:\n\n- All updates happen in-place, regardless of layout (flat vs. namespaced) and regardless of whether the skill was found via an agent host or via `--dir`.\n- The skill directory's inode is preserved, so symlinks, mounts, and external references continue to resolve.\n- Stale files from the previous version are removed.\n- A failure at any point (install, read, rename) leaves the existing skill completely untouched.\n\n## Tests\n\n- New acceptance test `` plants a namespaced skill, runs `gh skill update --dir`, and asserts the skill remains at the original namespaced path with no flat-path duplicate.\n- New unit test `TestSwapDirectoryContents_PreservesDestInode` asserts the skill directory's identity is preserved across a successful swap (via `os.SameFile`).\n- New unit test `TestSwapDirectoryContents_RollsBackOnFailure` exercises the rollback path and asserts the original content (including nested subdirs) is fully restored.\n- The \"namespaced skill with `--dir`\" unit test is updated to assert in-place behavior and stale-file cleanup.\n- The existing `install_failure_during_update_reports_error_and_continues` unit test had a harness shortcut that silently skipped its `verify` block whenever `wantErr` was set. That shortcut is removed so the \"preserve original on install failure\" assertion is now actually enforced.", "test_files": ["pkg/cmd/skills/update/update_test.go"], "test_cmd": "go test ./pkg/cmd/skills/update/...", "code_files": ["pkg/cmd/skills/update/update.go"], "additions": 242, "verified": "fails at parent, passes with the fix"} -{"pr": 13430, "title": "Derive digest algorithm from ref length in release verify commands", "base_commit": "b99d73d7421ae182b8ab3f5b4469dff93c792da9", "merge_commit": "f1c10e032a636540a5e28705682e203b118a47fa", "prompt": "Derive digest algorithm from ref length in release verify commands\n\nFixes #13429\n\n### Summary\n\nThe `gh release verify` and `gh release verify-asset` commands hard-coded a `sha1:` prefix when constructing the digest identifier for a release tag's commit SHA. Once GitHub repositories using SHA-256 commit digests are supported, that ref will be a 64-character SHA-256 hash and labeling it as `sha1:` is both misleading in user-facing output and incorrect for the attestation lookup.\n\n### Changes\n\n- Added a small shared helper `DigestAlgForRef(digest string) string` in `` that returns `\"sha256\"` for 64-character digests and `\"sha1\"` otherwise (preserving existing behavior).\n- Replaced the hard-coded `\"sha1\"` argument in `` and `` with a call to the new helper.\n- Added unit tests for `DigestAlgForRef` covering SHA-1, SHA-256, empty, and unexpected lengths.\n- Added `*_SHA256` variants of the existing `FailedNoAttestations` tests in both commands that assert the error message includes `sha256:`.\n\n### Verification\n\n- `go test ./pkg/cmd/release/shared/... ./pkg/cmd/release/verify/... ./pkg/cmd/release/verify-asset/...` passes.\n- `go build ./...` passes.\n- Existing SHA-1 tests continue to pass (no regression).", "test_files": ["pkg/cmd/release/shared/fetch_test.go", "pkg/cmd/release/verify-asset/verify_asset_test.go", "pkg/cmd/release/verify/verify_test.go"], "test_cmd": "go test ./pkg/cmd/release/shared/... ./pkg/cmd/release/verify/... ./pkg/cmd/release/verify-asset/...", "code_files": ["pkg/cmd/release/shared/fetch.go", "pkg/cmd/release/verify-asset/verify_asset.go", "pkg/cmd/release/verify/verify.go"], "additions": 153, "verified": "fails at parent, passes with the fix"} -{"pr": 13421, "title": "Allow agents as application for secrets", "base_commit": "909cbd7d8834b5fb63d671a55a540c2cfa04d669", "merge_commit": "e60f00e30880079adefd258c388c642b1e97ddf6", "prompt": "Allow agents as application for secrets\n\nFixes https://github.com/cli/cli/issues/13419\r\n\r\nI built it locally and ran `bin/gh secret set TEST_SECRET -a agents -b 123 -o ` and it worked exactly as expected.\r\n\r\nWhat might be missing is another test? I looked through it and added a little bit but maybe a maintainer has a little more input regarding that.", "test_files": ["pkg/cmd/secret/delete/delete_test.go", "pkg/cmd/secret/list/list_test.go", "pkg/cmd/secret/set/set_test.go", "pkg/cmd/secret/shared/shared_test.go"], "test_cmd": "go test ./pkg/cmd/secret/delete/... ./pkg/cmd/secret/list/... ./pkg/cmd/secret/set/... ./pkg/cmd/secret/shared/...", "code_files": ["pkg/cmd/secret/delete/delete.go", "pkg/cmd/secret/list/list.go", "pkg/cmd/secret/secret.go", "pkg/cmd/secret/set/set.go", "pkg/cmd/secret/shared/shared.go"], "additions": 199, "verified": "fails at parent, passes with the fix"} -{"pr": 13418, "title": "New `gh skill list` to inventory installed agent skills", "base_commit": "e6fa2faf9e3203c2caaa54dcb1526c5f093fa998", "merge_commit": "674e02a56b020549b45f970808ce7c20e179824a", "prompt": "New `gh skill list` to inventory installed agent skills\n\nCloses #13245 \r\n\r\n## Summary\r\n\r\nAdds `gh skill list` to inventory installed agent skills across known agent host directories.\r\n\r\n- Scans project and user skill install paths for all registered agent hosts\r\n- Deduplicates shared install directories like `.agents/skills`\r\n- Supports filtering with `--agent`, `--scope`, and custom scanning with `--dir`\r\n- Renders a concise table with `NAME`, `AGENT`, `SCOPE`, and `SOURCE`\r\n- Supports parseable output via `--json`, `--jq`, and `--template` with fields for `skillName`, `hosts`, `scope`, `sourceURL`, `version`, `pinned`, and `path`\r\n- Records privacy-safe `skill_list` telemetry with filters, output format, and result count\r\n\r\n\r\n## Testing\r\n\r\n- `go test ./pkg/cmd/skills/list/...`\r\n- `go test ./pkg/cmd/skills/...`", "test_files": ["pkg/cmd/skills/list/list_test.go"], "test_cmd": "go test ./pkg/cmd/skills/list/...", "code_files": ["pkg/cmd/skills/list/list.go", "pkg/cmd/skills/skills.go"], "additions": 1123, "verified": "fails at parent, passes with the fix"} -{"pr": 13413, "title": "Overwrite existing parent on `gh issue edit --add-sub-issue`", "base_commit": "6106bd30764b17caac56fa57876ff9f5536093d4", "merge_commit": "b08503368202901bbc1b6fc7936dff650eb32012", "prompt": "Overwrite existing parent on `gh issue edit --add-sub-issue`\n\nFix the `Sub issue may only have one parent` 422 that surfaces when `--add-sub-issue` is handed a candidate that already belongs to another parent. Pass `replaceParent: true` so the take-over is silent.\r\n\r\nStacked on top of #13412 (which is on top of #13411).", "test_files": ["acceptance/acceptance_test.go", "pkg/cmd/issue/edit/edit_test.go"], "test_cmd": "go test ./acceptance/... ./pkg/cmd/issue/edit/...", "code_files": ["api/queries_issue.go"], "additions": 221, "verified": "fails at parent, passes with the fix"} -{"pr": 13412, "title": "Rename Editable.Allowed to Selectable", "base_commit": "72859237e26439cfc23be9fb2a8144dc441a0826", "merge_commit": "2503dcfd79e372bfade8f41d05a2fbae57da1e26", "prompt": "Rename Editable.Allowed to Selectable\n\nAddresses [thread r3196020651](https://github.com/cli/cli/pull/13057#discussion_r3196020651) on #13057.\r\n\r\nStacked on top of #13411 (`-feedback`) since the field name is leveraged in the parent Issues 2.0 PR.", "test_files": ["acceptance/acceptance_test.go", "pkg/cmd/issue/edit/edit_test.go", "pkg/cmd/pr/shared/survey_test.go"], "test_cmd": "go test ./acceptance/... ./pkg/cmd/issue/edit/... ./pkg/cmd/pr/shared/...", "code_files": ["api/queries_issue.go", "pkg/cmd/issue/edit/edit.go", "pkg/cmd/pr/edit/edit.go", "pkg/cmd/pr/shared/editable.go"], "additions": 241, "verified": "fails at parent, passes with the fix"} -{"pr": 13403, "title": "Use int64 for GitHub database IDs", "base_commit": "1e6bb88c6d8503189bca408f7be162018b1acf84", "merge_commit": "0274077b56a5ef8e575358721149cd02888b2a5f", "prompt": "Use int64 for GitHub database IDs\n\n## Use `int64` for GitHub database IDs\r\n\r\nCloses #9247\r\n\r\n### Problem\r\n\r\nSeveral structs in the CLI use Go's `int` type for GitHub database ID fields. Go's `int` is platform-dependent and could be 32-bit on some architectures, while GitHub internally uses 64-bit integers for database IDs. The [REST API OpenAPI spec](https://github.com/github/rest-api-description) explicitly declares many resource IDs as `integer` with `format: int64` (e.g. repositories, GPG keys, SSH keys), and the [GraphQL API has migrated some `databaseId` fields from `Int` to `BigInt`](https://docs.github.com/en/graphql/overview/breaking-changes).\r\n\r\n### Changes\r\n\r\nUpdated all struct fields representing GitHub database IDs from `int` to `int64`, along with their downstream usage:\r\n\r\n- **Codespaces**: `Repository.ID`, `CreateCodespaceParams.RepositoryID`, `startCreateRequest.RepositoryID`, plus interface/mock/test cascading\r\n- **Rulesets**: `RulesetGraphQL.DatabaseId`, `RulesetREST.Id`, `BypassActors.ActorId`, `RulesetRule.RulesetId`, and the skills `rulesetsResponse.ID`\r\n- **Keys**: `deployKey.ID`, `gpgKey.ID`, `sshKey.ID`\r\n- **Cache**: `Cache.Id`, plus `parseCacheID` and `deleteCacheByID` (which still used `int`/`strconv.Atoi`)\r\n- **Autolinks**: `Autolink.ID`\r\n- **Agent tasks**: `JobActor.ID`, `JobPullRequest.ID`\r\n\r\nAll `strconv.Itoa()` calls on these fields were updated to `strconv.FormatInt(x, 10)`.\r\n\r\nGraphQL response structs were also audited - only `RulesetGraphQL.DatabaseId` needed updating. Others were already correct (`GitHubUser.DatabaseID` is `int64`, `FullDatabaseID` is `string` for GraphQL `BigInt`).\r\n\r\n### Future: Custom linter to prevent regressions\r\n\r\nA custom `go/analysis` linter that flags struct fields with ID-like names or JSON tags still using `int` has been prototyped on a separate branch: [`9247-idtype-linter`](https://github.com/cli/cli/compare/trunk...9247-idtype-linter). It's split out to keep this PR focused on the type changes. The linter correctly identifies all the violations fixed here.\r\n\r\n
\r\nLinter output BEFORE this PR (27 violations)\r\n\r\n```\r\n:38:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:43:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:20:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:25:2: struct field Id looks like a GitHub database ID but uses int; use int64 instead\r\n:153:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:807:2: struct field RepositoryID looks like a GitHub database ID but uses int; use int64 instead\r\n:858:2: struct field RepositoryID looks like a GitHub database ID but uses int; use int64 instead\r\n:203:4: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:216:4: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:260:4: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:591:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:614:2: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:621:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:639:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:660:2: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:666:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:858:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:879:2: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:885:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:14:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:6:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:16:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:13:2: struct field DatabaseId looks like a GitHub database ID but uses int; use int64 instead\r\n:27:2: struct field Id looks like a GitHub database ID but uses int; use int64 instead\r\n:33:3: struct field ActorId looks like a GitHub database ID but uses int; use int64 instead\r\n:53:2: struct field RulesetId looks like a GitHub database ID but uses int; use int64 instead\r\n:67:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n```\r\n\r\n
\r\n\r\n
\r\nLinter output AFTER this PR (0 violations)\r\n\r\n```\r\n(clean - no output)\r\n```\r\n\r\n
\r\n\r\n### Reviewer Notes\r\n\r\nWe should run A/C tests on this before merging.", "test_files": ["pkg/cmd/codespace/create_test.go"], "test_cmd": "go test ./pkg/cmd/codespace/...", "code_files": ["internal/codespaces/api/api.go", "pkg/cmd/agent-task/capi/job.go", "pkg/cmd/cache/delete/delete.go", "pkg/cmd/cache/shared/shared.go", "pkg/cmd/codespace/common.go", "pkg/cmd/codespace/create.go", "pkg/cmd/codespace/mock_api.go", "pkg/cmd/gpg-key/delete/delete.go", "pkg/cmd/gpg-key/delete/http.go", "pkg/cmd/repo/autolink/shared/autolink.go", "pkg/cmd/repo/deploy-key/list/http.go", "pkg/cmd/repo/deploy-key/list/list.go", "pkg/cmd/ruleset/list/list.go", "pkg/cmd/ruleset/shared/shared.go", "pkg/cmd/ruleset/view/view.go", "pkg/cmd/skills/publish/publish.go", "pkg/cmd/ssh-key/list/list.go", "pkg/cmd/ssh-key/shared/user_keys.go"], "additions": 61, "verified": "fails at parent, passes with the fix"} -{"pr": 13393, "title": "fix(copilot): hint to run copilot directly when exec fails", "base_commit": "9b505c3fb8c3e52100575adbad2da7c4714684d8", "merge_commit": "fe996d33ac852eafdb2769fcba063860be1b4e6a", "prompt": "fix(copilot): hint to run copilot directly when exec fails\n\n## Summary\n\nWhen `gh copilot` finds the `copilot` binary but fails to execute it (e.g., due to unusual characters like parentheses in the file path on Windows), the error message now includes a hint:\n\n```\nFailed to run '', try running `copilot` directly without `gh`.\n```\n\n## Details\n\nOn Windows, `cmd.exe` shims (e.g., those planted by VS Code or npm) can fail when the path contains special characters like `(`. Since `gh copilot` invokes the binary via `os/exec`, these failures surface as opaque errors. This change adds a practical workaround hint so users can still use Copilot CLI directly.\n\nRelated to #13106", "test_files": ["pkg/cmd/copilot/copilot_test.go"], "test_cmd": "go test ./pkg/cmd/copilot/...", "code_files": ["pkg/cmd/copilot/copilot.go"], "additions": 45, "verified": "fails at parent, passes with the fix"} -{"pr": 13362, "title": "Bump `gh copilot` telemetry sampling to 100%", "base_commit": "3c162a78efc4cf48f6582f261223166af2558a64", "merge_commit": "6176c3eee4b70d0ccf3c6a1503035d049ca4e81b", "prompt": "Bump `gh copilot` telemetry sampling to 100%\n\n## Description\r\n\r\nFixes https://github.com/github/cli/issues/1163\r\n\r\nBumps the telemetry sampling rate for the `gh copilot` command from 1% to 100%, to allow for better analysis of usage.", "test_files": ["pkg/cmd/copilot/copilot_test.go"], "test_cmd": "go test ./pkg/cmd/copilot/...", "code_files": ["pkg/cmd/copilot/copilot.go", "pkg/cmd/root/root.go"], "additions": 24, "verified": "fails at parent, passes with the fix"} -{"pr": 13068, "title": "Print `gh auth refresh` for 401 returns", "base_commit": "49d4747a81425418dd2e7bee7e33541b7c644ca9", "merge_commit": "7c439196c1fc4d645d091e83029370cef845587c", "prompt": "Print `gh auth refresh` for 401 returns\n\n`gh auth refresh` exists to make it simpler for users to refresh their tokens on expiration/scope mismatch, but help messages only suggest using it in limited scenarios, and not in a common case of a token expiring and the user receiving a 401 error. Now, the auth flow will detect this case, and for refreshable tokens (namely, tokens created by logging in with `gh auth login` in the first place), it will suggest using `gh auth refresh` for these cases.\r\n\r\nFixes https://github.com/cli/cli/issues/13066.", "test_files": ["internal/ghcmd/cmd_test.go", "pkg/cmd/auth/status/status_test.go"], "test_cmd": "go test ./internal/ghcmd/... ./pkg/cmd/auth/status/...", "code_files": ["internal/ghcmd/cmd.go", "pkg/cmd/auth/shared/writeable.go", "pkg/cmd/auth/status/status.go"], "additions": 108, "verified": "fails at parent, passes with the fix"} -{"pr": 13057, "title": "Add Issues 2.0 support: issue types, sub-issues, and relationships", "base_commit": "3c162a78efc4cf48f6582f261223166af2558a64", "merge_commit": "20147bf81fed534faf1b5c9be12638983b8e5e82", "prompt": "Add Issues 2.0 support: issue types, sub-issues, and relationships\n\n# Add Issues 2.0 support: issue types, sub-issues, and relationships\n\nCloses #10298\nCloses #9696\nCloses #11757\nCloses #12477\nCloses #12152\n\nAdds support for GitHub's Issues 2.0 features across `gh issue create`, `edit`, `view`, and `list`.\n\n## New flags\n\n### `gh issue create`\n\n| Flag | Description |\n|------|-------------|\n| `--type Bug` | Set the issue type |\n| `--parent 100` | Add as a sub-issue of the specified parent (number or URL) |\n| `--blocked-by 200,201` | Mark as blocked by these issues |\n| `--blocking 300` | Mark as blocking these issues |\n\nInteractive mode prompts for issue type when the repo has types configured.\n\n### `gh issue edit`\n\n| Flag | Description |\n|------|-------------|\n| `--type Bug` | Set the issue type |\n| `--set-parent 100` | Set or replace the parent issue |\n| `--remove-parent` | Remove the parent issue |\n| `--add-sub-issue 123,124` | Add sub-issues |\n| `--remove-sub-issue 123` | Remove a sub-issue |\n| `--add-blocked-by 200` | Add blocked-by relationship |\n| `--remove-blocked-by 201` | Remove blocked-by relationship |\n| `--add-blocking 300,301` | Add blocking relationships |\n| `--remove-blocking 300` | Remove blocking relationship |\n\nInteractive mode adds Type and Parent to the field picker. Batch editing\nsupported for `--type` across multiple issues.\n\n### `gh issue view`\n\nTTY output now displays:\n- Issue type on the state line (e.g., `Bug \u00b7 Open`)\n- Type, Parent, Blocked by, Blocking metadata rows\n- Sub-issues section with completion progress (e.g., `Sub-issues \u00b7 3/5 (60%)`)\n\nJSON output supports new fields: `issueType`, `parent`, `subIssues`,\n`subIssuesSummary`, `blockedBy`, `blocking`. The `subIssues`, `blockedBy`,\nand `blocking` fields are objects of the shape `{ \"nodes\": [...], \"totalCount\": N }`\n(rather than flat arrays) so consumers can detect when the truncation\nlimit is reached. Each node and the `parent` object include the canonical\n`id`.\n\n### `gh issue list`\n\n| Flag | Description |\n|------|-------------|\n| `--type Bug` | Filter issues by type |\n\n## GHES compatibility\n\n- **Issue types and sub-issues**: GA on all supported GHES versions (3.17+). No feature detection needed.\n- **Relationships** (blocked-by/blocking): Requires GHES 3.19+. Feature detection via `__type` schema introspection for the `blockedBy` field on the Issue type. Unsupported GHES versions receive a clear error message. Gated with `// TODO IssueRelationshipsCleanup` for cleanup when GHES 3.18 support ends.\n\n## Implementation details\n\n- Post-creation mutations for type, parent, and relationships (same pattern as projectV2 and assignee mutations)\n- `--blocking` and `--add-blocking` swap API arguments: calls `addBlockedBy(issueId=OTHER, blockingIssueId=THIS)`\n- Cross-repo references supported via URL for `--parent`, `--blocked-by`, `--blocking`, and sub-issue flags\n- Shared helpers extracted to `` (`ResolveIssueRef`, `ResolveIssueTypeName`)\n- Issue-only fields (Type, Parent) gated behind `Allowed` in the Editable pattern to prevent leaking into `gh pr edit`\n- Issue type names with spaces are quoted in search queries (`type:\"Bug Report\"`)\n\n## Testing\n\n- **34 new flag parsing tests** across create, edit, and list\n- **18 new HTTP mock behavior tests** covering mutations, error cases, and GHES unsupported paths\n- **5 new view tests** (TTY rendering, JSON export)\n- **2 regression tests** (issue-only fields don't leak into PR edit, no-new-fields view unchanged)\n\n## Changes\n\n17 files changed, 2105 insertions(+), 18 deletions(-)\n\n| File | Change |\n|------|--------|\n| `` | New types, Issue struct fields, mutation functions, helpers |\n| `` | GraphQL fragment cases, `issueOnlyFields` |\n| `` | ExportData cases for new fields |\n| `` | `IssueRelationshipsSupported` detection |\n| `` | New flags, post-creation mutations, interactive type picker |\n| `` | New flags, mutations, interactive Type/Parent picker |\n| `` | TTY rendering, conditional relationship fields |\n| `` | `--type` filter via search path |\n| `` | Shared `ResolveIssueRef`, `ResolveIssueTypeName` |\n| `` | `IssueType`/`Parent` fields, `Allowed` gating |\n| `` | `IssueType` in `FilterOptions` and search query |\n\n## Key notes for reviewers\n\n- **`--blocking` arg swap**: The GitHub API only has `addBlockedBy(issueId, blockingIssueId)`. To support `--blocking`, we swap the arguments \u2014 the OTHER issue becomes `issueId` and THIS issue becomes `blockingIssueId`. Same for `--remove-blocking`.\n- **Editable `Allowed` gating**: `IssueType` and `Parent` are added to the shared `Editable` struct but gated behind `Allowed = true`, which only the issue edit command sets. This prevents these issue-only fields from leaking into `gh pr edit`'s interactive picker. Regression test in ``.\n- **View relationship fields are GHES-gated**: `blockedBy` and `blocking` are only added to the GraphQL query when `IssueRelationshipsSupported` is true. Without this, `gh issue view` would break on GHES 3.17\u20133.18 where these fields don't exist.\n- **Batch edit type resolution**: When editing multiple issues with `--type`, the type name is resolved to an ID once before the loop rather than per-issue, avoiding N-1 redundant `RepoIssueTypes` API calls.\n\n## Manual test results\n\nAll scenarios tested against on github.com.\n\n24/24 scenarios passing (click to expand)\n\n### Issue types\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 1 | `create-with-type` | `gh issue create --title \"...\" --type Bug` | \u2705 [#1](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/1) created, type=Bug verified |\n| 2 | `create-error-invalid-type` | `gh issue create --title \"Oops\" --type Bugz` | \u2705 Error: `type \"Bugz\" not found; available types: Task, Bug, Feature` |\n| 3 | `edit-change-type` | `gh issue edit 1 --type Feature` | \u2705 Type changed to Feature verified |\n| 4 | `create-interactive-type` | `gh issue create` (interactive) | \u2705 [#21](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/21) Type picker shown, Bug selected and set |\n\n### Sub-issues\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 5 | `create-with-parent` | `gh issue create --title \"...\" --parent 3` | \u2705 [#4](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/4) created as sub-issue of #3 |\n| 6 | `edit-set-parent` | `gh issue edit 5 --set-parent 3` | \u2705 Parent set, verified via `--json parent` |\n| 7 | `edit-remove-parent` | `gh issue edit 5 --remove-parent` | \u2705 Parent removed, verified null |\n| 8 | `edit-add-sub-issues` | `gh issue edit 3 --add-sub-issue 6,7` | \u2705 Sub-issues added, verified via `--json subIssues` |\n| 9 | `edit-remove-sub-issue` | `gh issue edit 3 --remove-sub-issue 6` | \u2705 Sub-issue removed |\n| 10 | `edit-error-mutual-parent` | `gh issue edit 5 --set-parent 1 --remove-parent` | \u2705 Error: `specify only one of --set-parent or --remove-parent` |\n\n### Relationships\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 11 | `create-with-relationships` | `gh issue create --blocked-by 8 --blocking 9` | \u2705 [#11](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/11) blockedBy=#8, blocking=#9 verified |\n| 12 | `edit-add-remove-blocked-by` | `gh issue edit 14 --add-blocked-by 12 --remove-blocked-by 13` | \u2705 Relationships updated |\n| 13 | `edit-add-blocking` | `gh issue edit 14 --add-blocking 15,16` | \u2705 Blocking relationships added (swapped args verified) |\n\n### View\n\n| # | Scenario | Verification | Result |\n|---|----------|-------------|--------|\n| 14 | `view-with-type` | TTY shows `Bug \u00b7 Open` on state line, `Type: Bug` row | \u2705 |\n| 15 | `view-parent-sub-issues` | TTY shows Parent row, Sub-issues section with progress | \u2705 |\n| 16 | `view-child-parent` | TTY shows parent reference for child issue | \u2705 |\n| 17 | `view-relationships` | TTY shows Blocked by and Blocking rows | \u2705 |\n| 18 | `view-json-issueType` | `--json issueType` returns id, name, description, color | \u2705 |\n| 19 | `view-json-parent-subIssues` | `--json parent,subIssues,subIssuesSummary` returns structured data | \u2705 |\n| 20 | `view-json-relationships` | `--json blockedBy,blocking` returns `{ nodes: [...], totalCount: N }` with id, number, title, url, state | \u2705 |\n\n### List\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 21 | `list-filter-by-type` | `gh issue list --type Bug` | \u2705 Filtered results returned |\n\n### Combined\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 22 | `create-all-flags` | `gh issue create --type Task --parent 3 --blocked-by 8 --blocking 9` | \u2705 [#17](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/17) all fields set |\n| 23 | `edit-batch-type` | `gh issue edit 18 19 20 --type Bug` | \u2705 All 3 issues updated to Bug |\n| 24 | `edit-interactive` | `gh issue edit 21` (interactive) | \u2705 Picker shows Type and Parent fields |\n\n**TODO**: GHES-specific scenarios (`create-error-ghes-relationships`), but GHES feature detection does have unit test coverage with mock introspection responses.", "test_files": ["acceptance/acceptance_test.go", "api/export_pr_test.go", "internal/featuredetection/feature_detection_test.go", "pkg/cmd/issue/create/create_test.go", "pkg/cmd/issue/edit/edit_test.go", "pkg/cmd/issue/list/list_test.go", "pkg/cmd/issue/view/view_test.go", "pkg/cmd/pr/shared/params_test.go", "pkg/cmd/pr/shared/survey_test.go", "pkg/search/query_test.go"], "test_cmd": "go test ./acceptance/... ./api/... ./internal/featuredetection/... ./pkg/cmd/issue/create/... ./pkg/cmd/issue/edit/... ./pkg/cmd/issue/list/... ./pkg/cmd/issue/view/... ./pkg/cmd/pr/shared/... ./pkg/search/...", "code_files": ["api/export_pr.go", "api/queries_issue.go", "api/query_builder.go", "internal/featuredetection/feature_detection.go", "pkg/cmd/issue/create/create.go", "pkg/cmd/issue/edit/edit.go", "pkg/cmd/issue/list/list.go", "pkg/cmd/issue/shared/lookup.go", "pkg/cmd/issue/view/view.go", "pkg/cmd/pr/edit/edit.go", "pkg/cmd/pr/shared/editable.go", "pkg/cmd/pr/shared/params.go", "pkg/search/query.go"], "additions": 2925, "verified": "fails at parent, passes with the fix"} +{"pr": 13988, "title": "Fix `RESTWithNext` error type, repairing `gh status` and attestation retries", "base_commit": "55dbb4dc6b7edb10b48e3d7fc5bccd32318d1b55", "merge_commit": "e83adbc0642994fae7c39a9a012eb34b8c81f4f1", "prompt": "Fix `RESTWithNext` error type, repairing `gh status` and attestation retries\n\n## Description\r\n\r\nThe `api.Client.RESTWithNext` method did not handle errors in the same way as `api.Client.REST`. The latter wrapped the `go-gh` error into a new `api` error and the former simply returned it. From an error message point of view, there was no visible difference but programmatic checks for `errors.As(err, &api.HTTPError{})` would be true for `REST` and false for `RESTWithNext`.\r\n\r\nI cannot find any reason for this to have been intentional. I suspect what happened was that `RESTWithNext` previously used to handle this case via the following removed code:\r\n\r\n```go\r\n\tsuccess := resp.StatusCode >= 200 && resp.StatusCode < 300\r\n\tif !success {\r\n\t\treturn \"\", HandleHTTPError(resp)\r\n\t}\r\n```\r\n\r\nBut at some point `go-gh` was created, and the `go-gh` api client `Request`/`RequestWithContext` methods began returning errors for non success http status codes.\r\n\r\nThis broke two behaviours:\r\n * `gh status` is supposed to gracefully degrade if notifications or events are inaccessible.\r\n * `gh attestation` commands is supposed to retry attestation fetching for 500 status codes.\r\n\r\n## Notes\r\n\r\nThis is a prerequisite for consolidating GitHub API URL derivation onto `api.Client` (part of the groundwork for per-host API routing, #13717 / #13760), but it is a user-facing bugfix in its own right and is worth reviewing as such.", "test_files": ["api/client_test.go", "pkg/cmd/attestation/api/client_test.go", "pkg/cmd/status/status_test.go"], "test_cmd": "go test ./api/... ./pkg/cmd/attestation/api/... ./pkg/cmd/status/...", "code_files": ["api/client.go"], "additions": 156, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13987, "title": "Replace Windsurf with Devin in skill agents", "base_commit": "b1a9f7b183c12ae111ea63048d49aad2a53e9101", "merge_commit": "89f3699581659f514c4276bef0820d8caadbb67d", "prompt": "Replace Windsurf with Devin in skill agents\n\nFixes #13986\n\n- Replace the `windsurf` agent entry with `devin`\n- Install Devin skills in `.devin/skills` at project and user scope\n- Update install help text and registry tests\n\n## Testing\n\n- `go test ./internal/skills/... ./pkg/cmd/skills/...`", "test_files": ["internal/skills/registry/registry_test.go"], "test_cmd": "go test ./internal/skills/registry/...", "code_files": ["internal/skills/registry/registry.go", "pkg/cmd/skills/install/install.go"], "additions": 25, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13967, "title": "Fix skill picker label wrapping", "base_commit": "01b79dd983af0859e4e3d7454961ad3f08cf88b4", "merge_commit": "c2ad3b0eb7ead66eec3a8a61239e10a765332ec7", "prompt": "Fix skill picker label wrapping\n\nTruncates skill picker labels to the available terminal width so descriptions do not wrap onto a separate line.\r\n\r\nAdds coverage for multiple terminal widths.\r\n\r\n**Before**\r\n\"Screenshot\r\n\r\n**After (Examples with varying terminal width)**\r\n\"Screenshot\r\n\"Screenshot\r\n\"Screenshot", "test_files": ["pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./pkg/cmd/skills/install/...", "code_files": ["pkg/cmd/skills/install/install.go"], "additions": 50, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["multiSelectLabelMargin"]} +{"pr": 13864, "title": "Add Grok skill host support", "base_commit": "2b970995a3c63b7a7600f242d55d15185a82ed8b", "merge_commit": "9d95dd7344abe6388b59e25b8db54a3c7371cd6a", "prompt": "Add Grok skill host support\n\n## Summary\n- add Grok as a supported `gh skill` agent host\n- install project and user skills in `.grok/skills`\n\nFixes #13846", "test_files": ["internal/skills/registry/registry_test.go"], "test_cmd": "go test ./internal/skills/registry/...", "code_files": ["internal/skills/registry/registry.go", "pkg/cmd/skills/install/install.go", "pkg/cmd/skills/update/update.go"], "additions": 26, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13823, "title": "Add named field columns to gh project item-list", "base_commit": "bba0816a3808c7f7ea0a7e75be9d365b3410c524", "merge_commit": "efe3f165dd297c85fff11473dbf586f2d39fbf86", "prompt": "Add named field columns to gh project item-list\n\nProposal: cli/cli#13816\nTracking issue: github/planning-tracking#3431\n\n## What this does\n\nAdds named field columns to `gh project item-list`:\n\n```\ngh project item-list --owner --field \"Status\"\n```\n\n- `--field ` (repeatable) resolves the field by name and adds a column showing each item's value for it, with no separate field-ID preflight lookup.\n- `--field-id ` does the same by node ID. The two are mutually exclusive, and an unknown `--field-id` lists candidate fields as `name (id)` for symmetry with the name-based errors.\n- Name matching is case-insensitive, mirroring the Projects v2 platform API.\n\n## Relationship to cli/cli#13807\n\nThis builds on the same shared name-resolution layer as the `item-edit` PR cli/cli#13807. To keep the two PRs independently mergeable, each one adds its own copy of the small, self-contained field-resolver file `` (~90 lines: `ResolveFieldByName`, `ResolveFieldByID`, and their error types), with identical contents in both. Because the copies match exactly, Git merges them into a single file automatically, so the two PRs can land in either order without conflicts or duplicate-definition build errors. The item-list-only `FieldValueNodes.DisplayValue` helper lives in its own ``, so this PR leaves `` untouched.", "test_files": ["pkg/cmd/project/item-list/item_list_test.go", "pkg/cmd/project/shared/queries/queries_test.go", "pkg/cmd/project/shared/queries/resolve_fields_test.go"], "test_cmd": "go test ./pkg/cmd/project/item-list/... ./pkg/cmd/project/shared/queries/...", "code_files": ["pkg/cmd/project/item-list/item_list.go", "pkg/cmd/project/shared/queries/queries.go", "pkg/cmd/project/shared/queries/resolve_fields.go"], "additions": 813, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["fields", "DisplayValue"]} +{"pr": 13807, "title": "Add name-based resolution to gh project item-edit", "base_commit": "9d95dd7344abe6388b59e25b8db54a3c7371cd6a", "merge_commit": "d1897f89b294fc172e5b2993325ad1a0184f711d", "prompt": "Add name-based resolution to gh project item-edit\n\nProposal: #13816\nTracking issue: github/planning-tracking#3431\n\n> **Companion PR:** The `item-list` read path ships as a separate, independently-mergeable PR \u2014 #13823. Both PRs add their own copy of the small `` file, with identical contents. Because the two copies match exactly, Git merges them into one automatically, so the PRs can land in either order without conflicts.\n\n## What this does\n\nLets `gh project item-edit` address ProjectV2 fields, single-select options, and items by **name** instead of node IDs. Existing ID-based flags are fully backward compatible; the name flags are additive and optional.\n\n### `item-edit` (new name-based mode)\nMirrors `item-add` addressing:\n\n```\ngh project item-edit --owner --url --field \"Status\" --value \"In Progress\"\n```\n\n- `[]` positional + `--owner` resolve the project.\n- `--url` resolves the item within that project.\n- `--field ` resolves the field; `--value ` is dispatched by the field's data type (text/number/date, and for single-select the value is resolved as an option **name**).\n\nhttps://github.com/user-attachments/assets/4e7dceb1-0e4f-4706-9d95-37570a56cdcd\n\n### Mutual exclusivity (enforced in code)\nEvery name/ID pair fails clearly when both are set, never silently merged: `--field` xor `--field-id`, `--url` xor `--id`, and `--value` xor the typed value flags (`--text`/`--number`/`--date`/`--single-select-option-id`/`--iteration-id`). Name-based flags also require the project number argument. Invalid combos are rejected before any GraphQL request fires.\n\n### Errors are self-correctable\nName resolution happens before the ID-typed GraphQL mutation, so a bad name fails loudly with candidate names rather than `400 []`. Six failure modes mirror the sibling MCP PR (github/github-mcp-server#2760): field-not-found, field-ambiguous, option-not-found, option-ambiguous, wrong-field-type, item-not-in-project \u2014 surfaced as human-readable stderr messages listing candidates (CLI convention; not JSON).\n\n### Name matching is case-insensitive\nField and option names are matched with `strings.EqualFold` to mirror the Projects v2 platform API, which normalizes field names via a `name_slug` and downcases single-select option names before matching. Field names are case-insensitively unique on the platform; option names are not, so a case-only option collision still surfaces the `option_ambiguous` error.\n\n## Backwards compatibility\nThe command previously marked `--id` as a required cobra flag; this PR removes that requirement so an item can also be addressed by `--url`, but `--id` remains required in effect via a runtime guard (one of `--id` or `--url` must be set). Existing `--id`-based invocations are unchanged.\n\n## Out of scope / follow-ups\n- **`item-list` named columns** \u2014 ships as a separate, independently-mergeable PR (#13823).\n- MultiSelect and issue-field-value node types: the shared resolver dispatches option resolution by field data type so adding these later is additive, not a rewrite.\n- Bounding/paginating large read payloads (github/planning-tracking#3332).\n\n## Coordination with the sibling MCP PR (github/github-mcp-server#2760)\n- **Case-insensitive matching:** this PR matches names case-insensitively (see above). MCP #2760's resolver currently does case-sensitive matching and should be aligned so both surfaces behave identically.\n- **Read-path mutual exclusivity:** the companion `item-list` PR (#13823) enforces reject-both for its name/ID field selector. MCP #2760's read path currently silently merges `fields`+`field_names`; that should be aligned to the same reject-both behavior.", "test_files": ["pkg/cmd/project/item-edit/item_edit_test.go", "pkg/cmd/project/shared/queries/resolve_fields_test.go", "pkg/cmd/project/shared/queries/resolve_test.go"], "test_cmd": "go test ./pkg/cmd/project/item-edit/... ./pkg/cmd/project/shared/queries/...", "code_files": ["pkg/cmd/project/item-edit/item_edit.go", "pkg/cmd/project/shared/queries/queries.go", "pkg/cmd/project/shared/queries/resolve.go", "pkg/cmd/project/shared/queries/resolve_fields.go"], "additions": 1432, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["ResolveFieldByName", "number32"]} +{"pr": 13784, "title": "Support antigravity-cli and antigravity2.0 in gh skill", "base_commit": "397876d3dc2828f1c932c651b31ab78e72506da4", "merge_commit": "326faaac8b5c3b736ddc6bfa573cc97dc6452a24", "prompt": "Support antigravity-cli and antigravity2.0 in gh skill\n\n### Description\n\n`gh skill` recognized only one Antigravity surface. This registers `antigravity-cli` and `antigravity2.0` as skill agent hosts across every `gh skill` subcommand, at both project and user scope.\n\nFixes #13767\n\n### Key Points\n\nAntigravity ships several surfaces that share one project skills dir but read user-scope skills from different dirs. The two new [registry entries](https://github.com/cli/cli/blob/8c6bac8c9/internal/skills/registry/registry.go#L91-L104) map each surface to its documented path:\n\n```go\n// Both share the .agents/skills project dir; user dirs differ per surface.\nUserDir: \".gemini/antigravity-cli/skills\" // antigravity-cli\nUserDir: \".gemini/config/skills\" // antigravity2.0\n```\n\nBecause both reuse `.agents/skills` for the project dir, selecting them next to other `.agents/skills` agents still writes each skill once. Nothing else needs wiring: `--agent`, `--scope`, `InstallDir`, help text, and enum suggestions all derive from the registry slice.\n\n### Notes for reviewers\n\nStart in the [`registry` package](https://github.com/cli/cli/blob/8c6bac8c9/internal/skills/registry/registry.go), where the `AgentHost` entries live. The [registry tests](https://github.com/cli/cli/blob/8c6bac8c9/internal/skills/registry/registry_test.go) extend the existing `TestFindByID` and `TestInstallDir` tables with project- and user-scope cases for both agents.\n\nThe user-scope paths come from Antigravity's own docs, confirmed by hands-on testing (see Additional Context). `.gemini/config/skills` is the one global dir every surface reads, so `antigravity2.0` uses it; the CLI reads its own `.gemini/antigravity-cli/skills`.\n\n### Additional Context\n\n- [Antigravity skills docs](https://antigravity.google/docs/skills) give the official per-surface skill directories.\n- [Where does Antigravity look for Agent Skills?](https://atamel.dev/posts/2026/07-01_where_agy_agent_skills/) verifies by experiment which dir each surface actually reads.\n- [Choosing your surface](https://cloud.google.com/blog/topics/developers-practitioners/choosing-your-surface-antigravity-20-antigravity-cli-antigravity-ide-or-antigravity-sdk) explains the 2.0 / CLI / IDE split.", "test_files": ["internal/skills/registry/registry_test.go"], "test_cmd": "go test ./internal/skills/registry/...", "code_files": ["internal/skills/registry/registry.go"], "additions": 53, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13766, "title": "fix(skills): honor --dir without agent prompt", "base_commit": "ff4e4a255006ca98e3036bd0f8d98374a405835f", "merge_commit": "f1d112104821b055bb0c5656f2989f9213db71f6", "prompt": "fix(skills): honor --dir without agent prompt\n\n## Summary\n- skip the target-agent prompt when `gh skill install` is given `--dir`\n- keep using the default agent host internally only for shared post-install messaging / telemetry\n- add regression coverage for the interactive `--dir` path\n\nFixes #13763.\n\n## Testing\n- not run locally (avoided local Go test/build in this environment)", "test_files": ["pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./pkg/cmd/skills/install/...", "code_files": ["pkg/cmd/skills/install/install.go"], "additions": 34, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13723, "title": "Allow downloading release assets without authentication", "base_commit": "954ffc37e5931a9621868a50f05c8fb17e288f19", "merge_commit": "71fb4f5f4358c44c9328637752f0c15a47d84139", "prompt": "Allow downloading release assets without authentication\n\nFixes https://github.com/cli/cli/issues/2680\n\n### Description\n\n`gh release download` now works without authentication against public repositories, matching `gh extension install`. A token is still used when one is present.\n\n### Key Points\n\nThis change has three parts, from simplest to subtlest:\n\n- **Why drop the auth gate** - the login requirement is unnecessary for public downloads.\n- **Fixing the by-tag race to be success oriented** - an anonymous draft-lookup 403 no longer masks a release the REST lookup found.\n- **Honoring the gate under repo override** - the cobra plumbing fix that makes the dropped gate apply to `download`.\n\n#### Why drop the auth gate\n\nRelease endpoints for public repositories are readable anonymously over REST, so the login gate is unnecessary. Removing it is the same one-line `cmdutil.DisableAuthCheck` change that #13176 made for `gh extension install`.\n\nThe same logic here applies as with `gh extension install`:\n\n> What this says is that extension installation is going to happen unauthenticated whether we make it easier or not. We might as well make it easier and keep people installing through our core commands.\n\nIn `gh release download` - people are going to just `curl` the endpoint directly, and that degrades their experience since we provide a lot of abstractions over the API that make `gh` much more than a `curl`/`gh api` call.\n\nBut I'll broaden the conversation - we got feedback that folks are using `gh release download` more in CI and they want to secure their CI by not providing `gh` a token unless it's necessary. If you're downloading a public asset, that's not necessary and we force you to use a larger than necessary token scope.\n\n#### Fixing the by-tag race to be success oriented\n\nThe by-tag path creates problems that make this deviate a bit from `gh ext install` - that's one reason why this diff is a bit bigger. [`FetchRelease`](https://github.com/cli/cli/blob/f0a128ad1c33a41b7d1bdc93b5d507c842713d8a/pkg/cmd/release/shared/fetch.go) races a published-release REST lookup against a draft-release GraphQL lookup. GraphQL rejects anonymous requests, so the draft lookup returns a 403. The old selector returned the first result unless it was exactly `ErrReleaseNotFound`, so that 403 could win the race and mask a release the REST lookup found.\n\nThe selector now prefers a found release and only errors when both lookups fail. We used to return whichever result arrived first, treating only a not-found error as a reason to wait for the second:\n\n```go\n// Before - a non-not-found error from either lookup wins\nres := <-results\nif errors.Is(res.error, ErrReleaseNotFound) {\n res = <-results\n cancel()\n} else {\n cancel()\n <-results // drain the channel\n}\nreturn res.release, res.error\n```\n\nNow we take a success from either lookup, and only surface an error when both fail:\n\n```go\n// Now - a found release wins; an error needs both lookups to fail\nfirst := <-results\nif first.error == nil {\n cancel()\n <-results // drain the channel\n return first.release, nil\n}\n\nsecond := <-results\ncancel()\nif second.error == nil {\n return second.release, nil\n}\nif errors.Is(second.error, ErrReleaseNotFound) {\n return nil, second.error\n}\nreturn nil, first.error\n```\n\nThis also stops a transient draft-lookup failure from masking a real release for authenticated users.\n\n#### Honoring the gate under repo override\n\nRemoving the gate did not take effect on its own. `release` enables the `-R/--repo` override, and that installs a `PersistentPreRunE` on the `release` command. cobra runs only the nearest such hook walking up from the command you invoked, so the override hook shadows the [root auth gate](https://github.com/cli/cli/blob/7b681a4e8b67d203ccdaab5ff54570bdf6ca3669/pkg/cmd/root/root.go). The override re-runs the nearest ancestor hook to make up for that, but it handed *the ancestor* to the hook as the command, so the auth gate judged `release` instead of `download` and never saw download's `DisableAuthCheck`.\n\nThe fix keeps the invoked command and passes that leaf up, the node cobra would have judged if the repo override didn't muck with it:\n\n```go\n// Before - climbs by reassigning cmd, so the gate is judged against the ancestor\nfor cmd.HasParent() {\n cmd = cmd.Parent()\n if cmd.PersistentPreRunE != nil {\n return cmd.PersistentPreRunE(cmd, args)\n }\n}\n\n// Now - keep the invoked leaf and pass it up\nfor p := overrideCmd.Parent(); p != nil; p = p.Parent() {\n if p.PersistentPreRunE != nil {\n return p.PersistentPreRunE(cmd, args)\n }\n}\n```\n\nThis mirrors [cobra's own `execute` loop](https://github.com/spf13/cobra/blob/v1.10.2/command.go#L984-L986), which walks the parents but always hands them the leaf command. cobra's `EnableTraverseRunHooks` global would run the whole chain for us and maybe is a better long term fix, but it is process-wide and would change pre-run behavior for every command, so I'm not touching it here.\n\nI guarded this with an integration test, since the bug only surfaces with the pieces wired together. [`Test_EnableRepoOverride_authCheckIntegration`](https://github.com/cli/cli/blob/7b681a4e8b67d203ccdaab5ff54570bdf6ca3669/pkg/cmdutil/repo_override_test.go) builds a root gate, a repo-override parent, and a leaf, then asserts the gate judged the leaf: opting out with `DisableAuthCheck` skips the check, otherwise the gate still runs. It lives in `cmdutil` beside the helper, since the coupling is the helper's, not any one command's.\n\n### Notes for reviewers\n\nThree atomic commits:\n\n- `fix(release): don't let a failed draft lookup mask a found release` changes the selector in [`FetchRelease`](https://github.com/cli/cli/blob/f0a128ad1c33a41b7d1bdc93b5d507c842713d8a/pkg/cmd/release/shared/fetch.go) and adds a regression case to the existing `Test_downloadRun` table.\n- `feat(release): allow download without authentication` removes the gate in the [download command](https://github.com/cli/cli/blob/f0a128ad1c33a41b7d1bdc93b5d507c842713d8a/pkg/cmd/release/download/download.go).\n- `fix(cmdutil): honor DisableAuthCheck under repo-override parents` fixes [the repo-override helper](https://github.com/cli/cli/blob/7b681a4e8b67d203ccdaab5ff54570bdf6ca3669/pkg/cmdutil/repo_override.go) so the gate removal actually takes effect, and adds an integration test that pins the coupling between repo override and the root auth gate.\n\n### Additional Context\n- #2680 is the standing request to allow unauthenticated public requests, and its motivating example is `gh release download` on a public repo.\n- #13176 removed the same gate for `gh extension install` and is the precedent this follows.", "test_files": ["pkg/cmd/release/download/download_test.go", "pkg/cmdutil/repo_override_test.go"], "test_cmd": "go test ./pkg/cmd/release/download/... ./pkg/cmdutil/...", "code_files": ["pkg/cmd/release/download/download.go", "pkg/cmd/release/shared/fetch.go", "pkg/cmdutil/repo_override.go"], "additions": 142, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13722, "title": "Detect additional third-party coding agents", "base_commit": "dd26eb39b04db5bd282509787d2b8fbff11ba694", "merge_commit": "561d9f919cbdadc22f886a3eadb260ba0b4f49c5", "prompt": "Detect additional third-party coding agents\n\n### Description\n\nExtends agent detection so the User-Agent header can identify more third-party coding agents driving `gh`. Adds nine new agents and an extra environment-variable signal for one existing one.\n\n### Key Points\n\nNew agents detected: antigravity, augment-cli, replit, goose, cowork, cursor, cursor-cli, kiro, and pi. One existing agent gains an extra signal: claude-code also matches `CLAUDE_CODE`.\n\nDetection order encodes confidence. `AI_AGENT` still wins, followed by agent-specific variables. Two signals are deliberately low-confidence and ranked accordingly:\n\n- `REPL_ID` is present throughout any Replit environment, not only when a Replit agent drives the CLI, so it is annotated as broad in the code.\n- `GOOSE_PROVIDER` only means Goose is configured as a model provider, not that it is running, so Goose is checked absolutely last and any stronger signal wins.\n\nThe single-source block now matches precisely instead of loosely: `kiro` is an exact `TERM_PROGRAM` match, and `pi` is anchored to a path boundary (`/.pi/agent`, or the Windows `\\.pi\\agent`) so an incidental substring cannot trigger it.\n\nWe considered a Devin signal and dropped it. The only reliable marker is the existence of a `/opt/.devin` directory, and this detector is intentionally environment-only with no filesystem access. The environment-variable alternatives (for example an `EDITOR` substring) could produce false positives, and Devin's developer-terminal footprint is small, so a noisy signal was not worth the telemetry pollution.\n\n### Notes for reviewers\n\nTwo files change. Start in the [`agents` detector](https://github.com/cli/cli/blob/1d4ca33b34646ce1165067377fd89f01148cbaf3/internal/agents/detect.go): the constant block lists the new identifiers, and `detectWith` runs the ordered checks top to bottom, returning on first match. Read it straight down to see the precedence. Then the [detector tests](https://github.com/cli/cli/blob/1d4ca33b34646ce1165067377fd89f01148cbaf3/internal/agents/detect_test.go) add table cases for every new agent and signal, plus the precedence and negative cases (exact `kiro`, path-boundary `pi`, Goose losing to stronger signals).\n\nFirst-party documentation for each signal lives inline next to its check, or is marked `// No first-party docs` where none exists.", "test_files": ["internal/agents/detect_test.go"], "test_cmd": "go test ./internal/agents/...", "code_files": ["internal/agents/detect.go"], "additions": 190, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13681, "title": "fix(skills): install universal agent to ~/.agents/skills", "base_commit": "70bb306bd25eb407f90eabefd98824aed62cf519", "merge_commit": "cd68c69dbd5ea0793cece6355f0911ca6a68ffc1", "prompt": "fix(skills): install universal agent to ~/.agents/skills\n\nThe Universal agent host installed user-scoped skills to ~/.config/agents/skills, which is not the location scanned by the majority of compliant clients (Copilot CLI, Pi, OpenCode).\n\nPer the agentskills.io cross-client convention (https://agentskills.io/client-implementation/adding-skills-support), user-level skills for compliant clients live under ~/.agents/skills.\n\nThis PR switches UserDir for the universal agent to the shared project skills directory constant, matching the convention already used by Warp and Cline. Adds regression tests covering both project and user scope.\n\nFixes #13494", "test_files": ["internal/skills/registry/registry_test.go"], "test_cmd": "go test ./internal/skills/registry/...", "code_files": ["internal/skills/registry/registry.go"], "additions": 20, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13679, "title": "fix: show checks summary when all checks were cancelled", "base_commit": "70bb306bd25eb407f90eabefd98824aed62cf519", "merge_commit": "23f83e6ae18998fcd6ac0497f57a1d9ebd95862a", "prompt": "fix: show checks summary when all checks were cancelled\n\n### Description\n\n`gh pr checks` prints a blank summary when a PR's only checks were **cancelled**.\n\n`printSummary` (``) gates the summary block on:\n\n```go\nif counts.Failed+counts.Passed+counts.Skipping+counts.Pending > 0 {\n```\n\n`counts.Canceled` is omitted, so a cancelled-only result sums to 0, the block is skipped, and the `else if counts.Canceled > 0 { \"Some checks were cancelled\" }` branch becomes unreachable in exactly the case it was written for. The existing \"some cancelled\" test masked this by pairing a cancelled check with passing ones.\n\n### Fix\n\nAdd `counts.Canceled` to the guard.\n\n### Testing\n\nAdded `TestPrintSummary` (``) with a cancelled-only case asserting the \"Some checks were cancelled\" summary renders. Fails before, passes after; `go test ./pkg/cmd/pr/checks/` green.\n\nReported in #13680.", "test_files": ["pkg/cmd/pr/checks/output_test.go"], "test_cmd": "go test ./pkg/cmd/pr/checks/...", "code_files": ["pkg/cmd/pr/checks/output.go"], "additions": 56, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13675, "title": "Fix flaky TestHuhPrompterMultiSelectWithSearchPersistence on slow architectures", "base_commit": "015e6cf6ac39ad18138f868545f02359c3c8d3ef", "merge_commit": "954ffc37e5931a9621868a50f05c8fb17e288f19", "prompt": "Fix flaky TestHuhPrompterMultiSelectWithSearchPersistence on slow architectures\n\nFixes https://github.com/cli/cli/issues/13727\r\n\r\nThe test was using a fixed 50ms sleep (`waitForOptions()`) to wait for an async bubbletea search goroutine before sending the next keystroke. On slow architectures such as s390x under QEMU emulation, the goroutine scheduling and event loop round-trip can exceed 50ms, causing the `enter` key to arrive while `m.loading` is still `true`. In that state the field silently drops all keystrokes, the form never advances, and the 5s overall deadline fires with \"form.Run() did not complete in time\".\r\n\r\nThe fix replaces the blind sleep with proper channel-based synchronization. An `onSearchDone func()` hook is added to `multiSelectSearchField` (nil in production, set only in tests) and called at the end of `applySearchResult`. A new test helper `waitForSearch(field)` wires a one-shot `sync.Once`-guarded channel into that hook and blocks until it fires - guaranteeing the search is truly complete before the next step is sent, regardless of scheduler timing or machine speed.\r\n\r\n## Error log\r\n```\r\n[ 770s] --- FAIL: TestHuhPrompterMultiSelectWithSearchPersistence (5.42s)\r\n[ 770s] --- FAIL: TestHuhPrompterMultiSelectWithSearchPersistence/selections_persist_after_changing_search_query (5.31s) Context\r\n[ 770s] :478: form.Run() did not complete in time 54,494 tokens\r\n[ 770s] FAIL 5% used\r\n[ 770s] FAIL github.com/cli/cli/v2/internal/prompter 11.841s\r\n```", "test_files": ["internal/prompter/huh_prompter_test.go"], "test_cmd": "go test ./internal/prompter/...", "code_files": ["internal/prompter/multi_select_with_search.go"], "additions": 69, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["onSearchDone"]} +{"pr": 13624, "title": "Gracefully handle failed GitHub verifier initialization caused by missing trusted root", "base_commit": "bd81afda9ab653bd880298e6cfbf143541522a2f", "merge_commit": "d51b72243a01729cdc76a5b57a0866a2f1bfd2bc", "prompt": "Gracefully handle failed GitHub verifier initialization caused by missing trusted root\n\n\r\n\r\nGracefully handle the failure to initialize the GitHub verifier when the trusted root is missing.", "test_files": ["pkg/cmd/attestation/verification/sigstore_integration_test.go", "pkg/cmd/attestation/verification/sigstore_test.go"], "test_cmd": "go test ./pkg/cmd/attestation/verification/...", "code_files": ["pkg/cmd/attestation/verification/sigstore.go"], "additions": 39, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13620, "title": "feat(discussion): add comment command", "base_commit": "4166ecf2cada209714af79387a5243b334af764b", "merge_commit": "42db02df0c55f8ae556b15ca353887ee4043cd88", "prompt": "feat(discussion): add comment command\n\n## Add `gh discussion comment` command\r\n\r\nThis PR adds a new `gh discussion comment` command for managing comments and replies on GitHub Discussions, along with UX improvements to the `view` command.\r\n\r\n### `gh discussion comment`\r\n\r\nThe command supports adding, editing, and deleting comments and replies. The positional argument determines the action context:\r\n\r\n**Adding a top-level comment:**\r\n```\r\ngh discussion comment 123 --body \"Thanks for raising this\"\r\ngh discussion comment 123 --body-file response.md\r\n```\r\n\r\n**Replying to a comment (positional arg is a comment ID or URL):**\r\n```\r\ngh discussion comment DC_abc123 --body \"Good point\"\r\ngh discussion comment \"https://github.com/OWNER/REPO/discussions/123#discussioncomment-456\" --body \"I agree\"\r\n```\r\n\r\n**Editing a comment or reply:**\r\n```\r\ngh discussion comment DC_abc123 --edit --body \"Updated text\"\r\ngh discussion comment \"https://github.com/OWNER/REPO/discussions/123#discussioncomment-456\" --edit --body-file updated.md\r\n```\r\n\r\n**Deleting a comment or reply:**\r\n```\r\ngh discussion comment DC_abc123 --delete --yes\r\ngh discussion comment \"https://github.com/OWNER/REPO/discussions/123#discussioncomment-456\" --delete --yes\r\n```\r\n\r\nWhen no `--body` or `--body-file` is provided in an interactive terminal, an editor is opened for input. For edits, the editor is pre-populated with the existing comment body.\r\n\r\nComment URLs are resolved to node IDs via msgpack encoding of the repository database ID and comment database ID extracted from the URL fragment.\r\n\r\n### Changes to `gh discussion view`\r\n\r\nThe `--replies` flag now accepts both comment node IDs and comment URLs:\r\n\r\n```\r\ngh discussion view 123 --replies DC_abc123\r\ngh discussion view 123 --replies \"https://github.com/OWNER/REPO/discussions/123#discussioncomment-456\"\r\n```\r\n\r\nWhen a URL is provided, the repository and discussion number from the URL override the positional argument.\r\n\r\n### Tests\r\n\r\n- Added acceptance tests for the `comment` command covering add, reply, edit (including from file), delete (with confirmation prompt check), and URL-based operations.\r\n- Updated existing acceptance tests (`discussion-view.txtar`, `discussion-list.txtar`) to use the new `comment` command instead of raw `gh api graphql` mutations.\r\n- Added a test case in `discussion-view.txtar` to verify `--replies` works with a comment URL.\r\n- Unit tests cover flag parsing, all run paths (add/reply/edit/delete), comment URL resolution, body-file/stdin input, editor pre-population, and error cases.", "test_files": ["pkg/cmd/discussion/client/client_impl_test.go", "pkg/cmd/discussion/comment/comment_test.go", "pkg/cmd/discussion/shared/lookup_test.go", "pkg/cmd/discussion/view/view_test.go"], "test_cmd": "go test ./pkg/cmd/discussion/client/... ./pkg/cmd/discussion/comment/... ./pkg/cmd/discussion/shared/... ./pkg/cmd/discussion/view/...", "code_files": ["pkg/cmd/discussion/client/client.go", "pkg/cmd/discussion/client/client_impl.go", "pkg/cmd/discussion/client/client_mock.go", "pkg/cmd/discussion/client/types.go", "pkg/cmd/discussion/comment/comment.go", "pkg/cmd/discussion/create/create.go", "pkg/cmd/discussion/discussion.go", "pkg/cmd/discussion/edit/edit.go", "pkg/cmd/discussion/shared/lookup.go", "pkg/cmd/discussion/view/view.go"], "additions": 2666, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["ParseDiscussionOrCommentArg", "CommentOptions", "CommentDatabaseID"]} +{"pr": 13581, "title": "Auto-install official extension stubs in CI", "base_commit": "46480afb125d852c3edd2b4850a97c09b65c64cf", "merge_commit": "7ef9c54bf87fb17516e93a7ddb932efb7c660a35", "prompt": "Auto-install official extension stubs in CI\n\n### Description\n\nSkip the install prompt when `CI` is detected so official extension stubs (like `gh aw`, `gh stack`) auto-install instead of blocking on `Y/n` in CI runners and agent sessions.\n\n### Key Points\n\n- The previous version of this diff mirrored [`runCopilot`](https://github.com/cli/cli/blob/trunk/pkg/cmd/copilot/copilot.go) exactly: TTY always wins, so a pseudo-TTY agent session with `CI=1` still hung on the prompt. This revision flips the order so `CI` wins over TTY.\n- Intentional divergence from `gh copilot`: a follow-up could apply the same change there for consistency, but extension stubs are the more pressing case (agents call `gh ` frequently).\n- `CI` is detected via [`ci.IsCI()`](https://github.com/cli/cli/blob/trunk/internal/ci/ci.go) (any of `CI`, `BUILD_NUMBER`, or `RUN_ID`).\n\n### Notes for reviewers\n\nTwo commits, reviewable in order:\n\n1. [`fix(extension): auto-install official extension stubs in CI`](https://github.com/cli/cli/pull/13581/commits/ef8c7d8ab5c5539a5279f0c05d2d83d6a39f22be) is the behavior change. The [`official_extension_stub`](https://github.com/cli/cli/blob/ef8c7d8ab5c5539a5279f0c05d2d83d6a39f22be/pkg/cmd/root/official_extension_stub.go) file holds the new branching; the [stub test](https://github.com/cli/cli/blob/ef8c7d8ab5c5539a5279f0c05d2d83d6a39f22be/pkg/cmd/root/official_extension_stub_test.go) extends the existing run-function table test and clears CI env vars per subtest; the [acceptance test](https://github.com/cli/cli/blob/ef8c7d8ab5c5539a5279f0c05d2d83d6a39f22be/acceptance/testdata/telemetry/telemetry-for-official-extension-stub.txtar) clears CI env vars and expects non-zero exit plus install instructions on stderr.\n2. [`docs(extension): explain CI bypass in stub install logic`](https://github.com/cli/cli/pull/13581/commits/61a752068e271b699b67bdb39b90a10549adf332) is a one-line comment addressing review feedback that the non-TTY+CI behavior wasn't obvious from the code.\n\n### Additional Context\n\nManually verified locally with `gh stack` uninstalled. The TTY+CI case is the one this revision actually fixes versus the previous version of the diff:\n\n```\n$ CI=1 ./bin/gh stack --help # TTY + CI\nSuccessfully installed github/gh-stack # exit 0\n\n$ CI=1 ./bin/gh stack --help < /dev/null # non-TTY + CI\nSuccessfully installed github/gh-stack # exit 0\n\n$ ./bin/gh stack --help < /dev/null # non-TTY, no CI\ngh stack is available as an official extension.\nTo install it, run:\n gh extension install github/gh-stack # exit 1\n```", "test_files": ["pkg/cmd/root/official_extension_stub_test.go"], "test_cmd": "go test ./pkg/cmd/root/...", "code_files": ["pkg/cmd/root/official_extension_stub.go"], "additions": 57, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13580, "title": "feat: add `repo read-file` and `repo read-dir`", "base_commit": "d05c71a9da4bce4d57f622aacc6a8ee24e5e158e", "merge_commit": "733e35c04c4141468f8882d17b063414ffe36c57", "prompt": "feat: add `repo read-file` and `repo read-dir`\n\n## Summary\r\n\r\nThis PR adds two new (preview) commands for reading repository contents without cloning.\r\n\r\n### `gh repo read-file`\r\n\r\n```\r\ngh repo read-file [--ref ] [--output ] [--clobber] [--unsafe] [--json ] [--jq ] [--template ] [--repo ]\r\n```\r\n\r\nReads a single file from a repository and writes it to stdout (or to a file with `--output`).\r\n\r\n### `gh repo read-dir`\r\n\r\n```\r\ngh repo read-dir [] [--ref ] [--json ] [--jq ] [--template ] [--repo ]\r\n```\r\n\r\nLists the entries of a directory in a repository, showing each entry's type, name, and size.\r\n\r\nBoth commands are marked **preview**, default to the current repository's default branch (overridable with `--ref`), and support `--json`/`--jq`/`--template` for scripting.\r\n\r\n## Test plan\r\n\r\nYou can verify the command behaviour via acceptance tests and the following examples.\r\n\r\nThe commands below target public repositories so they can be run as-is.\r\n\r\n### `gh repo read-file`\r\n\r\n```sh\r\n# Read a file to stdout\r\ngh repo read-file README.md --repo cli/cli\r\n\r\n# Read from a specific tag/branch/commit\r\ngh repo read-file README.md --ref v2.0.0 --repo cli/cli\r\n\r\n# Write to a file, then confirm --clobber is required to overwrite\r\ngh repo read-file README.md --output --repo cli/cli\r\ngh repo read-file README.md --output --repo cli/cli # should error: already exists\r\ngh repo read-file README.md --output --clobber --repo cli/cli # should succeed\r\n\r\n# JSON output, with jq and template\r\ngh repo read-file README.md --json path,size,type --repo cli/cli\r\ngh repo read-file README.md --json path,size --jq '.size' --repo cli/cli\r\ngh repo read-file README.md --template '{{.path}} ({{.size}} bytes){{\"\\n\"}}' --repo cli/cli\r\n\r\n# Error: path does not exist\r\ngh repo read-file --repo cli/cli\r\n```\r\n\r\n### `gh repo read-dir`\r\n\r\n```sh\r\n# List the repository root\r\ngh repo read-dir --repo cli/cli\r\n\r\n# List a subdirectory (note executables render as `file*`)\r\ngh repo read-dir script --repo cli/cli\r\n\r\n# Mixed entry types in one listing: regular files, a symlink (RelNotes),\r\n# and a submodule (sha1collisiondetection)\r\ngh repo read-dir --repo git/git\r\n\r\n# List from a specific tag/branch/commit\r\ngh repo read-dir docs --ref v2.0.0 --repo cli/cli\r\n\r\n# Non-TTY / scriptable output (tab separated)\r\ngh repo read-dir script --repo cli/cli | cat\r\n\r\n# JSON output, with jq and template\r\ngh repo read-dir script --json name,type,mode,modeOctal --repo cli/cli\r\ngh repo read-dir script --json name,type --jq '.entries[] | select(.type==\"file\") | .name' --repo cli/cli\r\ngh repo read-dir --template '{{range .entries}}{{.type}}: {{.name}}{{\"\\n\"}}{{end}}' --repo cli/cli\r\n\r\n# Error: path points to a file, not a directory\r\ngh repo read-dir README.md --repo cli/cli\r\n\r\n# Error: path or ref does not exist\r\ngh repo read-dir nope/nope --repo cli/cli\r\n```", "test_files": ["internal/text/text_test.go", "pkg/cmd/repo/read-dir/read_dir_test.go", "pkg/cmd/repo/read-file/read_file_test.go"], "test_cmd": "go test ./internal/text/... ./pkg/cmd/repo/read-dir/... ./pkg/cmd/repo/read-file/...", "code_files": ["internal/text/text.go", "pkg/cmd/repo/read-dir/http.go", "pkg/cmd/repo/read-dir/read_dir.go", "pkg/cmd/repo/read-file/http.go", "pkg/cmd/repo/read-file/read_file.go", "pkg/cmd/repo/repo.go"], "additions": 2535, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["FormatSize", "NewCmdReadDir", "NewCmdReadFile"]} +{"pr": 13548, "title": "feat(skills): list available skills when install runs non-interactively", "base_commit": "01bcd474447b5034da20686c838ae4b0cf2b23f5", "merge_commit": "70bb306bd25eb407f90eabefd98824aed62cf519", "prompt": "feat(skills): list available skills when install runs non-interactively\n\n### Summary\n\nRunning `gh skill install ` (or `--from-local `) without a skill name in a non-interactive context previously errored with `must specify a skill name when not running interactively`. That made discovery awkward when piping or scripting \u2014 users had to switch to an interactive shell just to see what skills a repo offered.\n\nThis PR changes that behavior so the discovered skills are printed instead, mirroring (approximately) what the interactive picker shows. The list is rendered through `tableprinter`, so it auto-switches between a friendly TTY table and tab-separated `SKILL\\tDESCRIPTION` rows when piped \u2014 making it easy to grep / fzf / awk.\n\n### Before\n\n```console\n$ gh skill install github/awesome-copilot | grep review\nmust specify a skill name when not running interactively\n```\n\n### After\n\n```console\n$ gh skill install github/awesome-copilot | grep review\ncode-review Reviews pull requests\n\u2026\n```\n\nRe-running with a specific skill name installs as before. Same behavior for `--from-local`.\n\n### Context\n\nCame out of a Slack thread where this was identified as a UX gap: interactive mode supports browsing-then-installing, and non-interactive mode should provide the same discoverability so it composes with standard Unix tooling.\n\n### Changes\n\n- `selectSkillsWithSelector` now prints discovered skills via a new `listAvailableSkills` helper instead of erroring when `!canPrompt`.\n- Sentinel `errSkillsListed` lets `installRun` / `runLocalInstall` bail cleanly without going into the install path.\n- Descriptions are pre-fetched (same `fetchDescriptions` callback the picker uses) so the listed output is useful, not bare.\n- Help text + an example for piping into `grep` updated.\n- Tests: replaced the old \"errors\" case with a remote listing assertion and added an equivalent local install case.\n\n### Verification\n\n- `go test ./pkg/cmd/skills/...` \u2705\n- `make lint` \u2705", "test_files": ["pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./pkg/cmd/skills/install/...", "code_files": ["pkg/cmd/skills/install/install.go"], "additions": 105, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13541, "title": "feat: add `discussion` command set", "base_commit": "5c9ec1c5f51d682dbf65b0c16f856d8b9303adff", "merge_commit": "52e8e7427702e023d743318f748fceda4c0dcf37", "prompt": "feat: add `discussion` command set\n\nThis PR adds the `gh discussion` command group (list, view, create, edit) as a preview feature.\r\n\r\nHuge thanks to @maxbeizer for doing the heavy lifting on this. :copilot: was also instrumental in implementation, review, and polish throughout.\r\n\r\nI rewrote the commit history (81 commits squashed into 6) for easier review and a cleaner git log. Note that the commits are structured to split the work logically; intermediate commits may not compile independently.\r\n\r\n## Commands\r\n\r\n### `discussion list`\r\n\r\n```shell\r\ngh discussion list [--category ] [--state ] [--author ] [--label ] [--search ] [--limit ] [--json ] [--web]\r\n```\r\n\r\nList discussions in a repository with optional filters. ([8b73951](https://github.com/cli/cli/commit/8b73951ca75c6495a776e36da9fb48cc93000866))\r\n\r\n### `discussion view`\r\n\r\n```\r\ngh discussion view { | } [--comments [--limit ]] [--replies [--limit ]] [--json ] [--web]\r\n```\r\n\r\nView a discussion's details, comments, or replies to a specific comment. ([57008e7](https://github.com/cli/cli/commit/57008e797097e9b275ba7d1849ac750e762743e1))\r\n\r\n### `discussion create`\r\n\r\n```\r\ngh discussion create --title --body --category [--label ,...] [--body-file ]\r\n```\r\n\r\nCreate a new discussion, interactively or via flags. ([7bd67a8](https://github.com/cli/cli/commit/7bd67a840456436a1edc0f80dc85687f29b277af))\r\n\r\n### `discussion edit`\r\n\r\n```\r\ngh discussion edit { | } [--title ] [--body ] [--body-file ] [--category ] [--add-label ,...] [--remove-label ,...]\r\n```\r\n\r\nEdit a discussion's title, body, category, or labels, interactively or via flags. ([797effe](https://github.com/cli/cli/commit/797effe0295914e03f4b40cb873276da1d3b7e93))\r\n\r\n## Reusable components\r\n\r\n- **``**: GraphQL client for Discussions covering list, search, get, create, update, and label mutations with full pagination support. ([51b7653](https://github.com/cli/cli/commit/51b765381870dbf62390e0f7a297cc57ea771db7))\r\n- **``**: Shared utilities including argument parsing, category matching, label resolution, and client factory helpers. ([d3a1538](https://github.com/cli/cli/commit/d3a153872b9774009d620d0f3e12f993a21765eb))\r\n\r\n## Screenshots\r\n\r\n### `gh discussion list`\r\n\r\n![Terminal output showing a table of discussions with columns for number, state, title, category, author, and date](https://github.com/user-attachments/assets/094587eb-93f8-4b55-8bf9-5ed0965cc67d)\r\n\r\n### `gh discussion view 9094`\r\n\r\n![Terminal output showing discussion details including title, metadata, labels, and body text](https://github.com/user-attachments/assets/55dcf8c0-c4e0-4bc0-b073-8025c5490107)\r\n\r\n### `gh discussion view 9094 --comments --limit 1`\r\n\r\n![Terminal output showing a discussion with its threaded comments displayed below the body](https://github.com/user-attachments/assets/64ea48f7-2cd6-404e-b261-14eaf9a26eb1)\r\n\r\n### `gh discussion view 9094 --replies DC_kwDODKw3uc4AkHLS --limit 1`\r\n\r\n![Terminal output showing paginated replies to a specific discussion comment](https://github.com/user-attachments/assets/87bad4fe-82f3-4124-91c4-c581d9e2a0ef)\r\n\r\n### `gh discussion create` (non-interactive)\r\n\r\n![Terminal showing discussion creation via flags with title, body, category, and label options](https://github.com/user-attachments/assets/bf65e362-7983-4b97-af48-2840fa74c3b9)\r\n\r\n### `gh discussion create` (interactive, category selection)\r\n\r\n![Terminal showing an interactive prompt listing available discussion categories to choose from](https://github.com/user-attachments/assets/7fdcf50f-b74d-4e5d-87a3-9a05aa9bbc78)\r\n\r\n### `gh discussion create` (interactive)\r\n\r\n![Terminal showing interactive prompts for title and body input during discussion creation](https://github.com/user-attachments/assets/764ea1e8-2225-4c6f-9803-77e438e1d1fa)\r\n\r\n### `gh discussion edit` (interactive)\r\n\r\n1. Select fields to update\r\n![Terminal showing a multi-select prompt asking which fields to edit: title, body, category](https://github.com/user-attachments/assets/be05bef6-4dce-44bf-a78e-9b907ef70d46)\r\n\r\n2. Populate new values\r\n![Terminal showing input prompts pre-filled with current values for the selected fields](https://github.com/user-attachments/assets/b2bb8fa8-b0e0-475a-a7ff-a7d6fcf29b0b)\r\n\r\n3. Submit\r\n![Terminal showing the discussion URL printed after a successful edit](https://github.com/user-attachments/assets/9d8e9c31-020f-4961-8ce7-d43bf685ca0d)\r\n\r\n### `gh discussion edit` (non-interactive)\r\n\r\n![Terminal showing discussion edit via flags updating title, body, and labels in one command](https://github.com/user-attachments/assets/5fc60e1b-227a-4ee1-b5b7-2b6281561c5f)", "test_files": ["acceptance/acceptance_test.go", "pkg/cmd/discussion/client/client_test.go", "pkg/cmd/discussion/comment/comment_test.go", "pkg/cmd/discussion/create/create_test.go", "pkg/cmd/discussion/edit/edit_test.go", "pkg/cmd/discussion/list/list_test.go", "pkg/cmd/discussion/shared/labels_test.go", "pkg/cmd/discussion/shared/lookup_test.go", "pkg/cmd/discussion/view/view_test.go"], "test_cmd": "go test ./acceptance/... ./pkg/cmd/discussion/client/... ./pkg/cmd/discussion/comment/... ./pkg/cmd/discussion/create/... ./pkg/cmd/discussion/edit/... ./pkg/cmd/discussion/list/... ./pkg/cmd/discussion/shared/... ./pkg/cmd/discussion/view/...", "code_files": ["pkg/cmd/discussion/client/client.go", "pkg/cmd/discussion/client/client_mock.go", "pkg/cmd/discussion/client/types.go", "pkg/cmd/discussion/comment/comment.go", "pkg/cmd/discussion/create/create.go", "pkg/cmd/discussion/discussion.go", "pkg/cmd/discussion/edit/edit.go", "pkg/cmd/discussion/list/list.go", "pkg/cmd/discussion/shared/categories.go", "pkg/cmd/discussion/shared/client.go", "pkg/cmd/discussion/shared/labels.go", "pkg/cmd/discussion/shared/lookup.go", "pkg/cmd/discussion/view/view.go", "pkg/cmd/root/root.go"], "additions": 13075, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["DiscussionClient"]} +{"pr": 13523, "title": "Support custom CLAUDE_CONFIG_DIR in install", "base_commit": "9a593ce81b593dee752cc11737d1a3ef768e52b3", "merge_commit": "412c5fae63239705c0120e76ec9508307d2709a0", "prompt": "Support custom CLAUDE_CONFIG_DIR in install\n\n\r\n\r\nFixes https://github.com/cli/cli/issues/13262\r\n\r\n## Summary\r\n\r\nThis updates Claude Code user-scope skill directory resolution to honor `CLAUDE_CONFIG_DIR` when it is set. User-scope installs for Claude Code now write to `$CLAUDE_CONFIG_DIR/skills`, falling back to the existing `~/.claude/skills` location when the environment variable is not present.\r\n\r\nThe change lives in the shared agent registry path resolver so `gh skill install`, `gh skill list`, and `gh skill update` all use the same location for Claude Code user-scope skills.\r\n\r\n## Tests\r\n\r\n- Added registry coverage for Claude Code's default user-scope path and `CLAUDE_CONFIG_DIR` override.\r\n- Added an install regression test that verifies `gh skill install --agent claude-code --scope user` writes under `$CLAUDE_CONFIG_DIR/skills` and not `~/.claude/skills`.\r\n- Ran `go test ./internal/skills/registry ./pkg/cmd/skills/...`.", "test_files": ["internal/skills/registry/registry_test.go", "pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./internal/skills/registry/... ./pkg/cmd/skills/install/...", "code_files": ["internal/skills/registry/registry.go"], "additions": 69, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["claudeConfigDirEnv", "All"]} +{"pr": 13471, "title": "add --all flag to install all skills in a repo", "base_commit": "8bd56966ac5e2fe7d3162ad424dd11d877f4815e", "merge_commit": "e6fa2faf9e3203c2caaa54dcb1526c5f093fa998", "prompt": "add --all flag to install all skills in a repo\n\nFixes https://github.com/cli/cli/issues/13453\r\n\r\n## Summary\r\n- Add `--all` to `gh skill install` so users can install every discovered skill from a repository without providing an individual skill name.\r\n- Reuse the existing all-skills selection path, including collision detection, so skills that would overwrite each other still fail with guidance.\r\n- Update help text, examples, and tests for flag parsing, validation, and multi-skill remote installs.\r\n\r\n## Testing\r\n- `go test ./pkg/cmd/skills/install`", "test_files": ["pkg/cmd/skills/install/install_test.go"], "test_cmd": "go test ./pkg/cmd/skills/install/...", "code_files": ["pkg/cmd/skills/install/install.go"], "additions": 78, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["All"]} +{"pr": 13469, "title": "Improve update --all command if some skills don't have metadata", "base_commit": "81ed6d4e355446dbb4c1a7b038534948f7c195b8", "merge_commit": "5cfb648ace667b394728332d492b3e1a017baafa", "prompt": "Improve update --all command if some skills don't have metadata\n\n\r\n\r\nFixes https://github.com/cli/cli/issues/13369\r\n\r\n## Summary\r\n\r\nFixes `gh skill update --all` so it no longer prompts for skills that are missing GitHub metadata.\r\n\r\nWhen `--all` is used, skills without GitHub metadata are skipped with a notice, matching the flag's \"without prompting\" behavior.\r\n\r\n## Changes\r\n\r\n- Skip missing-metadata skills when `--all` is used instead of prompting interactively.\r\n- Keep the warning summary so users know which skills were skipped.\r\n- Update the warning text to explain that users can run `gh skill update ` interactively to add metadata, or reinstall the skill.\r\n- Add a regression test to verify `--all` does not prompt for missing metadata.\r\n\r\n## Testing\r\n\r\n- `go test ./pkg/cmd/skills/update/... -run TestUpdateRun -count=1`\r\n- `go test ./pkg/cmd/skills/... -count=1`", "test_files": ["pkg/cmd/skills/update/update_test.go"], "test_cmd": "go test ./pkg/cmd/skills/update/...", "code_files": ["pkg/cmd/skills/update/update.go"], "additions": 39, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13459, "title": "fix discovery support in nested dirs", "base_commit": "8bd56966ac5e2fe7d3162ad424dd11d877f4815e", "merge_commit": "7f885723a9cb72cd4318eef3c831f8a7bb9d9093", "prompt": "fix discovery support in nested dirs\n\n\r\n\r\nFixes https://github.com/cli/cli/issues/13372\r\n\r\n## Summary\r\n\r\nFix `gh skill install` and `gh skill preview` handling for nested skill paths.\r\n\r\n## What changed\r\n\r\n- Allow `--allow-hidden-dirs` to discover hidden skill directories at any repository depth, not only at the root.\r\n- Treat deeply nested skill arguments as exact repo-relative paths in both `install` and `preview`.\r\n - Example: `packages/agent-skills/my-skill`\r\n- Preserve one-slash selectors like `owner/skill` as namespaced skill names, not paths.\r\n- Share path-detection logic between `install` and `preview`.\r\n- Add regression tests for nested hidden directories, arbitrary nested path installs, and arbitrary nested path previews.\r\n\r\n## Product note\r\n\r\nThis does not change the recommended Agent Skills publishing conventions or broaden search/publish behavior. Non-standard layouts are only supported when the user explicitly provides an exact path.\r\n\r\n## Testing\r\n\r\n- `go test ./internal/skills/discovery ./pkg/cmd/skills/install ./pkg/cmd/skills/preview -run 'TestIsSkillPath|Test_isSkillPath|TestPreviewRun|TestMatchHiddenDirConventions|TestDiscoverSkillsWithOptions|TestDiscoverLocalSkillsWithOptions|TestInstallRun' -count=1`\r\n- `go test ./internal/skills/discovery ./pkg/cmd/skills/... -count=1`\r\n- `git diff --check`", "test_files": ["internal/skills/discovery/discovery_test.go", "pkg/cmd/skills/install/install_test.go", "pkg/cmd/skills/preview/preview_test.go"], "test_cmd": "go test ./internal/skills/discovery/... ./pkg/cmd/skills/install/... ./pkg/cmd/skills/preview/...", "code_files": ["internal/skills/discovery/discovery.go", "pkg/cmd/skills/install/install.go", "pkg/cmd/skills/preview/preview.go"], "additions": 271, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["DiscoverSkillByPathWithOptions", "All"]} +{"pr": 13449, "title": "fix(skills): stage updates in a temp dir and swap in-place", "base_commit": "a6012aca41ffa2d943ea9f12094a7398ce2e4b7e", "merge_commit": "01bcd474447b5034da20686c838ae4b0cf2b23f5", "prompt": "fix(skills): stage updates in a temp dir and swap in-place\n\nCloses #13370.\n\n## Problem\n\n`gh skill update` could relocate skills and delete the original install directory:\n\n- With `--dir`, namespaced layouts (`{dir}/{namespace}/{name}/SKILL.md`) had their install base computed by walking `filepath.Dir` twice, putting the new install one level too shallow. The post-install migration step then `RemoveAll`ed the original namespaced directory, taking the user's files with it.\n- Without `--dir`, the same migration cleanup quietly relocated namespaced skills found under an agent install directory to a flat layout, even though users expect updates to land where the skill currently lives.\n\nEither path breaks symlinks, mounts, and any external references that point at the skill directory.\n\n## Fix\n\nEach update is staged into a sibling directory of the existing skill dir (same filesystem, so renames are atomic). On success the contents are swapped in via per-entry rename:\n\n1. Move every existing entry from the skill dir into a sibling backup dir.\n2. Move every staged entry into the skill dir.\n3. Remove the backup.\n\nIf any step fails, the rollback path removes any freshly installed entries and moves the originals back from the backup.\n\nThis guarantees:\n\n- All updates happen in-place, regardless of layout (flat vs. namespaced) and regardless of whether the skill was found via an agent host or via `--dir`.\n- The skill directory's inode is preserved, so symlinks, mounts, and external references continue to resolve.\n- Stale files from the previous version are removed.\n- A failure at any point (install, read, rename) leaves the existing skill completely untouched.\n\n## Tests\n\n- New acceptance test `` plants a namespaced skill, runs `gh skill update --dir`, and asserts the skill remains at the original namespaced path with no flat-path duplicate.\n- New unit test `TestSwapDirectoryContents_PreservesDestInode` asserts the skill directory's identity is preserved across a successful swap (via `os.SameFile`).\n- New unit test `TestSwapDirectoryContents_RollsBackOnFailure` exercises the rollback path and asserts the original content (including nested subdirs) is fully restored.\n- The \"namespaced skill with `--dir`\" unit test is updated to assert in-place behavior and stale-file cleanup.\n- The existing `install_failure_during_update_reports_error_and_continues` unit test had a harness shortcut that silently skipped its `verify` block whenever `wantErr` was set. That shortcut is removed so the \"preserve original on install failure\" assertion is now actually enforced.", "test_files": ["pkg/cmd/skills/update/update_test.go"], "test_cmd": "go test ./pkg/cmd/skills/update/...", "code_files": ["pkg/cmd/skills/update/update.go"], "additions": 242, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["swapDirectoryContents"]} +{"pr": 13430, "title": "Derive digest algorithm from ref length in release verify commands", "base_commit": "b99d73d7421ae182b8ab3f5b4469dff93c792da9", "merge_commit": "f1c10e032a636540a5e28705682e203b118a47fa", "prompt": "Derive digest algorithm from ref length in release verify commands\n\nFixes #13429\n\n### Summary\n\nThe `gh release verify` and `gh release verify-asset` commands hard-coded a `sha1:` prefix when constructing the digest identifier for a release tag's commit SHA. Once GitHub repositories using SHA-256 commit digests are supported, that ref will be a 64-character SHA-256 hash and labeling it as `sha1:` is both misleading in user-facing output and incorrect for the attestation lookup.\n\n### Changes\n\n- Added a small shared helper `DigestAlgForRef(digest string) string` in `` that returns `\"sha256\"` for 64-character digests and `\"sha1\"` otherwise (preserving existing behavior).\n- Replaced the hard-coded `\"sha1\"` argument in `` and `` with a call to the new helper.\n- Added unit tests for `DigestAlgForRef` covering SHA-1, SHA-256, empty, and unexpected lengths.\n- Added `*_SHA256` variants of the existing `FailedNoAttestations` tests in both commands that assert the error message includes `sha256:`.\n\n### Verification\n\n- `go test ./pkg/cmd/release/shared/... ./pkg/cmd/release/verify/... ./pkg/cmd/release/verify-asset/...` passes.\n- `go build ./...` passes.\n- Existing SHA-1 tests continue to pass (no regression).", "test_files": ["pkg/cmd/release/shared/fetch_test.go", "pkg/cmd/release/verify-asset/verify_asset_test.go", "pkg/cmd/release/verify/verify_test.go"], "test_cmd": "go test ./pkg/cmd/release/shared/... ./pkg/cmd/release/verify/... ./pkg/cmd/release/verify-asset/...", "code_files": ["pkg/cmd/release/shared/fetch.go", "pkg/cmd/release/verify-asset/verify_asset.go", "pkg/cmd/release/verify/verify.go"], "additions": 153, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["DigestAlgForRef"]} +{"pr": 13421, "title": "Allow agents as application for secrets", "base_commit": "909cbd7d8834b5fb63d671a55a540c2cfa04d669", "merge_commit": "e60f00e30880079adefd258c388c642b1e97ddf6", "prompt": "Allow agents as application for secrets\n\nFixes https://github.com/cli/cli/issues/13419\r\n\r\nI built it locally and ran `bin/gh secret set TEST_SECRET -a agents -b 123 -o ` and it worked exactly as expected.\r\n\r\nWhat might be missing is another test? I looked through it and added a little bit but maybe a maintainer has a little more input regarding that.", "test_files": ["pkg/cmd/secret/delete/delete_test.go", "pkg/cmd/secret/list/list_test.go", "pkg/cmd/secret/set/set_test.go", "pkg/cmd/secret/shared/shared_test.go"], "test_cmd": "go test ./pkg/cmd/secret/delete/... ./pkg/cmd/secret/list/... ./pkg/cmd/secret/set/... ./pkg/cmd/secret/shared/...", "code_files": ["pkg/cmd/secret/delete/delete.go", "pkg/cmd/secret/list/list.go", "pkg/cmd/secret/secret.go", "pkg/cmd/secret/set/set.go", "pkg/cmd/secret/shared/shared.go"], "additions": 199, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["Agents", "shared.Agents"]} +{"pr": 13418, "title": "New `gh skill list` to inventory installed agent skills", "base_commit": "e6fa2faf9e3203c2caaa54dcb1526c5f093fa998", "merge_commit": "674e02a56b020549b45f970808ce7c20e179824a", "prompt": "New `gh skill list` to inventory installed agent skills\n\nCloses #13245 \r\n\r\n## Summary\r\n\r\nAdds `gh skill list` to inventory installed agent skills across known agent host directories.\r\n\r\n- Scans project and user skill install paths for all registered agent hosts\r\n- Deduplicates shared install directories like `.agents/skills`\r\n- Supports filtering with `--agent`, `--scope`, and custom scanning with `--dir`\r\n- Renders a concise table with `NAME`, `AGENT`, `SCOPE`, and `SOURCE`\r\n- Supports parseable output via `--json`, `--jq`, and `--template` with fields for `skillName`, `hosts`, `scope`, `sourceURL`, `version`, `pinned`, and `path`\r\n- Records privacy-safe `skill_list` telemetry with filters, output format, and result count\r\n\r\n\r\n## Testing\r\n\r\n- `go test ./pkg/cmd/skills/list/...`\r\n- `go test ./pkg/cmd/skills/...`", "test_files": ["pkg/cmd/skills/list/list_test.go"], "test_cmd": "go test ./pkg/cmd/skills/list/...", "code_files": ["pkg/cmd/skills/list/list.go", "pkg/cmd/skills/skills.go"], "additions": 1123, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["ListOptions"]} +{"pr": 13413, "title": "Overwrite existing parent on `gh issue edit --add-sub-issue`", "base_commit": "6106bd30764b17caac56fa57876ff9f5536093d4", "merge_commit": "b08503368202901bbc1b6fc7936dff650eb32012", "prompt": "Overwrite existing parent on `gh issue edit --add-sub-issue`\n\nFix the `Sub issue may only have one parent` 422 that surfaces when `--add-sub-issue` is handed a candidate that already belongs to another parent. Pass `replaceParent: true` so the take-over is silent.\r\n\r\nStacked on top of #13412 (which is on top of #13411).", "test_files": ["acceptance/acceptance_test.go", "pkg/cmd/issue/edit/edit_test.go"], "test_cmd": "go test ./acceptance/... ./pkg/cmd/issue/edit/...", "code_files": ["api/queries_issue.go"], "additions": 221, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13412, "title": "Rename Editable.Allowed to Selectable", "base_commit": "72859237e26439cfc23be9fb2a8144dc441a0826", "merge_commit": "2503dcfd79e372bfade8f41d05a2fbae57da1e26", "prompt": "Rename Editable.Allowed to Selectable\n\nAddresses [thread r3196020651](https://github.com/cli/cli/pull/13057#discussion_r3196020651) on #13057.\r\n\r\nStacked on top of #13411 (`-feedback`) since the field name is leveraged in the parent Issues 2.0 PR.", "test_files": ["acceptance/acceptance_test.go", "pkg/cmd/issue/edit/edit_test.go", "pkg/cmd/pr/shared/survey_test.go"], "test_cmd": "go test ./acceptance/... ./pkg/cmd/issue/edit/... ./pkg/cmd/pr/shared/...", "code_files": ["api/queries_issue.go", "pkg/cmd/issue/edit/edit.go", "pkg/cmd/pr/edit/edit.go", "pkg/cmd/pr/shared/editable.go"], "additions": 241, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["Selectable"]} +{"pr": 13403, "title": "Use int64 for GitHub database IDs", "base_commit": "1e6bb88c6d8503189bca408f7be162018b1acf84", "merge_commit": "0274077b56a5ef8e575358721149cd02888b2a5f", "prompt": "Use int64 for GitHub database IDs\n\n## Use `int64` for GitHub database IDs\r\n\r\nCloses #9247\r\n\r\n### Problem\r\n\r\nSeveral structs in the CLI use Go's `int` type for GitHub database ID fields. Go's `int` is platform-dependent and could be 32-bit on some architectures, while GitHub internally uses 64-bit integers for database IDs. The [REST API OpenAPI spec](https://github.com/github/rest-api-description) explicitly declares many resource IDs as `integer` with `format: int64` (e.g. repositories, GPG keys, SSH keys), and the [GraphQL API has migrated some `databaseId` fields from `Int` to `BigInt`](https://docs.github.com/en/graphql/overview/breaking-changes).\r\n\r\n### Changes\r\n\r\nUpdated all struct fields representing GitHub database IDs from `int` to `int64`, along with their downstream usage:\r\n\r\n- **Codespaces**: `Repository.ID`, `CreateCodespaceParams.RepositoryID`, `startCreateRequest.RepositoryID`, plus interface/mock/test cascading\r\n- **Rulesets**: `RulesetGraphQL.DatabaseId`, `RulesetREST.Id`, `BypassActors.ActorId`, `RulesetRule.RulesetId`, and the skills `rulesetsResponse.ID`\r\n- **Keys**: `deployKey.ID`, `gpgKey.ID`, `sshKey.ID`\r\n- **Cache**: `Cache.Id`, plus `parseCacheID` and `deleteCacheByID` (which still used `int`/`strconv.Atoi`)\r\n- **Autolinks**: `Autolink.ID`\r\n- **Agent tasks**: `JobActor.ID`, `JobPullRequest.ID`\r\n\r\nAll `strconv.Itoa()` calls on these fields were updated to `strconv.FormatInt(x, 10)`.\r\n\r\nGraphQL response structs were also audited - only `RulesetGraphQL.DatabaseId` needed updating. Others were already correct (`GitHubUser.DatabaseID` is `int64`, `FullDatabaseID` is `string` for GraphQL `BigInt`).\r\n\r\n### Future: Custom linter to prevent regressions\r\n\r\nA custom `go/analysis` linter that flags struct fields with ID-like names or JSON tags still using `int` has been prototyped on a separate branch: [`9247-idtype-linter`](https://github.com/cli/cli/compare/trunk...9247-idtype-linter). It's split out to keep this PR focused on the type changes. The linter correctly identifies all the violations fixed here.\r\n\r\n
\r\nLinter output BEFORE this PR (27 violations)\r\n\r\n```\r\n:38:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:43:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:20:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:25:2: struct field Id looks like a GitHub database ID but uses int; use int64 instead\r\n:153:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:807:2: struct field RepositoryID looks like a GitHub database ID but uses int; use int64 instead\r\n:858:2: struct field RepositoryID looks like a GitHub database ID but uses int; use int64 instead\r\n:203:4: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:216:4: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:260:4: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:591:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:614:2: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:621:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:639:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:660:2: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:666:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:858:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:879:2: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:885:3: struct field RepoID looks like a GitHub database ID but uses int; use int64 instead\r\n:14:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:6:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:16:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n:13:2: struct field DatabaseId looks like a GitHub database ID but uses int; use int64 instead\r\n:27:2: struct field Id looks like a GitHub database ID but uses int; use int64 instead\r\n:33:3: struct field ActorId looks like a GitHub database ID but uses int; use int64 instead\r\n:53:2: struct field RulesetId looks like a GitHub database ID but uses int; use int64 instead\r\n:67:2: struct field ID looks like a GitHub database ID but uses int; use int64 instead\r\n```\r\n\r\n
\r\n\r\n
\r\nLinter output AFTER this PR (0 violations)\r\n\r\n```\r\n(clean - no output)\r\n```\r\n\r\n
\r\n\r\n### Reviewer Notes\r\n\r\nWe should run A/C tests on this before merging.", "test_files": ["pkg/cmd/codespace/create_test.go"], "test_cmd": "go test ./pkg/cmd/codespace/...", "code_files": ["internal/codespaces/api/api.go", "pkg/cmd/agent-task/capi/job.go", "pkg/cmd/cache/delete/delete.go", "pkg/cmd/cache/shared/shared.go", "pkg/cmd/codespace/common.go", "pkg/cmd/codespace/create.go", "pkg/cmd/codespace/mock_api.go", "pkg/cmd/gpg-key/delete/delete.go", "pkg/cmd/gpg-key/delete/http.go", "pkg/cmd/repo/autolink/shared/autolink.go", "pkg/cmd/repo/deploy-key/list/http.go", "pkg/cmd/repo/deploy-key/list/list.go", "pkg/cmd/ruleset/list/list.go", "pkg/cmd/ruleset/shared/shared.go", "pkg/cmd/ruleset/view/view.go", "pkg/cmd/skills/publish/publish.go", "pkg/cmd/ssh-key/list/list.go", "pkg/cmd/ssh-key/shared/user_keys.go"], "additions": 61, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13393, "title": "fix(copilot): hint to run copilot directly when exec fails", "base_commit": "9b505c3fb8c3e52100575adbad2da7c4714684d8", "merge_commit": "fe996d33ac852eafdb2769fcba063860be1b4e6a", "prompt": "fix(copilot): hint to run copilot directly when exec fails\n\n## Summary\n\nWhen `gh copilot` finds the `copilot` binary but fails to execute it (e.g., due to unusual characters like parentheses in the file path on Windows), the error message now includes a hint:\n\n```\nFailed to run '', try running `copilot` directly without `gh`.\n```\n\n## Details\n\nOn Windows, `cmd.exe` shims (e.g., those planted by VS Code or npm) can fail when the path contains special characters like `(`. Since `gh copilot` invokes the binary via `os/exec`, these failures surface as opaque errors. This change adds a practical workaround hint so users can still use Copilot CLI directly.\n\nRelated to #13106", "test_files": ["pkg/cmd/copilot/copilot_test.go"], "test_cmd": "go test ./pkg/cmd/copilot/...", "code_files": ["pkg/cmd/copilot/copilot.go"], "additions": 45, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["findCopilotBinaryFunc"]} +{"pr": 13362, "title": "Bump `gh copilot` telemetry sampling to 100%", "base_commit": "3c162a78efc4cf48f6582f261223166af2558a64", "merge_commit": "6176c3eee4b70d0ccf3c6a1503035d049ca4e81b", "prompt": "Bump `gh copilot` telemetry sampling to 100%\n\n## Description\r\n\r\nFixes https://github.com/github/cli/issues/1163\r\n\r\nBumps the telemetry sampling rate for the `gh copilot` command from 1% to 100%, to allow for better analysis of usage.", "test_files": ["pkg/cmd/copilot/copilot_test.go"], "test_cmd": "go test ./pkg/cmd/copilot/...", "code_files": ["pkg/cmd/copilot/copilot.go", "pkg/cmd/root/root.go"], "additions": 24, "verified": "fails at parent, passes with the fix", "grader": "unit", "invented_symbols": []} +{"pr": 13068, "title": "Print `gh auth refresh` for 401 returns", "base_commit": "49d4747a81425418dd2e7bee7e33541b7c644ca9", "merge_commit": "7c439196c1fc4d645d091e83029370cef845587c", "prompt": "Print `gh auth refresh` for 401 returns\n\n`gh auth refresh` exists to make it simpler for users to refresh their tokens on expiration/scope mismatch, but help messages only suggest using it in limited scenarios, and not in a common case of a token expiring and the user receiving a 401 error. Now, the auth flow will detect this case, and for refreshable tokens (namely, tokens created by logging in with `gh auth login` in the first place), it will suggest using `gh auth refresh` for these cases.\r\n\r\nFixes https://github.com/cli/cli/issues/13066.", "test_files": ["internal/ghcmd/cmd_test.go", "pkg/cmd/auth/status/status_test.go"], "test_cmd": "go test ./internal/ghcmd/... ./pkg/cmd/auth/status/...", "code_files": ["internal/ghcmd/cmd.go", "pkg/cmd/auth/shared/writeable.go", "pkg/cmd/auth/status/status.go"], "additions": 108, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["authRecoveryCommand"]} +{"pr": 13057, "title": "Add Issues 2.0 support: issue types, sub-issues, and relationships", "base_commit": "3c162a78efc4cf48f6582f261223166af2558a64", "merge_commit": "20147bf81fed534faf1b5c9be12638983b8e5e82", "prompt": "Add Issues 2.0 support: issue types, sub-issues, and relationships\n\n# Add Issues 2.0 support: issue types, sub-issues, and relationships\n\nCloses #10298\nCloses #9696\nCloses #11757\nCloses #12477\nCloses #12152\n\nAdds support for GitHub's Issues 2.0 features across `gh issue create`, `edit`, `view`, and `list`.\n\n## New flags\n\n### `gh issue create`\n\n| Flag | Description |\n|------|-------------|\n| `--type Bug` | Set the issue type |\n| `--parent 100` | Add as a sub-issue of the specified parent (number or URL) |\n| `--blocked-by 200,201` | Mark as blocked by these issues |\n| `--blocking 300` | Mark as blocking these issues |\n\nInteractive mode prompts for issue type when the repo has types configured.\n\n### `gh issue edit`\n\n| Flag | Description |\n|------|-------------|\n| `--type Bug` | Set the issue type |\n| `--set-parent 100` | Set or replace the parent issue |\n| `--remove-parent` | Remove the parent issue |\n| `--add-sub-issue 123,124` | Add sub-issues |\n| `--remove-sub-issue 123` | Remove a sub-issue |\n| `--add-blocked-by 200` | Add blocked-by relationship |\n| `--remove-blocked-by 201` | Remove blocked-by relationship |\n| `--add-blocking 300,301` | Add blocking relationships |\n| `--remove-blocking 300` | Remove blocking relationship |\n\nInteractive mode adds Type and Parent to the field picker. Batch editing\nsupported for `--type` across multiple issues.\n\n### `gh issue view`\n\nTTY output now displays:\n- Issue type on the state line (e.g., `Bug \u00b7 Open`)\n- Type, Parent, Blocked by, Blocking metadata rows\n- Sub-issues section with completion progress (e.g., `Sub-issues \u00b7 3/5 (60%)`)\n\nJSON output supports new fields: `issueType`, `parent`, `subIssues`,\n`subIssuesSummary`, `blockedBy`, `blocking`. The `subIssues`, `blockedBy`,\nand `blocking` fields are objects of the shape `{ \"nodes\": [...], \"totalCount\": N }`\n(rather than flat arrays) so consumers can detect when the truncation\nlimit is reached. Each node and the `parent` object include the canonical\n`id`.\n\n### `gh issue list`\n\n| Flag | Description |\n|------|-------------|\n| `--type Bug` | Filter issues by type |\n\n## GHES compatibility\n\n- **Issue types and sub-issues**: GA on all supported GHES versions (3.17+). No feature detection needed.\n- **Relationships** (blocked-by/blocking): Requires GHES 3.19+. Feature detection via `__type` schema introspection for the `blockedBy` field on the Issue type. Unsupported GHES versions receive a clear error message. Gated with `// TODO IssueRelationshipsCleanup` for cleanup when GHES 3.18 support ends.\n\n## Implementation details\n\n- Post-creation mutations for type, parent, and relationships (same pattern as projectV2 and assignee mutations)\n- `--blocking` and `--add-blocking` swap API arguments: calls `addBlockedBy(issueId=OTHER, blockingIssueId=THIS)`\n- Cross-repo references supported via URL for `--parent`, `--blocked-by`, `--blocking`, and sub-issue flags\n- Shared helpers extracted to `` (`ResolveIssueRef`, `ResolveIssueTypeName`)\n- Issue-only fields (Type, Parent) gated behind `Allowed` in the Editable pattern to prevent leaking into `gh pr edit`\n- Issue type names with spaces are quoted in search queries (`type:\"Bug Report\"`)\n\n## Testing\n\n- **34 new flag parsing tests** across create, edit, and list\n- **18 new HTTP mock behavior tests** covering mutations, error cases, and GHES unsupported paths\n- **5 new view tests** (TTY rendering, JSON export)\n- **2 regression tests** (issue-only fields don't leak into PR edit, no-new-fields view unchanged)\n\n## Changes\n\n17 files changed, 2105 insertions(+), 18 deletions(-)\n\n| File | Change |\n|------|--------|\n| `` | New types, Issue struct fields, mutation functions, helpers |\n| `` | GraphQL fragment cases, `issueOnlyFields` |\n| `` | ExportData cases for new fields |\n| `` | `IssueRelationshipsSupported` detection |\n| `` | New flags, post-creation mutations, interactive type picker |\n| `` | New flags, mutations, interactive Type/Parent picker |\n| `` | TTY rendering, conditional relationship fields |\n| `` | `--type` filter via search path |\n| `` | Shared `ResolveIssueRef`, `ResolveIssueTypeName` |\n| `` | `IssueType`/`Parent` fields, `Allowed` gating |\n| `` | `IssueType` in `FilterOptions` and search query |\n\n## Key notes for reviewers\n\n- **`--blocking` arg swap**: The GitHub API only has `addBlockedBy(issueId, blockingIssueId)`. To support `--blocking`, we swap the arguments \u2014 the OTHER issue becomes `issueId` and THIS issue becomes `blockingIssueId`. Same for `--remove-blocking`.\n- **Editable `Allowed` gating**: `IssueType` and `Parent` are added to the shared `Editable` struct but gated behind `Allowed = true`, which only the issue edit command sets. This prevents these issue-only fields from leaking into `gh pr edit`'s interactive picker. Regression test in ``.\n- **View relationship fields are GHES-gated**: `blockedBy` and `blocking` are only added to the GraphQL query when `IssueRelationshipsSupported` is true. Without this, `gh issue view` would break on GHES 3.17\u20133.18 where these fields don't exist.\n- **Batch edit type resolution**: When editing multiple issues with `--type`, the type name is resolved to an ID once before the loop rather than per-issue, avoiding N-1 redundant `RepoIssueTypes` API calls.\n\n## Manual test results\n\nAll scenarios tested against on github.com.\n\n24/24 scenarios passing (click to expand)\n\n### Issue types\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 1 | `create-with-type` | `gh issue create --title \"...\" --type Bug` | \u2705 [#1](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/1) created, type=Bug verified |\n| 2 | `create-error-invalid-type` | `gh issue create --title \"Oops\" --type Bugz` | \u2705 Error: `type \"Bugz\" not found; available types: Task, Bug, Feature` |\n| 3 | `edit-change-type` | `gh issue edit 1 --type Feature` | \u2705 Type changed to Feature verified |\n| 4 | `create-interactive-type` | `gh issue create` (interactive) | \u2705 [#21](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/21) Type picker shown, Bug selected and set |\n\n### Sub-issues\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 5 | `create-with-parent` | `gh issue create --title \"...\" --parent 3` | \u2705 [#4](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/4) created as sub-issue of #3 |\n| 6 | `edit-set-parent` | `gh issue edit 5 --set-parent 3` | \u2705 Parent set, verified via `--json parent` |\n| 7 | `edit-remove-parent` | `gh issue edit 5 --remove-parent` | \u2705 Parent removed, verified null |\n| 8 | `edit-add-sub-issues` | `gh issue edit 3 --add-sub-issue 6,7` | \u2705 Sub-issues added, verified via `--json subIssues` |\n| 9 | `edit-remove-sub-issue` | `gh issue edit 3 --remove-sub-issue 6` | \u2705 Sub-issue removed |\n| 10 | `edit-error-mutual-parent` | `gh issue edit 5 --set-parent 1 --remove-parent` | \u2705 Error: `specify only one of --set-parent or --remove-parent` |\n\n### Relationships\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 11 | `create-with-relationships` | `gh issue create --blocked-by 8 --blocking 9` | \u2705 [#11](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/11) blockedBy=#8, blocking=#9 verified |\n| 12 | `edit-add-remove-blocked-by` | `gh issue edit 14 --add-blocked-by 12 --remove-blocked-by 13` | \u2705 Relationships updated |\n| 13 | `edit-add-blocking` | `gh issue edit 14 --add-blocking 15,16` | \u2705 Blocking relationships added (swapped args verified) |\n\n### View\n\n| # | Scenario | Verification | Result |\n|---|----------|-------------|--------|\n| 14 | `view-with-type` | TTY shows `Bug \u00b7 Open` on state line, `Type: Bug` row | \u2705 |\n| 15 | `view-parent-sub-issues` | TTY shows Parent row, Sub-issues section with progress | \u2705 |\n| 16 | `view-child-parent` | TTY shows parent reference for child issue | \u2705 |\n| 17 | `view-relationships` | TTY shows Blocked by and Blocking rows | \u2705 |\n| 18 | `view-json-issueType` | `--json issueType` returns id, name, description, color | \u2705 |\n| 19 | `view-json-parent-subIssues` | `--json parent,subIssues,subIssuesSummary` returns structured data | \u2705 |\n| 20 | `view-json-relationships` | `--json blockedBy,blocking` returns `{ nodes: [...], totalCount: N }` with id, number, title, url, state | \u2705 |\n\n### List\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 21 | `list-filter-by-type` | `gh issue list --type Bug` | \u2705 Filtered results returned |\n\n### Combined\n\n| # | Scenario | Command | Result |\n|---|----------|---------|--------|\n| 22 | `create-all-flags` | `gh issue create --type Task --parent 3 --blocked-by 8 --blocking 9` | \u2705 [#17](https://github.com/bagtoad-enterprise-cloud-testing/garbage/issues/17) all fields set |\n| 23 | `edit-batch-type` | `gh issue edit 18 19 20 --type Bug` | \u2705 All 3 issues updated to Bug |\n| 24 | `edit-interactive` | `gh issue edit 21` (interactive) | \u2705 Picker shows Type and Parent fields |\n\n**TODO**: GHES-specific scenarios (`create-error-ghes-relationships`), but GHES feature detection does have unit test coverage with mock introspection responses.", "test_files": ["acceptance/acceptance_test.go", "api/export_pr_test.go", "internal/featuredetection/feature_detection_test.go", "pkg/cmd/issue/create/create_test.go", "pkg/cmd/issue/edit/edit_test.go", "pkg/cmd/issue/list/list_test.go", "pkg/cmd/issue/view/view_test.go", "pkg/cmd/pr/shared/params_test.go", "pkg/cmd/pr/shared/survey_test.go", "pkg/search/query_test.go"], "test_cmd": "go test ./acceptance/... ./api/... ./internal/featuredetection/... ./pkg/cmd/issue/create/... ./pkg/cmd/issue/edit/... ./pkg/cmd/issue/list/... ./pkg/cmd/issue/view/... ./pkg/cmd/pr/shared/... ./pkg/search/...", "code_files": ["api/export_pr.go", "api/queries_issue.go", "api/query_builder.go", "internal/featuredetection/feature_detection.go", "pkg/cmd/issue/create/create.go", "pkg/cmd/issue/edit/edit.go", "pkg/cmd/issue/list/list.go", "pkg/cmd/issue/shared/lookup.go", "pkg/cmd/issue/view/view.go", "pkg/cmd/pr/edit/edit.go", "pkg/cmd/pr/shared/editable.go", "pkg/cmd/pr/shared/params.go", "pkg/search/query.go"], "additions": 2925, "verified": "fails at parent, passes with the fix", "grader": "cli", "invented_symbols": ["IssueRelationshipsSupported", "IssueType"]} diff --git a/src/adherence/adapters/children.py b/src/adherence/adapters/children.py index 25bef2c..aa532d6 100644 --- a/src/adherence/adapters/children.py +++ b/src/adherence/adapters/children.py @@ -28,6 +28,7 @@ import os import sqlite3 import sys +import time from pathlib import Path @@ -36,9 +37,26 @@ def db_path() -> Path: return Path(base) / "opencode" / "opencode.db" -def children(root: str) -> list[tuple[str, str]]: +def children(root: str, attempts: int = 5, delay: float = 0.6 + ) -> list[tuple[str, str]]: """[(session_id, agent_name)] for every descendant of `root`, in creation order. Recursive: a subagent may dispatch its own.""" + # Retried, because this runs the instant `opencode run` returns and a + # child session written moments earlier may not be visible yet. A + # single miss is not a small error: measured against the recording + # proxy, a trial reporting 26 calls had made 66, and two whole + # subagent sessions -- 40 calls, ~1.5M input tokens -- were absent + # from the transcript while the proxy had them all. E3 is a claim + # about exactly those tokens. + for attempt in range(attempts): + found = _query(root) + if found or attempt == attempts - 1: + return found + time.sleep(delay) + return [] + + +def _query(root: str) -> list[tuple[str, str]]: p = db_path() if not p.exists(): return [] diff --git a/src/adherence/classify.py b/src/adherence/classify.py new file mode 100644 index 0000000..d88bd04 --- /dev/null +++ b/src/adherence/classify.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Decide which grader can judge each existing task, by compiling. + + python3 -m adherence.classify --tasks scenarios-pr/cli-cli \\ + --mirror fixtures/cli-cli.git + +`mkpr` makes this decision for new tasks. Task sets extracted before the +distinction existed carry no `grader` field, and this backfills it in +place. + +The decision cannot be made by reading diffs. A static scan for +identifiers the PR's code adds and its tests reference finds the obvious +cases and **misses the rest**: measured against the validation grid, it +flagged 13393 and 13967 correctly but missed 13624 (`cannot use +verifier.noVerifierSet()` -- a signature change on an existing type) and +13675 (`field.onSearchDone.Store undefined` -- a field added to an +existing struct). Neither adds a top-level declaration, so neither shows +up in a declaration scan, and both are just as unpassable. + +The compiler is the only authority on whether a test compiles. So this +does what the runtime does: check the PR's tests out onto the parent tree +and build them. A build error naming something undefined means the test +cannot compile against ANY implementation that chose different names, and +the task goes to the CLI grader. Anything else -- tests that build and +fail on assertions -- stays with the unit grader. + +Costs one build per task and no GPU. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +from adherence.mkpr import ECOSYSTEMS, names_the_test_requires, packages_for, warm_at + + +def classify_one(mirror: Path, task: dict, env: dict, eco: dict) -> tuple[str, list[str], str]: + """(grader, invented_symbols, note) for one task.""" + tests = task.get("test_files") or [] + if not tests: + return "unit", [], "no test files" + pkgs = packages_for(tests, eco) + with tempfile.TemporaryDirectory(prefix="adh-classify-") as d: + try: + subprocess.run(["git", "clone", "--local", "--shared", "--quiet", + "--no-checkout", str(mirror), d], + check=True, capture_output=True, timeout=600) + subprocess.run(["git", "checkout", "--detach", "--quiet", + task["base_commit"]], cwd=d, check=True, + capture_output=True, timeout=600) + warm_at(d, env, eco) + subprocess.run(["git", "checkout", task["merge_commit"], "--"] + + tests, cwd=d, check=True, capture_output=True, + timeout=600) + except (OSError, subprocess.SubprocessError) as e: + return "unit", [], f"could not materialize: {str(e)[-120:]}" + + # `go vet` builds the test binary without running it, which is the + # cheapest way to ask the compiler the only question that matters. + cmd = "go vet " + " ".join(pkgs) if pkgs else "go build ./..." + try: + r = subprocess.run(cmd, shell=True, cwd=d, env=env, + capture_output=True, text=True, timeout=900) + except subprocess.SubprocessError: + return "unit", [], "vet timed out" + if r.returncode == 0: + return "unit", [], "tests compile at the parent" + invented = names_the_test_requires(r.stdout + r.stderr) + if invented: + return "cli", invented, "tests name what the fix introduces" + return "unit", [], "compile failure, but not a missing identifier" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--tasks", required=True) + ap.add_argument("--mirror", required=True) + ap.add_argument("--only", default="") + args = ap.parse_args() + + gt = Path(args.tasks) / "ground-truth.jsonl" + if not gt.is_file(): + sys.exit(f"no ground-truth.jsonl in {args.tasks}") + tasks = [json.loads(x) for x in gt.read_text().splitlines() if x.strip()] + if args.only: + want = {int(x) for x in args.only.split(",")} + tasks = [t for t in tasks if t["pr"] in want] + + mirror = Path(args.mirror).resolve() + eco = ECOSYSTEMS["go"] + import os + env = dict(os.environ) + env["GOFLAGS"] = "-mod=mod" + + counts = {"unit": 0, "cli": 0} + for t in tasks: + grader, syms, note = classify_one(mirror, t, env, eco) + t["grader"] = grader + t["invented_symbols"] = syms + counts[grader] += 1 + detail = f" [{', '.join(syms[:3])}]" if syms else "" + print(f" #{t['pr']:<7}{grader:<5}{note}{detail}", file=sys.stderr, + flush=True) + + all_tasks = [json.loads(x) for x in gt.read_text().splitlines() if x.strip()] + by_pr = {t["pr"]: t for t in tasks} + with open(gt, "w") as f: + for t in all_tasks: + f.write(json.dumps(by_pr.get(t["pr"], t)) + "\n") + print(f"\n{counts['unit']} unit-graded, {counts['cli']} cli-graded " + f"-> {gt}", file=sys.stderr) + print("re-run `make mkscenarios` so the scenarios carry the choice", + file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/adherence/cligrade.py b/src/adherence/cligrade.py new file mode 100644 index 0000000..0c05ef5 --- /dev/null +++ b/src/adherence/cligrade.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Grade a feature PR at the command-line boundary, not the Go boundary. + +The problem this exists for: applying a PR's own unit tests to an agent's +independent implementation requires the agent to have chosen the +maintainers' exact internal identifiers. Measured on the validation grid, +every task whose tests referenced new symbols scored 0-5% -- while the +agent had already passed `diff_coverage` with 13/13 pre-edit probes in the +right directories. It did the work and failed on a name. + +The obvious repair -- tell the agent the required API surface -- is worse +than the disease. A symbol list *is* routing information (`CreateOptions. +IssueType` names the subsystem), so it would hand every arm a piece of +exactly what the treatment is supposed to supply, and bias the primary +outcome toward the null in the treatment's own channel. A null result +would then be uninterpretable. + +So grade what the PR *publicly promises* instead. `gh issue create --type` +is a user-facing contract: it appears in the PR body the agent already +receives, and it is identical whatever the agent names its internals. +Nothing here is added to the prompt; the grader simply stops asking a +question it never gave the agent the information to answer. + +The oracle is the merge commit's own binary, so the expectations are still +the maintainers' and not the experimenter's: + + reference = build(merge commit) what the PR actually shipped + candidate = build(agent's tree) what the agent produced + compare observable CLI behaviour + +Offline and deterministic -- `--help` and argument validation need no +network, which is what makes this runnable inside the sandbox. + +Stdlib only. +""" +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +from adherence.gradelib import Check, bad, ok, skip + +BUILD_TIMEOUT = 600 +RUN_TIMEOUT = 60 + +# A flag line in Go/cobra help output: two spaces, optional -x, then --name. +_FLAG = re.compile(r"^\s+(?:-\w,\s+)?--([a-z0-9][\w-]*)") + + +def command_path(code_files: list[str]) -> list[str]: + """The CLI subcommand a PR touches, from its file paths. + + `pkg/cmd/issue/create/create.go` -> ["issue", "create"]. Mechanical, so + the battery is derived rather than authored -- an experimenter choosing + which commands to test is an experimenter choosing the result.""" + best: list[str] = [] + for f in code_files or []: + parts = Path(f).parts + if len(parts) < 3 or parts[0] != "pkg" or parts[1] != "cmd": + continue + words = [p for p in parts[2:-1] if p and not p.endswith(".go")] + if len(words) > len(best): + best = words + return best + + +def build(tree: Path, out: Path, env=None) -> tuple[bool, str]: + try: + r = subprocess.run(["go", "build", "-o", str(out), "./cmd/gh"], + cwd=tree, capture_output=True, text=True, + timeout=BUILD_TIMEOUT, env=env) + except (OSError, subprocess.SubprocessError) as e: + return False, str(e)[-400:] + return r.returncode == 0, (r.stderr or r.stdout)[-800:] + + +def flags(binary: Path, argv: list[str], env=None) -> set[str]: + """Flag names a command advertises. Empty on any failure -- the caller + distinguishes 'no flags' from 'could not ask'.""" + try: + r = subprocess.run([str(binary), *argv, "--help"], + capture_output=True, text=True, + timeout=RUN_TIMEOUT, env=env) + except (OSError, subprocess.SubprocessError): + return set() + return {m.group(1) for line in (r.stdout + r.stderr).splitlines() + if (m := _FLAG.match(line))} + + +def grade_cli(sandbox: Path, mirror: Path, merge: str, + code_files: list[str], env=None) -> list[Check]: + """Differential CLI grading against the PR's own merge commit. + + Returns `ungradeable` rather than `fail` for anything that is the + harness's problem -- a toolchain that will not build the reference, a + PR that touches no command -- because a grader that cannot run must + never be scored as a model that got it wrong.""" + checks: list[Check] = [] + argv = command_path(code_files) + if not argv: + # No command surface, and this task is only here because its unit + # tests name symbols the fix introduces -- so neither grader can + # judge it. Reporting `skip` left diff_coverage and scope as the + # only checks that ran, and all_pass is "every non-ungradeable + # check passed": the verdict became "touched the right file and + # nothing else", which an agent passes by adding a comment. + # Measured: 5 of 5 such trials recorded as PASS. + # + # `bad` rather than `skip`, because a task nothing can grade must + # not be able to yield a pass. The suite should refuse it at + # extraction; this is the backstop for one that got through. + return [bad("cli.ungradeable", + f"no grader can judge this task: its unit tests name " + f"symbols the fix introduces, and it touches no " + f"pkg/cmd/... command so there is no CLI surface to " + f"compare. Files: {(code_files or [])[:3]}")] + + # --- the candidate: whatever the agent produced -------------------- + cand = sandbox / ".adh-gh-candidate" + built, err = build(sandbox, cand, env) + if not built: + # This IS a model result: the agent left the tree not compiling. + return [bad("cli.builds", + f"the agent's tree does not build: {err[-300:]}")] + checks.append(ok("cli.builds", "the agent's tree compiles")) + + # --- the oracle: the merge commit, built the same way --------------- + import tempfile + with tempfile.TemporaryDirectory(prefix="adh-ref-") as d: + ref_tree = Path(d) + try: + subprocess.run(["git", "clone", "--local", "--shared", "--quiet", + "--no-checkout", str(mirror), str(ref_tree)], + check=True, capture_output=True, timeout=300) + subprocess.run(["git", "checkout", "--detach", "--quiet", merge], + cwd=ref_tree, check=True, capture_output=True, + timeout=300) + except (OSError, subprocess.SubprocessError) as e: + checks.append(skip("cli.surface", + f"could not materialize the reference: {e}")) + return checks + ref = ref_tree / "gh-reference" + ok_ref, ref_err = build(ref_tree, ref, env) + if not ok_ref: + checks.append(skip("cli.surface", + f"the reference commit does not build here, " + f"so there is nothing to compare against: " + f"{ref_err[-200:]}")) + return checks + + want = flags(ref, argv, env) + got = flags(cand, argv, env) + if not want: + checks.append(skip("cli.surface", + f"reference `gh {' '.join(argv)} --help` " + f"advertised no flags; nothing to compare")) + return checks + + missing = sorted(want - got) + checks.append( + ok("cli.surface", + f"`gh {' '.join(argv)}` advertises all {len(want)} flag(s) " + f"the PR's own binary does") + if not missing else + bad("cli.surface", + f"missing {len(missing)} flag(s) the PR shipped: " + f"{missing[:8]}. Compared against the merge commit's own " + f"binary, so these are the maintainers' names, not the " + f"experimenter's")) + + # Flags the agent added beyond the PR: reported, never failed. A + # different valid implementation may expose more. + extra = sorted(got - want) + checks.append(skip("cli.extra_surface", + f"also advertises {len(extra)} flag(s) the PR " + f"does not: {extra[:8]}") + if extra else + ok("cli.extra_surface", "no flags beyond the PR's")) + return checks diff --git a/src/adherence/design.py b/src/adherence/design.py new file mode 100644 index 0000000..d169f45 --- /dev/null +++ b/src/adherence/design.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""What each arm is, how they differ, and the rules the run is bound by. + + python3 -m adherence.design # one-shot text listing + +Orientation, like adherence.tasks, and for the same reason: an operator +reading "a3 0.82x vs a2" has to know what a2 and a3 actually are before +the number means anything, and the answer currently lives in a 600-line +registration document. The arm surfaces are read off disk rather than +described, so this cannot drift from what a trial is actually handed. + +Static: reads the arms directory and the registered constants. Never a run. +Stdlib only. +""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +from adherence import REPO_ROOT +from adherence.suitedata import ARMS + +# Files an agent is handed without asking. Anything under .subagents/ is +# loaded on demand by routing, which is the claim under test. +ALWAYS_LOADED = ("AGENTS.md", "CLAUDE.md") + +# Why each arm exists, in the terms a reader of a result needs. The +# registry in suitedata carries the one-line role; this is what the arm is +# FOR and what its number means when compared. +ARM_PURPOSE = { + "a0": "The floor. No instruction surface at all, so it answers what any " + "instruction costs and buys before asking which one is better. A " + "treatment that cannot beat this is not worth its own bytes.", + "a1": "The practical control: the repository's own AGENTS.md, recovered " + "verbatim from the pinned commit and never authored here. This is " + "what a real team already has, so it is the comparison anyone " + "deciding whether to adopt the pattern actually cares about.", + "a2": "The scientific control: the same content as the treatment, " + "delivered as one file instead of a router plus modules. It " + "isolates DELIVERY from CONTENT -- without it, a win could just " + "mean the treatment's text is better written.", + "a3": "The pattern as shipped: a small always-loaded router that points " + "at bounded contexts loaded on demand. The primary outcome is its " + "marginal input tokens against a2.", + "a4": "The same context set delivered by spawning a context-scoped " + "subagent rather than inline. Answers E3, the claim that subagent " + "handoff is free, on parent+child totals rather than the parent's.", + "a5": "The minimal alternative raised in the original discussion: a " + "short AGENTS.md plus tools, no routing. Guards against the " + "pattern being credited for what a few good sentences would do.", +} + +# The rules the run is bound by, each with the failure it exists to +# prevent. Wording is deliberately close to docs/EVAL.md so the two cannot +# drift into saying different things. +GROUND_RULES = [ + ("Primary outcome", + "Marginal input tokens in the directed arm against the content-matched " + "monolith (a2), paired on scenario, geometric mean, 95% CI from a " + "cluster bootstrap over scenarios."), + ("Cost is meaningless alone", + "No cost number is reportable without the pass rate beside it. Cheaper " + "and worse is a trade, not a win, and the honest output is the (cost, " + "success) plane."), + ("Calibration band", + "A scenario counts only if its pooled pass rate lands in [0.25, 0.80]. " + "Outside that it cannot discriminate between arms. Dropped scenarios " + "are logged and never re-authored to make them discriminate."), + ("Harness faults are not model failures", + "An adapter that did not complete, or a transcript that failed schema " + "validation, is `ungradeable` and excluded -- never scored as a model " + "that got it wrong."), + ("Validation never becomes evidence", + "Every record carries `purpose`. The registered analysis reads only " + "rows marked `experiment`; anything unlabelled is treated as " + "validation, because the safe reading of no label is not 'this is real " + "data'."), + ("Two grading tiers, never pooled", + "Unit-graded tasks are the primary evidence. Tasks graded at the " + "command line (Amendment 2) are reported as a separate tier, because " + "a flag comparison is a weaker signal than a passing unit test."), + ("The proxy is authoritative", + "On any disagreement about tokens or round trips, the recording proxy " + "wins over the adapter. It counts calls by construction: a round trip " + "is a request it handled."), + ("Deviations are disclosed, not forbidden", + "Any departure from the registration is stated in the results with " + "what changed and why. Undisclosed deviation is the only thing " + "actually prohibited."), +] + + +def _files(d: Path): + return sorted(p for p in d.rglob("*") if p.is_file()) if d.is_dir() else [] + + +def load(arms_dir: Path | None = None) -> list[dict]: + """One record per materialized arm, measured off disk.""" + arms_dir = arms_dir or (REPO_ROOT / "fixtures" / "cli-cli.arms") + out = [] + for key, (name, role) in ARMS.items(): + if key == "-": + continue + d = arms_dir / key + files = _files(d) + always = [f for f in files if f.name in ALWAYS_LOADED] + ondemand = [f for f in files + if ".subagents" in f.as_posix() and f.suffix == ".md"] + blob = b"".join(f.read_bytes() for f in always) + out.append({ + "arm": key, "name": name, "role": role, + "purpose": ARM_PURPOSE.get(key, ""), + "present": bool(files), + "always_bytes": sum(f.stat().st_size for f in always), + "always_files": [f.name for f in always], + "ondemand_files": [f.name for f in ondemand], + "ondemand_bytes": sum(f.stat().st_size for f in ondemand), + "sha8": hashlib.sha256(blob).hexdigest()[:8] if blob else "", + }) + return out + + +def deltas(rows, ref="a1") -> dict: + """Always-loaded bytes relative to the reference arm.""" + base = next((r["always_bytes"] for r in rows if r["arm"] == ref), 0) + return {r["arm"]: r["always_bytes"] - base for r in rows} + + +def main(): + rows = load() + d = deltas(rows) + print(f"{'arm':<5}{'name':<22}{'always':>9}{'vs a1':>9}{'on demand':>11}" + f" role") + for r in rows: + if not r["present"]: + print(f"{r['arm']:<5}{r['name']:<22}{'not built':>9}") + continue + print(f"{r['arm']:<5}{r['name']:<22}{r['always_bytes']:>9,}" + f"{d[r['arm']]:>+9,}{r['ondemand_bytes']:>11,} {r['role']}") + print("\nground rules") + for k, v in GROUND_RULES: + print(f" {k}\n {v}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/adherence/live.py b/src/adherence/live.py index 94c0f42..0f18f46 100644 --- a/src/adherence/live.py +++ b/src/adherence/live.py @@ -23,6 +23,8 @@ import glob import json import os +import sqlite3 +import subprocess import tempfile import time from pathlib import Path @@ -43,7 +45,21 @@ def out_dirs(tmp: str | None = None): def _alive(pid: int) -> bool: - if not pid: + """Does this pid exist? POSIX only -- see below. + + `os.kill(pid, 0)` is the standard existence probe on POSIX, where + signal 0 is "check, do not send". On Windows it is not a probe at all: + `signal.CTRL_C_EVENT == 0`, so the call delivers a real Ctrl-C to the + target's console group. The selftest wrote a marker carrying its own + pid, this function checked it, and the check raised KeyboardInterrupt + inside an unrelated subprocess several tests later. Sixteen seconds + into the Windows job, with a traceback pointing at code that had + nothing to do with it. + + A liveness check must never be able to affect what it observes, so on + Windows it declines to answer and the caller falls back to evidence + that cannot misfire.""" + if not pid or os.name != "posix": return False try: os.kill(pid, 0) @@ -52,6 +68,40 @@ def _alive(pid: int) -> bool: return True +def busy_out_dirs() -> set[str] | None: + """Out-dirs named on the command line of some live process. + + The runner passes the out-dir to the adapter as an argument, so it is + right there in `ps` for the whole life of the trial. That makes it a + direct liveness signal, and a much better one than the fallback it + replaces: without a marker there is no pid, so liveness was inferred + from "the stream file moved recently", which reports a run killed ten + seconds ago as still running for another three minutes. Observed + exactly that after a teardown -- four rows claiming to run while `ps` + showed nothing at all. + + None means ps was unavailable and the caller should fall back.""" + try: + out = subprocess.run(["ps", "-eo", "args"], capture_output=True, + text=True, timeout=10) + except (OSError, subprocess.SubprocessError): + return None + if out.returncode != 0: + return None + busy = set() + for line in out.stdout.splitlines(): + for tok in line.split(): + if "adh-out-" in tok: + # prompt.txt lives inside the out-dir; take the directory. + p = tok.rstrip("/") + if not p.endswith("adh-out-") and "/adh-out-" in f"/{p}": + while p and "adh-out-" not in os.path.basename(p): + p = os.path.dirname(p) + if p: + busy.add(p) + return busy + + # The stream nests everything one level down under "part"; the useful # fields are part.tokens for a step_finish and part.tool / part.state.input # for a tool_use. Read off opencode 1.18.10 -- the same moving target the @@ -82,6 +132,7 @@ def _stream(path: Path) -> dict: spawns: list[str] = [] first_ts = last_ts = 0 partial = 0 + root_sid = "" try: with open(path, errors="replace") as f: for line in f: @@ -99,6 +150,8 @@ def _stream(path: Path) -> dict: first_ts = first_ts or ts last_ts = ts sid = e.get("sessionID") or "" + if sid and not root_sid: + root_sid = sid t = e.get("type") if t == "step_finish": calls += 1 @@ -139,7 +192,164 @@ def _stream(path: Path) -> dict: "sessions": len(sessions), "subagent_calls": sum(sorted(sessions.values())[:-1]) if len(sessions) > 1 else 0, "first_ts": first_ts, "last_ts": last_ts, - "partial_lines": partial} + "root_session": root_sid, "partial_lines": partial} + + +def _subagents(out_dir: Path, root_session: str) -> dict: + """Child sessions this trial has dispatched, read live. + + A subagent runs in its OWN opencode session, and neither the root + stream nor the root export carries a single one of its inference calls + -- measured at 3.1x under-report on s13. So the stream this view reads + cannot see subagent cost at all, and reporting its totals without + saying so would understate exactly the number E3 is a claim about. + + The runner gives every trial its own XDG_DATA_HOME under the out-dir, + so opencode's store is right here while the run is live. Read-only and + best-effort: the writer is active, the schema is private and may move + between versions, and a viewer must never block a run. On any failure + this reports nothing rather than a number it cannot stand behind.""" + db = out_dir / "xdg" / "data" / "opencode" / "opencode.db" + empty = {"sessions": [], "root": root_session, "readable": False, + "calls": 0, "tok_in": 0, "tok_out": 0} + if not db.exists(): + return dict(empty) + try: + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True, timeout=2) + con.execute("PRAGMA query_only=ON") + rows = list(con.execute( + "SELECT id, parent_id, COALESCE(agent,'') FROM session " + "WHERE parent_id IS NOT NULL ORDER BY time_created ASC")) + kids = [(sid, agent) for sid, _parent, agent in rows] + calls = tok_in = tok_out = 0 + if kids: + ids = [s for s, _a in kids] + placeholders = ",".join("?" * len(ids)) + q = f"SELECT data FROM part WHERE session_id IN ({placeholders})" + for (data,) in con.execute(q, ids): + try: + d = json.loads(data) + except (json.JSONDecodeError, TypeError): + continue + # Same event the root stream calls step_finish; the store + # spells it with a hyphen. + if d.get("type") == "step-finish": + calls += 1 + tk = d.get("tokens") or {} + tok_in += tk.get("input") or 0 + tok_out += tk.get("output") or 0 + con.close() + except Exception: + return dict(empty) + return {"sessions": kids, "root": root_session, "readable": True, + "calls": calls, "tok_in": tok_in, "tok_out": tok_out} + + +MAX_BODY = 20000 + + +def _pretty(body) -> str: + """Readable text for an expanded event. + + Two jobs. Keep the line structure -- a file listing, a diff and a stack + trace are mostly structure, and joining them into one paragraph throws + away the thing being read. And re-indent JSON: a tool's input arrives + as a dict and its output is often a serialized blob, both of which are + a wall on one line and obvious over twelve.""" + if isinstance(body, (dict, list)): + try: + return json.dumps(body, indent=2)[:MAX_BODY] + except (TypeError, ValueError): + return str(body)[:MAX_BODY] + text = str(body or "") + stripped = text.strip() + if stripped[:1] in "[{" and stripped[-1:] in "]}": + try: + return json.dumps(json.loads(stripped), indent=2)[:MAX_BODY] + except (json.JSONDecodeError, ValueError): + pass + return text.replace("\r\n", "\n").replace("\r", "\n")[:MAX_BODY] + + +def preview(e: dict, width: int) -> str: + """One line for the list: the first non-empty line of the body.""" + for ln in (e.get("text") or "").splitlines(): + ln = ln.strip() + if ln: + return ln[:width] + return "" + + +def activity(out_dir: Path, limit: int = 400) -> list[dict]: + """The trial's recent events, with tool output. + + The NDJSON stream carries a tool call's *input* and status but not + what it returned -- so a viewer built on the stream alone can say "it + ran go test" and never what go test said, which is the one thing worth + knowing while a run is in flight. opencode's store keeps the output, + and the runner puts that store in the out-dir. + + Read-only and best-effort, like everything else here: the writer is + live and the schema is private.""" + db = out_dir / "xdg" / "data" / "opencode" / "opencode.db" + if not db.exists(): + return [] + try: + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True, timeout=2) + con.execute("PRAGMA query_only=ON") + kids = {s for (s,) in con.execute( + "SELECT id FROM session WHERE parent_id IS NOT NULL")} + rows = list(con.execute( + "SELECT id, session_id, data, time_created FROM part " + "ORDER BY time_created DESC LIMIT ?", (limit * 6,))) + con.close() + except Exception: + return [] + + out = [] + for pid, sid, data, ts in rows: + try: + d = json.loads(data) + except (json.JSONDecodeError, TypeError): + continue + kind = d.get("type") + who = "subagent" if sid in kids else "agent" + if kind == "tool": + st = d.get("state") or {} + inp = st.get("input") or {} + body = st.get("error") or st.get("output") or "" + out.append({ + # The store's own primary key. A viewer anchored to a list + # POSITION follows "whatever is newest", so a new event + # arriving while you read one swaps it out from under you. + "id": pid, + "ts": ts, "who": who, "kind": "tool", + "name": d.get("tool") or "?", + "target": _target(inp), + "status": st.get("status") or "", + # Structure preserved. Collapsing whitespace made every + # JSON blob and every file listing one unreadable line -- + # and structure is most of what a tool result IS. + "text": _pretty(body), + "input": _pretty(inp) if inp else "", + "failed": bool(st.get("error")), + }) + elif kind == "text": + body = (d.get("text") or "").strip() + if body: + out.append({"id": pid, "ts": ts, "who": who, "kind": "text", + "name": "", "target": "", "status": "", + "text": _pretty(d.get("text") or ""), + "input": "", "failed": False}) + out.sort(key=lambda e: e["ts"]) + out = out[-limit:] + # Absolute position in the run, counted from the start. A reader + # scanning a newest-first list needs a number that means something + # other than "how far down the screen you are" -- this one says how + # many events the trial has produced and where this sits among them. + for i, e in enumerate(out, 1): + e["n"] = i + return out def _scenario_info(scenario: str, root: Path) -> dict: @@ -177,6 +387,7 @@ def snapshot(tmp: str | None = None, root: Path | None = None, """One record per run currently in flight, newest activity first.""" root = root or REPO_ROOT now = now or time.time() + busy = busy_out_dirs() runs = [] for d in out_dirs(tmp): p = Path(d) @@ -202,9 +413,18 @@ def snapshot(tmp: str | None = None, root: Path | None = None, # transcript.jsonl is written after the harness returns, so its # presence means the agent is done and grading is under way. graded = (p / "transcript.jsonl").is_file() + # Same reason as runner.wait_or_kill: the root stream goes silent + # for the whole time a subagent is working, so a delegating trial + # reads as stalled when it is the busiest thing on the machine. + # opencode's store runs in SQLite WAL mode, so live writes land in + # `-wal` and the main .db mtime only moves on checkpoint -- measured + # 234s stale while `-wal` was 33s fresh. Watching the .db alone made + # a delegating trial look idle for a whole checkpoint interval. + store = p / "xdg" / "data" / "opencode" / "opencode.db" + watched = [stdout, store, Path(str(store) + "-wal"), p] try: - touched = max(os.path.getmtime(x) for x in - (stdout, p) if os.path.exists(x)) + touched = max(os.path.getmtime(x) for x in watched + if os.path.exists(x)) except (OSError, ValueError): touched = 0.0 age = now - touched if touched else 1e9 @@ -214,11 +434,15 @@ def snapshot(tmp: str | None = None, root: Path | None = None, except OSError: elapsed = 0.0 - # No marker means no pid to check, so liveness falls back to "the - # stream moved recently", which is the same evidence a viewer would - # use by eye. - alive = (_alive(int(m["pid"])) if m.get("pid") - else age < STALE_S) + # Liveness, best evidence first: a process still naming this + # out-dir, then the runner's recorded pid, and only if `ps` is + # unavailable the weak "stream moved recently" guess. + if busy is not None: + alive = d in busy + elif m.get("pid") and os.name == "posix": + alive = _alive(int(m["pid"])) + else: + alive = age < STALE_S if age > ABANDONED_S and not alive: continue # debris from a killed run @@ -229,9 +453,16 @@ def snapshot(tmp: str | None = None, root: Path | None = None, if st["first_ts"]: elapsed = max(elapsed, (now * 1000 - st["first_ts"]) / 1000.0) + kids = _subagents(p, st.get("root_session", "")) timeout = int(m.get("timeout") or 0) - if graded: + if graded and alive: state = "grading" + elif graded: + # A transcript with no live process is a FINISHED trial whose + # directory was kept (--keep-sandbox) or not yet swept. It used + # to read "grading" forever, with a deadline clock still + # ticking past 100% against a deadline that no longer applies. + state = "done" elif not alive: state = "dead" elif age > STALE_S: @@ -255,10 +486,29 @@ def snapshot(tmp: str | None = None, root: Path | None = None, # --timeout overrides scenario.yaml, so substituting the # scenario's value would overstate the remaining budget on # exactly the runs closest to being killed. - "budget": (elapsed / timeout) if timeout else None, + # The adapter deadline only governs a trial that is still + # generating. Grading is unbounded and a finished trial is not + # racing anything, so a percentage there is meaningless -- it + # read 104% on a run that had already stopped. + "budget": ((min(elapsed / timeout, 1.0) if timeout else None) + if state in ("running", "stalled") else None), "idle_s": age, "state": state, "unlabelled": bool(m.get("unlabelled")), + # Dispatched subagents, from opencode's own store. `tok_in` + # above is ROOT ONLY -- their calls are in separate sessions + # the root stream never sees. + "child_sessions": len(kids["sessions"]), + "child_agents": [a for _s, a in kids["sessions"] if a], + "children_readable": kids["readable"], + "child_calls": kids["calls"], + "child_tok_in": kids["tok_in"], + "child_tok_out": kids["tok_out"], + # The number E3 is actually a claim about: parent + every + # child. Quoting the root total alone would confirm "subagent + # handoff is 0-cost" by leaving the cost out of the number. + "total_calls": st["calls"] + kids["calls"], + "total_tok_in": st["tok_in"] + kids["tok_in"], "info": _scenario_info(m.get("scenario", ""), root), **st, }) @@ -277,8 +527,9 @@ def summarize(runs) -> dict: "running": sum(1 for r in runs if r["state"] == "running"), "grading": sum(1 for r in runs if r["state"] == "grading"), "stalled": sum(1 for r in runs if r["state"] in ("stalled", "dead")), - "calls": sum(r["calls"] for r in runs), - "tok_in": sum(r["tok_in"] for r in runs), + "calls": sum(r["total_calls"] for r in runs), + "tok_in": sum(r["total_tok_in"] for r in runs), + "child_calls": sum(r["child_calls"] for r in runs), } @@ -295,21 +546,78 @@ def fmt_age(s) -> str: return f"{s // 3600}h{(s % 3600) // 60:02d}m" +def sweep(tmp: str | None = None, dry: bool = False) -> tuple[int, int]: + """Remove sandboxes and out-dirs no live process is using. + + A trial that completes cleans up after itself; one that is killed does + not, and the leftovers are not harmless. They are what the live view + reads, so a torn-down run keeps rendering as rows long after it is + gone, and at ~30MB of git checkout apiece they are not free either. + + Refuses while anything is running, because the only way to decide a + directory is dead is to know nothing is using it.""" + import shutil + busy = busy_out_dirs() + if busy is None: + raise RuntimeError("cannot enumerate processes; refusing to delete " + "directories that might be in use") + if busy: + raise RuntimeError( + f"{len(busy)} run(s) still active. Stop them first -- deleting a " + f"sandbox under a running agent produces a garbage transcript " + f"and a result row that looks real.") + root = tmp or tempfile.gettempdir() + freed = n = 0 + for d in sorted(glob.glob(os.path.join(root, "adh-*"))): + try: + size = sum(f.stat().st_size for f in Path(d).rglob("*") + if f.is_file()) + except OSError: + size = 0 + n += 1 + freed += size + if not dry: + shutil.rmtree(d, ignore_errors=True) + return n, freed + + def main(): + import argparse + ap = argparse.ArgumentParser() + ap.add_argument("--clean", action="store_true", + help="delete sandboxes and out-dirs from finished or " + "killed runs; refuses while anything is running") + ap.add_argument("--dry-run", action="store_true") + a = ap.parse_args() + if a.clean: + try: + n, freed = sweep(dry=a.dry_run) + except RuntimeError as e: + print(e) + return 1 + verb = "would remove" if a.dry_run else "removed" + print(f"{verb} {n} director{'y' if n == 1 else 'ies'}, " + f"{freed / 1e6:.0f} MB") + return 0 + runs = snapshot() if not runs: print("nothing in flight") return 0 s = summarize(runs) print(f"{s['running']} running, {s['grading']} grading, " - f"{s['stalled']} stalled — {s['calls']} calls, " - f"{s['tok_in']:,} input tokens so far\n") + f"{s['stalled']} stalled — {s['calls']} calls " + f"({s['child_calls']} in subagents), " + f"{s['tok_in']:,} input tokens so far\n" + f"calls and tokens are parent + children (§7)\n") print(f"{'scenario':<20}{'arm':>4}{'t':>3}{'state':>9}{'calls':>7}" - f"{'tools':>7}{'tok_in':>11}{'elapsed':>9}{'budget':>8} activity") + f"{'tools':>7}{'sub':>5}{'tok_in':>12}{'elapsed':>9}" + f"{'budget':>8} activity") for r in runs: print(f"{r['scenario'][:19]:<20}{r['arm']:>4}{r['trial']:>3}" - f"{r['state']:>9}{r['calls']:>7}{r['tools']:>7}" - f"{r['tok_in']:>11,}{fmt_age(r['elapsed_s']):>9}" + f"{r['state']:>9}{r['total_calls']:>7}{r['tools']:>7}" + f"{r['child_sessions']:>5}" + f"{r['total_tok_in']:>12,}{fmt_age(r['elapsed_s']):>9}" f"{fmt_budget(r['budget']):>8} {r['last_tool'][:50]}") if any(r["unlabelled"] for r in runs): print("\n? arm/trial: run started before the runner wrote its marker; " diff --git a/src/adherence/matrix_tui.py b/src/adherence/matrix_tui.py index a09b686..b908d7e 100644 --- a/src/adherence/matrix_tui.py +++ b/src/adherence/matrix_tui.py @@ -24,20 +24,60 @@ import os import sys import time +from pathlib import Path +from adherence import design as dz from adherence import live as lv from adherence import suitedata as sd +from adherence import tasks as tk from adherence.tui.framework import TuiApp, curses_main -VIEWS = ("live", "cells", "arms", "cost", "calib") +# Run data first, then the reference pages. They are separated in the tab +# bar because they answer different questions: everything left of the +# divider changes as the grid runs, everything right of it is ground truth +# that does not move, and mixing them invites reading a static table as a +# live one. +VIEWS = ("live", "cells", "arms", "cost", "calib", "tasks", "design") +REFERENCE_FROM = VIEWS.index("tasks") VIEW_HELP = { - "live": "runs in flight, read off their streams — [space] opens detail", - "cells": "every (arm, scenario) cell — [space] opens detail", + "live": "running · graded · per arm×scenario [h/l] section " + "[j/k] row [space] detail", + "tasks": "what each scenario asks for and how it is judged — no run " + "data [j/k] row [space] detail", + "design": "what each arm is, how they differ, and the run's ground " + "rules [j/k] row [space] detail", + "cells": "every (arm, scenario) cell [j/k] row [space] detail " + "(median/mean/p90)", "arms": "per-arm rollup; ratios are paired on scenario, geometric", "cost": "pass rate vs tokens — ★ is the Pareto frontier (§5)", "calib": "adapter vs proxy per run; >2% fails the H4 gate (§3.2)", } SORTS = ("tag", "pass", "tok") +SECTIONS = ("running", "summary", "graded") + + +def started_local(row) -> str: + """`provenance.started_at` as local wall-clock HH:MM:SS. + + Stored UTC, because a result that cannot be placed on a shared clock + is not replayable; shown local, because the operator reading it is + comparing against their own terminal.""" + ts = (row.get("provenance") or {}).get("started_at", "") + if not ts: + return "—" + try: + import datetime as _dt + d = _dt.datetime.fromisoformat(ts.replace("Z", "+00:00")) + # Date included because a grid runs for hours and can cross + # midnight; a bare clock time then sorts and reads wrongly. + return d.astimezone().strftime("%m-%d %H:%M:%S") + except (ValueError, TypeError): + return ts[-8:] or "—" +# Per-section sorts, cycled with [s] on whichever section has focus. +GRADED_SORTS = ("recent", "started", "verdict", "tok", "dur") +SUM_SORTS = ("tag", "pass", "tok", "dur", "left") +TASK_SORTS = ("id", "grader", "files", "dirs", "pr") +DESIGN_SORTS = ("arm", "always", "ondemand") GATE = 0.02 # H4 tolerance @@ -56,6 +96,7 @@ def __init__(self, stdscr, pattern="", ref="a1", proxy=None, self.proxy_path = proxy self.view = 0 self.sort = 0 + self.sort_desc = False self.editing = False self.buf = "" self.cursor = 0 @@ -65,6 +106,37 @@ def __init__(self, stdscr, pattern="", ref="a1", proxy=None, self.live = [] self.live_cursor = 0 self.live_sel = "" # out_dir the cursor is anchored to + self.act_cursor = 0 # selected activity event, newest-first index + self.act_sel = "" # its stable id -- see _anchor_act + self.act_scroll = 0 + self.act_follow = True # stay pinned to newest until you move + self.act_open = False + # Which of the live view's three tables has the cursor. [space] + # opens the selected row of whichever one is focused. + self.live_section = 0 # 0 running, 1 graded, 2 summary + self.graded_cursor = 0 + self.sum_cursor = 0 + # Each section scrolls independently. They used to be truncated to + # whatever fit, with no scroll and no indication -- so 25 graded + # rows showed 6 and the cursor could sit on one of the 19 nobody + # could see. + self.live_scroll = 0 + self.graded_scroll = 0 + self.sum_scroll = 0 + self.graded_sort = 0 + self.sum_sort = 0 + self._tasks = None # static; loaded once, never per tick + self._design = None + self.task_sort = 0 + self.task_desc = False + self.design_sort = 0 + self.design_desc = False + self.pane_scroll = 0 # shared by the scrollable detail panes + # Direction is separate from key, so [S] flips without losing the + # column you picked. Defaults are the useful end: newest graded + # first, cheapest cell first. + self.graded_desc = True + self.sum_desc = False self.proxy_rows = [] self.stamp = 0.0 self.seen_mtime = 0.0 @@ -83,6 +155,35 @@ def __init__(self, stdscr, pattern="", ref="a1", proxy=None, # ---- data ------------------------------------------------------------ + def per_cell(self, cells) -> int: + """Trials each cell will run, for the `left` and `eta` columns. + + Read from the batch's own recorded argv, because the runner already + wrote down how it was launched and a second opinion can only drift + from it. + + It used to be `expect / len(cells)`, which is wrong in a way that + only shows up mid-run: the denominator is cells that have PRODUCED a + row, not cells the suite will run. Measured at 86/120, eighteen + cells had reported against an expected 120, so it concluded 7 trials + per cell -- and every finished 5-trial cell claimed 2 still to go, + with an eta attached to work that was already done. The estimate + only converges once the last scenario starts, which is exactly when + nobody needs it. + + Falls back to the fullest cell observed: never an overestimate, so + `left` can read low early but never invents remaining work.""" + for r in reversed(self.rows): + argv = ((r.get("provenance") or {}).get("argv")) or [] + for i, a in enumerate(argv): + if a == "--trials" and i + 1 < len(argv): + try: + return max(1, int(argv[i + 1])) + except (TypeError, ValueError): + break + break # newest row is enough; one batch, one argv + return max((c["trials"] for c in cells), default=0) + def reload(self): self.cells = sd.load_cells(paths=self.files) self.rows = sd.load_rows(self.files) @@ -119,6 +220,82 @@ def reload_live(self): self.live_sel = self.live[min(self.live_cursor, len(self.live) - 1)]["out_dir"] + def scrollbar(self, y0, rows, x, total, offset): + """A track with a proportional thumb, drawn down column `x`. + + Rendered whenever the list is longer than the window, so it is + possible to tell 'there is nothing more' from 'you are at the top + of a long list' -- which a plain truncated list cannot say.""" + C = self.curses + if rows <= 0: + return + if total <= rows: + for i in range(rows): + self._put(y0 + i, x, "│", C.A_DIM) + return + size = max(1, int(rows * rows / total)) + span = rows - size + pos = int(span * offset / max(1, total - rows)) if span else 0 + for i in range(rows): + inside = pos <= i < pos + size + self._put(y0 + i, x, "█" if inside else "│", + 0 if inside else C.A_DIM) + + def section_lengths(self): + """Rows in each live section, in display order.""" + return [len(self.live), len(self.sum_cells()), + len(self.graded_rows())] + + def _move_live(self, step): + """Move one row, crossing into the next section at the edges. + + The three tables read as one column on screen, so [j] at the last + row of `running` should land on the first row of `graded` rather + than stop dead and wait for an [h]/[l] nobody thinks to press. + [h]/[l] still jump whole sections.""" + lens = self.section_lengths() + cursors = [self.live_cursor, self.sum_cursor, self.graded_cursor] + sec = self.live_section + cur = cursors[sec] + step + if cur < 0: + # off the top: previous non-empty section, at its last row + prev = next((i for i in range(sec - 1, -1, -1) if lens[i]), None) + if prev is None: + cur = 0 + else: + sec, cur = prev, lens[prev] - 1 + elif cur >= lens[sec]: + nxt = next((i for i in range(sec + 1, 3) if lens[i]), None) + if nxt is None: + cur = max(0, lens[sec] - 1) + else: + sec, cur = nxt, 0 + self.live_section = sec + cursors[sec] = max(0, cur) + self.live_cursor, self.sum_cursor, self.graded_cursor = cursors + if sec == 0: + self._anchor_live() + + def _move_rows(self, step): + """Move the results cursor, clamped to the rows that exist. + + It was unbounded, and `view_cells` clamps `scroll` but not + `cursor` -- so holding [j] walked the selection off the end of the + list, the highlight vanished, and [space] then opened a detail + pane for a row that was never on screen.""" + v = VIEWS[self.view] + n = (len(self.task_rows()) if v == "tasks" + else len(self.design_rows()) if v == "design" + else len(self.visible())) + if not n: + self.cursor = self.scroll = 0 + return + self.cursor = max(0, min(self.cursor + step, n - 1)) + # Keep the cursor in view without yanking the window around. + if self.cursor < self.scroll: + self.scroll = self.cursor + self.scroll = max(0, min(self.scroll, max(0, n - 1))) + def _anchor_live(self): """Pin the selection to a run, not to a row number.""" if self.live: @@ -131,13 +308,10 @@ def visible(self): for f, want in self.sel.items(): if want: cs = [c for c in cs if c[f] in want] - if SORTS[self.sort] == "pass": - cs = sorted(cs, key=lambda c: (-c["pass_rate"], c["tag"])) - elif SORTS[self.sort] == "tok": - cs = sorted(cs, key=lambda c: (-c["tok"], c["tag"])) - else: - cs = sorted(cs, key=lambda c: c["tag"]) - return cs + keys = {"pass": lambda c: (c["pass_rate"], c["tag"]), + "tok": lambda c: (c["tok"], c["tag"]), + "tag": lambda c: c["tag"]} + return sorted(cs, key=keys[SORTS[self.sort]], reverse=self.sort_desc) # ---- chrome ---------------------------------------------------------- @@ -152,11 +326,27 @@ def header(self, max_x): self._put(0, 1, "adherence-suite", C.A_BOLD) x = 17 for i, name in enumerate(VIEWS): + if i == REFERENCE_FROM: + self._put(0, x, " │ ", C.A_DIM) + x += 3 attr = C.color_pair(5) if i == self.view else C.A_DIM self._put(0, x, f" {name} ", attr) x += len(name) + 3 filt = self.buf if self.editing else (self.pattern or "all") - tail = f"filter:{filt} sort:{SORTS[self.sort]} ref:{self.ref}" + # The sort shown must be the one [s] would change here. It used to + # be the cells sort on every tab, so `sort:tok` sat above a task + # list that [s] did not touch. + if v == "tasks": + sort = f"{TASK_SORTS[self.task_sort]} " \ + f"{'v' if self.task_desc else '^'}" + elif v == "design": + sort = f"{DESIGN_SORTS[self.design_sort]} " \ + f"{'v' if self.design_desc else '^'}" + elif v == "live": + sort = SECTIONS[self.live_section] + else: + sort = f"{SORTS[self.sort]} {'v' if self.sort_desc else '^'}" + tail = f"filter:{filt} sort:{sort} ref:{self.ref}" self._put(0, max(x + 2, max_x - len(tail) - 2), tail, C.A_BOLD if self.editing else C.A_DIM) self._put(1, 1, VIEW_HELP[v], C.A_DIM) @@ -169,7 +359,10 @@ def header(self, max_x): pct = 100.0 * n / self.expect bar_w = 18 fill = int(bar_w * min(n / self.expect, 1.0)) - prog = (f"{'#' * fill}{'.' * (bar_w - fill)} " + # Bracketed, and a track character that cannot be mistaken for + # an ellipsis: an empty bar rendered as "................" + # reads as truncated output, not as 0%. + prog = (f"[{'█' * fill}{'·' * (bar_w - fill)}] " f"{n}/{self.expect} {pct:.0f}%") done = n >= self.expect self._put(1, max(len(VIEW_HELP[v]) + 3, max_x - len(prog) - 2), @@ -183,12 +376,41 @@ def header(self, max_x): def footer(self, max_y, max_x, total, avail): C = self.curses n_live = sum(1 for r in self.live if r["state"] == "running") - keys = [("[tab] view", C.A_DIM), - (f"[L] live·{n_live}", C.color_pair(2) if n_live else C.A_DIM), - ("[s] sort", C.A_DIM), - ("[/] filter", C.A_DIM), ("[F] clear", C.A_DIM), - ("[p] pick", C.A_DIM), ("[space] detail", C.A_DIM), - ("[r] reload", C.A_DIM), ("[q] quit", C.A_DIM)] + live_v = VIEWS[self.view] == "live" + # Depth-accurate: advertise only what the current level responds + # to. A legend listing keys that do nothing here is worse than no + # legend, because it is checked once and then trusted. + if self.act_open: + keys = [("[j/k] scroll", C.A_DIM), ("[space] back to activity", + C.A_BOLD), + ("[esc] root", C.A_DIM), ("[Q] quit", C.A_DIM)] + elif self.detail and live_v and self.live_section == 0: + keys = [("[j/k] event", C.A_DIM), ("[space] expand", C.A_DIM), + ("[q] back to list", C.A_BOLD), ("[esc] root", C.A_DIM), + ("[Q] quit", C.A_DIM)] + elif self.detail: + keys = [("[q] back", C.A_BOLD), ("[esc] root", C.A_DIM), + ("[Q] quit", C.A_DIM)] + elif live_v: + keys = [("[tab/shift-tab] view", C.A_DIM), + (f"[h/l] section: {SECTIONS[self.live_section]}", + C.color_pair(5)), + ("[j/k] row", C.A_DIM), ("[s] sort [S] asc/desc", + C.A_DIM), + ("[space] detail", C.A_DIM), ("[r] reload", C.A_DIM), + ("[q] quit", C.A_DIM)] + else: + keys = [("[tab/shift-tab] view", C.A_DIM), + (f"[L] live·{n_live}", + C.color_pair(2) if n_live else C.A_DIM), + ("[s] sort [S] asc/desc", C.A_DIM), + ("[/] filter", C.A_DIM), + ("[F] clear", C.A_DIM), ("[p] pick", C.A_DIM)] + # cost and calib have no detail pane; advertising one is worse + # than offering none, because the key is tried and does nothing. + if VIEWS[self.view] in ("cells", "tasks", "design"): + keys.append(("[space] detail", C.A_DIM)) + keys += [("[r] reload", C.A_DIM), ("[q] quit", C.A_DIM)] if self.editing: keys = [("type a filter · [enter] apply · [esc] cancel", C.A_BOLD)] self.render_footer_items(max_y, keys) @@ -201,7 +423,10 @@ def footer(self, max_y, max_x, total, avail): def render(self, max_y, max_x): self.header(max_x) - body = max_y - 3 + # Two rows more than the chrome needs: one for the scrolling + # section's range indicator, one blank, so neither the last row of + # content nor the "N of M" line sits flush against the legend. + body = max_y - 5 if self.picking: total, avail = self.view_picker(body, max_x) self.footer(max_y, max_x, total, avail) @@ -209,18 +434,46 @@ def render(self, max_y, max_x): cs = self.visible() total = avail = 0 v = VIEWS[self.view] + if v == "tasks": + rows = self.task_rows() + if self.detail and rows: + self.cursor = max(0, min(self.cursor, len(rows) - 1)) + self.view_task_detail(rows[self.cursor], max_y, max_x) + else: + total, avail = self.view_tasks(rows, body, max_x) + self.footer(max_y, max_x, total, avail) + return + if v == "design": + rows = self.design_rows() + if self.detail and rows: + self.cursor = max(0, min(self.cursor, len(rows) - 1)) + self.view_design_detail(rows[self.cursor], max_y, max_x) + else: + total, avail = self.view_design(rows, body, max_x) + self.footer(max_y, max_x, total, avail) + return if v == "live": # Deliberately not filtered by `pattern`: that filter is a # results-side idea (arm/scenario cells), and silently hiding a # running job because of it would be the worst possible # behaviour for a monitor. - if self.detail and self.live: + if self.detail and self.live_section == 2 and self.rows: + rows = self.graded_rows() + self.graded_cursor = max(0, min(self.graded_cursor, + len(rows) - 1)) + self.view_result_detail(rows[self.graded_cursor], + max_y, max_x) + elif self.detail and self.live_section == 1 and self.cells: + cs = self.sum_cells() + self.sum_cursor = max(0, min(self.sum_cursor, len(cs) - 1)) + self.view_detail(cs[self.sum_cursor], max_y, max_x) + elif self.detail and self.live: self.live_cursor = max(0, min(self.live_cursor, len(self.live) - 1)) self.view_live_detail(self.live[self.live_cursor], max_y, max_x) else: - total, avail = self.view_live(body, max_x) + total, avail = self.view_live(body, max_x, max_y) self.footer(max_y, max_x, total, avail) return if not cs and v != "calib": @@ -247,47 +500,273 @@ def render(self, max_y, max_x): total, avail = self.view_calib(body, max_x) self.footer(max_y, max_x, total, avail) - STATE_COLOR = {"running": 2, "grading": 5, "stalled": 4, "dead": 4} + STATE_COLOR = {"running": 2, "grading": 5, "done": 1, + "stalled": 4, "dead": 4} - def view_live(self, body, max_x): + def view_live(self, body, max_x, max_y=0): + """Three stacked sections: what is running, what has just been + graded, and the per-(arm, scenario) rollup that says whether the + grid is on pace. A single flat table cannot answer 'is it working' + and 'how long will this take' at the same time.""" C = self.curses + max_y = max_y or (body + 3) runs = self.live - if not runs: - self._put(3, 3, "nothing in flight — start a run, or [tab] to " - "the results views", C.A_DIM) - return 0, 0 + y = 3 + # Priority, not proportion. Running and summary are bounded -- + # one row per concurrent trial, one per (arm, scenario) cell -- so + # they fit and should simply be shown whole. Graded grows without + # bound for the life of the run, so it is the section that absorbs + # the scrolling. Sharing the space proportionally instead gave + # graded 22 rows it did not need while truncating a 3-row running + # table, and on a 24-line terminal pushed graded off the screen + # entirely -- absent, not truncated, with nothing saying so. + HEAD = 3 + n_run, n_sum, n_grad = len(runs), len(self.cells), len(self.rows) + overhead = 2 + HEAD * (bool(n_run) + bool(n_sum) + bool(n_grad)) + spare = max(1, body - overhead) + self.run_budget = min(n_run, spare) + spare -= self.run_budget + self.sum_budget = min(n_sum, spare) + spare -= self.sum_budget + # Graded takes the remainder, and at least one row so the section + # never disappears without a scrollbar to explain itself. + self.sec_budget = max(1, spare) + s = lv.summarize(runs) - self._put(3, 1, f"{s['running']} running · {s['grading']} grading · " - f"{s['stalled']} stalled · {s['calls']} calls · " - f"{s['tok_in']:,} input tokens", C.A_BOLD) - self._put(4, 1, f"{'scenario':<20}{'arm':>4}{'t':>3}{'state':>9}" - f"{'calls':>7}{'tools':>7}{'sub':>5}{'tok_in':>12}" - f"{'elapsed':>9}{'budget':>8} doing", C.A_DIM) - avail = max(1, body - 3) - self.live_cursor = max(0, min(self.live_cursor, len(runs) - 1)) - for i, r in enumerate(runs[:avail]): - y = 5 + i - base = C.color_pair(5) if i == self.live_cursor else 0 - self._put(y, 1, f"{r['scenario'][:19]:<20}", base) - self._put(y, 21, f"{r['arm']:>4}", C.A_DIM if r["unlabelled"] + done = len(self.rows) + self._put(y, 1, f"{s['running']} running · {s['grading']} grading · " + f"{s['stalled']} stalled · {done} graded" + + (f"/{self.expect}" if self.expect else "") + + f" · {s['calls']} calls · {s['tok_in']:,} in", + C.A_BOLD) + y += 2 + + # ---- running --------------------------------------------------- + if runs: + self._put(y, 1, "running" + (" ◀" if self.live_section == 0 + else ""), + C.color_pair(5) if self.live_section == 0 else C.A_DIM) + y += 1 + self._put(y, 1, f"{'scenario':<20}{'arm':>4}{'t':>3}{'state':>9}" + f"{'calls':>7}{'tools':>6}{'sub':>5}{'tok_in':>12}" + f"{'elapsed':>9}{'budget':>8} doing", C.A_DIM) + y += 1 + self.live_cursor = max(0, min(self.live_cursor, len(runs) - 1)) + rows_r = max(1, min(len(runs), self.run_budget)) + if self.live_cursor < self.live_scroll: + self.live_scroll = self.live_cursor + elif self.live_cursor >= self.live_scroll + rows_r: + self.live_scroll = self.live_cursor - rows_r + 1 + self.live_scroll = max(0, min(self.live_scroll, + max(0, len(runs) - rows_r))) + top = y + shown = runs[self.live_scroll:self.live_scroll + rows_r] + for i_off, r in enumerate(shown): + i = self.live_scroll + i_off + base = (C.color_pair(5) + if i == self.live_cursor and self.live_section == 0 + else 0) + self._put(y, 1, f"{r['scenario'][:19]:<20}", base) + self._put(y, 21, f"{r['arm']:>4}", + C.A_DIM if r["unlabelled"] else 0) + self._put(y, 25, + f"{r['trial'] if r['trial'] >= 0 else '?':>3}", + C.A_DIM) + self._put(y, 28, f"{r['state']:>9}", + C.color_pair(self.STATE_COLOR.get(r["state"], 0))) + self._put(y, 37, f"{r['total_calls']:>7}", 0) + self._put(y, 44, f"{r['tools']:>6}", C.A_DIM) + # Child SESSIONS, from opencode's own store -- the root + # stream carries none of a subagent's calls, so counting + # session ids in the stream always answered 0. + self._put(y, 50, f"{r['child_sessions']:>5}", + C.color_pair(3) if r["child_sessions"] else C.A_DIM) + self._put(y, 55, f"{r['total_tok_in']:>12,}", + C.color_pair(2)) + self._put(y, 67, f"{lv.fmt_age(r['elapsed_s']):>9}", C.A_DIM) + b = r["budget"] + self._put(y, 76, f"{lv.fmt_budget(b):>8}", + C.color_pair(4) if b and b > 0.8 else C.A_DIM) + self._put(y, 86, r["last_tool"][:max(0, max_x - 88)], C.A_DIM) + y += 1 + if len(runs) > rows_r: + self.scrollbar(top, rows_r, max_x - 2, len(runs), + self.live_scroll) + self._put(y, 1, + f" {self.live_scroll + 1}-" + f"{self.live_scroll + rows_r} of {len(runs)}" + f" [j/k] scrolls", C.A_DIM) + y += 1 + else: + self._put(y, 3, "nothing in flight", C.A_DIM) + y += 1 + y += 1 + + cells = self.sum_cells() + # ---- per (arm, scenario) rollup -------------------------------- + if not cells: + self._put(y, 1, "summary — per arm x scenario" + + (" ◀" if self.live_section == 1 else ""), + C.color_pair(5) if self.live_section == 1 else C.A_BOLD) + self._put(y + 1, 3, "nothing graded yet — a cell appears once a " + "trial finishes", C.A_DIM) + y += 3 + if cells: + self._put(y, 1, f"summary — per arm x scenario, sorted by " + f"{SUM_SORTS[self.sum_sort]} " + f"{'v' if self.sum_desc else '^'}" + + (" ◀" if self.live_section == 1 else ""), + C.color_pair(5) if self.live_section == 1 else C.A_BOLD) + y += 1 + self._put(y, 1, f"{'arm/scenario':<24}{'done':>6}{'ung':>5}" + f"{'pass':>6}{'med tok':>11}{'p90 tok':>11}" + f"{'calls':>7}{'tools':>7}{'sub':>5}{'abnd':>6}" + f"{'med dur':>9}{'left':>5}{'eta':>8}", C.A_DIM) + y += 1 + per_cell = self.per_cell(cells) + self.sum_cursor = max(0, min(self.sum_cursor, len(cells) - 1)) + rows_s = max(1, min(len(cells), self.sum_budget)) + if self.sum_cursor < self.sum_scroll: + self.sum_scroll = self.sum_cursor + elif self.sum_cursor >= self.sum_scroll + rows_s: + self.sum_scroll = self.sum_cursor - rows_s + 1 + self.sum_scroll = max(0, min(self.sum_scroll, + max(0, len(cells) - rows_s))) + top_s = y + for ci_off, c in enumerate(cells[self.sum_scroll: + self.sum_scroll + rows_s]): + ci = self.sum_scroll + ci_off + hl = (C.color_pair(5) + if ci == self.sum_cursor and self.live_section == 1 else 0) - self._put(y, 25, f"{r['trial'] if r['trial'] >= 0 else '?':>3}", - C.A_DIM) - self._put(y, 28, f"{r['state']:>9}", - C.color_pair(self.STATE_COLOR.get(r["state"], 0))) - self._put(y, 37, f"{r['calls']:>7}", 0) - self._put(y, 44, f"{r['tools']:>7}", C.A_DIM) - self._put(y, 51, f"{r['sessions'] - 1 if r['sessions'] else 0:>5}", - C.A_DIM) - self._put(y, 56, f"{r['tok_in']:>12,}", C.color_pair(2)) - self._put(y, 68, f"{lv.fmt_age(r['elapsed_s']):>9}", C.A_DIM) - # Red once the run is close enough to its timeout that it is - # probably going to be killed rather than finish. - b = r["budget"] - self._put(y, 77, f"{lv.fmt_budget(b):>8}", - C.color_pair(4) if b and b > 0.8 else C.A_DIM) - self._put(y, 87, r["last_tool"][:max(0, max_x - 89)], C.A_DIM) - return len(runs), avail + left = max(0, per_cell - c["trials"]) if per_cell else 0 + eta = left * c["dur_s"] + self._put(y, 1, f"{c['tag'][:23]:<24}", hl) + self._put(y, 25, f"{c['trials']:>6}", C.A_DIM) + self._put(y, 31, f"{c['ungradeable']:>5}", + C.color_pair(3) if c["ungradeable"] else C.A_DIM) + self._put(y, 36, f"{c['pass_rate']:>5.0f}%", + self.pass_attr(c["pass_rate"])) + skewed = (c["abandoned"] and c["tok_worked"] + and c["tok_worked"] > c["tok"] * 1.05) + self._put(y, 42, f"{c['tok']:>11,.0f}" + + ("*" if skewed else " "), + C.color_pair(4) if skewed else C.color_pair(2)) + # p90 in red when the tail is far past the middle: a median + # that halves while the tail doubles is not a saving. + tail = (c["p90_tok"] / c["tok"]) if c["tok"] else 0 + self._put(y, 53, f"{c['p90_tok']:>11,.0f}", + C.color_pair(4) if tail >= 2 else C.A_DIM) + self._put(y, 64, f"{c['calls']:>7.0f}", C.A_DIM) + self._put(y, 71, f"{c['avg_tools']:>7.1f}", C.A_DIM) + self._put(y, 78, f"{c['n_subagents']:>5.0f}", + C.color_pair(3) if c["n_subagents"] else C.A_DIM) + # Abandons in red: they never pass, and they drag the + # median token count down, so a cheap-looking cell with + # abandons is not cheap. + self._put(y, 83, f"{c['abandoned']:>6}", + C.color_pair(4) if c["abandoned"] else C.A_DIM) + self._put(y, 89, f"{lv.fmt_age(c['dur_s']):>9}", C.A_DIM) + self._put(y, 98, f"{left:>5}" if per_cell else f"{'—':>5}", + C.A_DIM) + self._put(y, 103, f"{lv.fmt_age(eta):>8}" if eta + else f"{'—':>8}", C.A_DIM) + y += 1 + if len(cells) > rows_s: + self.scrollbar(top_s, rows_s, max_x - 2, len(cells), + self.sum_scroll) + self._put(y, 1, + f" {self.sum_scroll + 1}-" + f"{self.sum_scroll + rows_s} of {len(cells)}" + f" [j/k] scrolls", C.A_DIM) + y += 1 + y += 1 # separator; the block swap dropped it + # ---- graded ---------------------------------------------------- + # Newest first: a run in progress is judged by what just landed, + # not by what landed an hour ago. + recent = self.graded_rows() + if not recent: + self._put(y, 1, "graded" + + (" ◀" if self.live_section == 2 else ""), + C.color_pair(5) if self.live_section == 2 else C.A_BOLD) + self._put(y + 1, 3, "no results yet — the first row lands when a " + "trial finishes and is graded", C.A_DIM) + y += 3 + if recent: + self._put(y, 1, f"graded — sorted by " + f"{GRADED_SORTS[self.graded_sort]} " + f"{'v' if self.graded_desc else '^'}" + + (" ◀" if self.live_section == 2 else ""), + C.color_pair(5) if self.live_section == 2 else C.A_BOLD) + y += 1 + self._put(y, 1, f"{'scenario':<20}{'arm':>4}{'t':>3}" + f"{'verdict':>13}{'calls':>7}{'tok_in':>12}" + f"{'started':>18}{'dur':>9} failing", C.A_DIM) + y += 1 + self.graded_cursor = max(0, min(self.graded_cursor, + len(recent) - 1)) + rows_g = max(1, min(len(recent), self.sec_budget)) + if self.graded_cursor < self.graded_scroll: + self.graded_scroll = self.graded_cursor + elif self.graded_cursor >= self.graded_scroll + rows_g: + self.graded_scroll = self.graded_cursor - rows_g + 1 + self.graded_scroll = max(0, min(self.graded_scroll, + max(0, len(recent) - rows_g))) + window_g = recent[self.graded_scroll:self.graded_scroll + rows_g] + top_g = y + for gi_off, r in enumerate(window_g): + gi = self.graded_scroll + gi_off + hl = (C.color_pair(5) + if gi == self.graded_cursor and self.live_section == 2 + else 0) + m = r.get("metrics") or {} + ung = [c for c in r["checks"] + if c.get("name") == "adapter" + and c.get("status") != "pass"] + verdict = ("ungradeable" if ung else + "pass" if r["all_pass"] else "fail") + col = (C.color_pair(3) if ung else + C.color_pair(1) if r["all_pass"] else C.color_pair(4)) + if ung: + # The reason lives in the adapter check's evidence. An + # ungradeable row showed "—" here, so the one column + # that explains the verdict was blank on exactly the + # rows whose verdict needs explaining. + fails = " ".join( + str(ung[0].get("evidence", "")).split()) or "harness" + else: + fails = ", ".join(c["name"] for c in r["checks"] + if c["status"] == "fail") or "—" + self._put(y, 1, f"{r['scenario'][:19]:<20}", hl) + self._put(y, 21, f"{r.get('arm', '-'):>4}", C.A_DIM) + self._put(y, 25, f"{r['trial']:>3}", C.A_DIM) + self._put(y, 28, f"{verdict:>13}", col) + # When the trial STARTED, not when its row was written -- + # results land out of order now that each is emitted as it + # finishes, so "recent" in the sort and "started" here + # answer two different questions. + self._put(y, 41, f"{m.get('calls', 0):>7}", C.A_DIM) + self._put(y, 48, f"{m.get('tok_in_billed', 0):>12,}", C.A_DIM) + # When it began and how long it took, side by side: with + # results landing in completion order the two together are + # what place a trial in the run. + self._put(y, 60, f"{started_local(r):>18}", C.A_DIM) + self._put(y, 78, f"{lv.fmt_age(r.get('duration_s', 0)):>9}", + C.A_DIM) + self._put(y, 89, fails[:max(0, max_x - 91)], + C.color_pair(3) if ung else C.A_DIM) + y += 1 + if len(recent) > rows_g: + self.scrollbar(top_g, rows_g, max_x - 2, len(recent), + self.graded_scroll) + self._put(y, 1, + f" {self.graded_scroll + 1}-" + f"{self.graded_scroll + rows_g} of {len(recent)}" + f" [j/k] scrolls", C.A_DIM) + y += 1 + y += 1 + + return 0, 0 def view_live_detail(self, r, max_y, max_x): C = self.curses @@ -336,6 +815,11 @@ def wrapped(label, text, limit=6): y += 1 info = r["info"] + if self.act_open: + # The expanded event gets the whole pane. Keeping the run block + # on screen left about eight lines for the thing being read. + self.view_event(r, max_y, max_x) + return line("scenario", r["scenario"], C.A_BOLD) line("arm / trial", f"{r['arm']} / " f"{r['trial'] if r['trial'] >= 0 else '?'}" @@ -347,12 +831,30 @@ def wrapped(label, text, limit=6): f" budget {lv.fmt_budget(r['budget'])}" f" idle {lv.fmt_age(r['idle_s'])}", C.color_pair(self.STATE_COLOR.get(r["state"], 0))) - line("cost so far", f"{r['calls']} calls · {r['tok_in']:,} in · " - f"{r['tok_out']:,} out · cache " - f"{r['cache_read']:,}r/{r['cache_write']:,}w") - if r["sessions"] > 1: - line("subagents", f"{r['sessions'] - 1} session(s), " - f"{r['subagent_calls']} of the calls above") + # parent + children, which is the only total §7 permits quoting as + # a cost. The split is shown because E3 is a claim about the child + # half specifically. + line("cost so far", f"{r['total_calls']} calls · " + f"{r['total_tok_in']:,} in · " + f"{r['tok_out'] + r['child_tok_out']:,} out · " + f"cache {r['cache_read']:,}r/" + f"{r['cache_write']:,}w") + line(" parent", f"{r['calls']} calls · {r['tok_in']:,} in") + if r["child_sessions"]: + line(" subagents", f"{r['child_calls']} calls · " + f"{r['child_tok_in']:,} in " + f"({r['child_sessions']} session(s): " + f"{', '.join(r['child_agents'][:4]) or '?'})", + C.color_pair(3)) + share = (r["child_tok_in"] / r["total_tok_in"] + if r["total_tok_in"] else 0) + line("", f"subagents are {share:.0%} of this trial's input " + f"tokens — read live from opencode's store, since the " + f"root stream carries none of them", C.A_DIM) + elif not r["children_readable"] and r["spawns"]: + line("subagents", f"{len(r['spawns'])} spawn(s) seen in the " + f"stream; opencode's store was not readable", + C.A_DIM) if info.get("pr"): line("source PR", f"#{info['pr']} base " f"{info.get('base_commit', '')[:12]}") @@ -380,14 +882,449 @@ def wrapped(label, text, limit=6): for sp in r["spawns"][:3]: line("", f"· {sp}", C.A_DIM) y += 1 - line("last tool", r["last_tool"] or "—") - wrapped("last message", r["last_text"] or "—", limit=4) - y += 1 line("sandbox", r["sandbox"] or "—", C.A_DIM) line("out dir", r["out_dir"], C.A_DIM) if r["partial_lines"]: line("note", f"{r['partial_lines']} unparsed line(s) — normal " f"while the stream is being written", C.A_DIM) + y += 1 + + y += 1 + + # ---- activity --------------------------------------------------- + # Newest first, indexed by position in the run, and deliberately + # one line each: showing every tool's output inline turned the pane + # into a wall nobody could scan. [space] opens the selected one. + room = max_y - y - 2 + if room < 4: + return + try: + ev = lv.activity(Path(r["out_dir"])) + except Exception: + ev = [] + ev = ev[::-1] # newest first + self._anchor_act(ev) + + if not ev: + self._put(y, 1, "activity", C.A_BOLD) + self._put(y + 1, 3, "nothing recorded yet", C.A_DIM) + return + + self._put(y, 1, f"activity — newest first, {len(ev)} events", + C.A_BOLD) + hint = "[j/k] move [space] open [q] back" + self._put(y, max(0, max_x - len(hint) - 3), hint, C.A_DIM) + y += 1 + rows = max_y - y - 2 + # Keep the cursor inside the window without yanking the view. + if self.act_cursor < self.act_scroll: + self.act_scroll = self.act_cursor + elif self.act_cursor >= self.act_scroll + rows: + self.act_scroll = self.act_cursor - rows + 1 + self.act_scroll = max(0, min(self.act_scroll, max(0, len(ev) - rows))) + for i, e in enumerate(ev[self.act_scroll:self.act_scroll + rows]): + idx = self.act_scroll + i + base = C.color_pair(5) if idx == self.act_cursor else 0 + self._put(y + i, 1, f"{e['n']:>5}", C.A_DIM) + self._put(y + i, 7, "sub" if e["who"] == "subagent" else " ", + C.color_pair(3) if e["who"] == "subagent" else C.A_DIM) + if e["kind"] == "tool": + label = f"{e['name']} {e['target']}" + mark = "!" if e["failed"] else " " + self._put(y + i, 11, f"{mark}{label}"[:max_x - 15], + C.color_pair(4) if e["failed"] else base) + else: + self._put(y + i, 11, + f" say {lv.preview(e, max_x - 20)}", + C.A_DIM if idx != self.act_cursor else base) + self.scrollbar(y, rows, max_x - 2, len(ev), self.act_scroll) + + def _anchor_act(self, ev): + """Resolve the activity cursor against a list that grows at the top. + + The cursor is a position, and new events arrive at position 0, so + without this the selection walks backwards one row per event and an + expanded view silently swaps to a different tool call while you are + reading it. The id is the store's primary key, so it survives. + + `act_follow` keeps the default behaviour useful: before you move, + the cursor tracks the newest event, which is what "what is it doing" + means. The first [j]/[k] pins it.""" + if not ev: + self.act_cursor, self.act_sel = 0, "" + return + if self.act_follow: + self.act_cursor, self.act_sel = 0, ev[0]["id"] + return + idx = next((i for i, e in enumerate(ev) + if e["id"] == self.act_sel), None) + if idx is not None: + self.act_cursor = idx + else: + # The selected event rolled out of the window entirely. + self.act_cursor = max(0, min(self.act_cursor, len(ev) - 1)) + self.act_sel = ev[self.act_cursor]["id"] + + def graded_rows(self): + """Graded results in the order the table shows them.""" + def verdict_rank(r): + ung = any(c.get("name") == "adapter" and c.get("status") != "pass" + for c in r["checks"]) + return (0 if ung else 1 if r["all_pass"] else 2) + + mode = GRADED_SORTS[self.graded_sort] + keys = { + "recent": lambda r: r.get("_seq", 0), + # Arrival order and start order diverge under --jobs>1, and + # each answers a different question: what landed last, versus + # what the run was doing at a given moment. + "started": lambda r: ((r.get("provenance") or {}) + .get("started_at", ""), r.get("_seq", 0)), + "verdict": lambda r: (verdict_rank(r), r.get("_seq", 0)), + "tok": lambda r: (r.get("metrics") or {}).get("tok_in_billed", 0), + "dur": lambda r: r.get("duration_s", 0), + } + return sorted(self.rows, key=keys[mode], reverse=self.graded_desc) + + def sum_cells(self): + """Per-(arm, scenario) cells in the order the table shows them.""" + cs = sd.load_cells(paths=self.files) + mode = SUM_SORTS[self.sum_sort] + pc = self.per_cell(cs) + keys = { + "tag": lambda c: c["tag"], + "pass": lambda c: (c["pass_rate"], c["tag"]), + "tok": lambda c: c["tok"], + "dur": lambda c: c["dur_s"], + # The eta the column actually shows. It used to be + # `dur_s * trials` -- time already SPENT -- so sorting by `left` + # ranked finished-and-slow cells above ones with trials still to + # run, which is the opposite of what the column is for. + "left": lambda c: (max(0, pc - c["trials"]) * c["dur_s"], c["tag"]), + } + return sorted(cs, key=keys[mode], reverse=self.sum_desc) + + def view_result_detail(self, r, max_y, max_x): + """One graded trial: every check with its evidence. + + The graded table can only fit check *names*; the evidence is where + a fail is actually explained -- which test command ran, what it + printed, which files the agent touched against the real diff.""" + C = self.curses + m = r.get("metrics") or {} + self._put(3, 1, f"{r['scenario']} arm {r.get('arm', '-')} " + f"trial {r['trial']}", C.A_BOLD) + ung = any(c.get("name") == "adapter" and c.get("status") != "pass" + for c in r["checks"]) + verdict = ("ungradeable — harness fault, excluded from pass rates" + if ung else "pass" if r["all_pass"] else "fail") + self._put(4, 1, verdict, C.color_pair(3) if ung + else C.color_pair(1) if r["all_pass"] else C.color_pair(4)) + facts = [ + ("duration", sd.fmt_duration(r.get("duration_s", 0))), + ("calls", f"{m.get('calls', 0)} " + f"({m.get('subagent_calls', 0)} in subagents)"), + ("tok_in billed", f"{m.get('tok_in_billed', 0):,}"), + ("tok_in marginal", f"{m.get('tok_in_marginal', 0):,}"), + ("first call input", f"{m.get('first_call_input', 0):,}"), + ("probes to 1st edit", f"{m.get('probes_to_first_edit', 0)}"), + ("first edit", str(m.get("first_edit") or "—")[:max_x - 32]), + ("purpose", r.get("purpose", "(unlabelled)")), + ("schema errors", ", ".join(r.get("schema_errors") or []) or "none"), + ] + y = 6 + for k, v in facts: + self._put(y, 3, f"{k:<22}", C.A_DIM) + self._put(y, 26, v) + y += 1 + y += 1 + self._put(y, 1, "checks", C.A_BOLD) + y += 1 + for c in r["checks"]: + if y >= max_y - 2: + break + col = (C.color_pair(1) if c["status"] == "pass" else + C.color_pair(4) if c["status"] == "fail" else C.A_DIM) + self._put(y, 3, f"{c['status']:<12}", col) + self._put(y, 16, c["name"]) + y += 1 + ev = " ".join(str(c.get("evidence", "")).split()) + width = max_x - 22 + while ev and y < max_y - 2: + self._put(y, 20, ev[:width], C.A_DIM) + ev = ev[width:] + y += 1 + + def view_event(self, r, max_y, max_x): + """One activity event, full pane, structure intact. + + Wraps long lines but keeps the author's own line breaks and their + leading indentation: a continuation is indented past the original + so a wrapped line is never mistaken for a new one. JSON arrives + pre-indented from live._pretty.""" + C = self.curses + try: + ev = lv.activity(Path(r["out_dir"]))[::-1] + except Exception: + ev = [] + if not ev: + self._put(3, 3, "event no longer available", C.A_DIM) + return + self._anchor_act(ev) + e = ev[self.act_cursor] + + head = (f"#{e['n']} {e['name']} {e['target']}" + if e["kind"] == "tool" else f"#{e['n']} message") + self._put(3, 1, head[:max_x - 2], + C.color_pair(4) if e["failed"] else C.A_BOLD) + meta = (f"{r['scenario']} {r['arm']}/{r['trial']} {e['who']}" + f" {e['status']} [j/k] scroll [space] back") + self._put(4, 1, meta[:max_x - 2], C.A_DIM) + + width = max_x - 6 + lines = [] + for raw in e["text"].splitlines(): + indent = len(raw) - len(raw.lstrip()) + pad = " " * min(indent, 12) + body = raw.strip() + if not body: + lines.append("") + continue + cur = pad + for word in body.split(): + if len(cur) + len(word) + 1 > width and cur.strip(): + lines.append(cur) + cur = pad + " " + word # continuation, indented + else: + cur = f"{cur} {word}" if cur.strip() else pad + word + if cur.strip(): + lines.append(cur) + + y0 = 6 + rows = max_y - y0 - 2 + self.act_scroll = max(0, min(self.act_scroll, + max(0, len(lines) - rows))) + for i, ln in enumerate(lines[self.act_scroll:self.act_scroll + rows]): + self._put(y0 + i, 3, ln[:width], + C.color_pair(4) if e["failed"] else 0) + self.scrollbar(y0, rows, max_x - 2, len(lines), self.act_scroll) + pos = f"{self.act_scroll + 1}-{min(self.act_scroll + rows, len(lines))}" \ + f"/{len(lines)} lines" + self._put(max_y - 2, max(0, max_x - len(pos) - 3), pos, C.A_DIM) + + def task_rows(self): + """Scenario descriptions, filtered by the same grammar as cells.""" + if self._tasks is None: + try: + self._tasks = tk.load() + except Exception: + self._tasks = [] + rows = [t for t in self._tasks + if sd.matches(t["id"], self.pattern)] + mode = TASK_SORTS[self.task_sort] + keys = { + "id": lambda t: t["id"], + # cli-graded first when descending: it is the exception and the + # thing worth finding. + "grader": lambda t: (t["grader"], t["id"]), + "files": lambda t: (len(t["code_files"]), t["id"]), + "dirs": lambda t: (len(t["dirs"]), t["id"]), + "pr": lambda t: (str(t["pr"]), t["id"]), + } + return sorted(rows, key=keys[mode], reverse=self.task_desc) + + def view_tasks(self, rows, body, max_x): + C = self.curses + if not rows: + self._put(3, 3, "no scenarios found", C.A_DIM) + return 0, 0 + s = tk.summarize(rows) + self._put(3, 1, f"{s['n']} scenarios — {s['unit']} judged by the " + f"PR's unit tests, {s['cli']} at the command line, " + f"{s['synthetic']} synthetic", C.A_BOLD) + self._put(4, 1, f"{'scenario':<20}{'PR':>7}{'grader':>8}{'files':>7}" + f"{'dirs':>6} what it asks for", C.A_DIM) + avail = max(1, body - 3) + self.cursor = max(0, min(self.cursor, len(rows) - 1)) + if self.cursor < self.scroll: + self.scroll = self.cursor + elif self.cursor >= self.scroll + avail: + self.scroll = self.cursor - avail + 1 + self.scroll = max(0, min(self.scroll, max(0, len(rows) - avail))) + for i, t in enumerate(rows[self.scroll:self.scroll + avail]): + y, idx = 5 + i, self.scroll + i + base = C.color_pair(5) if idx == self.cursor else 0 + self._put(y, 1, f"{t['id'][:19]:<20}", base) + self._put(y, 21, f"{str(t['pr'] or '—'):>7}", C.A_DIM) + # cli-graded is the exception and worth spotting at a glance. + self._put(y, 28, f"{t['grader'] or '—':>8}", + C.color_pair(3) if t["grader"] == "cli" else C.A_DIM) + self._put(y, 36, f"{len(t['code_files']):>7}", C.A_DIM) + self._put(y, 43, f"{len(t['dirs']):>6}", C.A_DIM) + self._put(y, 51, t["title"][:max(0, max_x - 53)], base) + self.scrollbar(5, avail, max_x - 2, len(rows), self.scroll) + return len(rows), avail + + def view_task_detail(self, t, max_y, max_x): + """One scenario: what it asks, where the answer lives, how it is + judged, and what a verdict from that grader does and does not + mean.""" + C = self.curses + w = max_x - 4 + self._put(3, 1, f"{t['id']}" + + (f" PR #{t['pr']}" if t["pr"] else "") + + f" base {t['base_commit']}", C.A_BOLD) + self._put(4, 1, t["title"][:w], 0) + + y = 6 + label, meaning = tk.GRADER_MEANING.get( + t["grader"], ("(no PR grader)", "Synthetic scenario: its own " + "grade.py decides.")) + self._put(y, 1, "judged by", C.A_DIM) + self._put(y, 16, label, + C.color_pair(3) if t["grader"] == "cli" else 0) + y += 1 + for ln in self._wrap(meaning, w - 16): + if y >= max_y - 4: + break + self._put(y, 16, ln, C.A_DIM) + y += 1 + if t["invented"]: + self._put(y, 16, "forced by: " + ", ".join(t["invented"][:6]), + C.A_DIM) + y += 1 + y += 1 + + for label, value in ( + ("answer lives in", ", ".join(t["dirs"][:6]) or "—"), + ("files the PR hit", f"{len(t['code_files'])}: " + + ", ".join(t["code_files"][:4])), + ("checked with", t["test_cmd"] or "CLI flag comparison"), + ("timeout", f"{t['timeout']}s"), + ): + self._put(y, 1, f"{label:<15}", C.A_DIM) + self._put(y, 16, str(value)[:w - 16]) + y += 1 + y += 1 + + self._put(y, 1, f"prompt ({t['prompt_lines']} lines) — this is all " + f"the agent is given", C.A_BOLD) + y += 1 + # Keep the author's line breaks; a PR body is headings and tables. + lines = [] + for raw in t["prompt"].splitlines(): + lines += self._wrap(raw, w - 4) or [""] + self._pane(lines, y, max_y, max_x) + + def _pane(self, lines, y0, max_y, max_x, x=3): + """Draw a list of lines in the space that is left, with a scrollbar. + + Detail panes used to render until they ran out of screen and then + simply stop -- on a short terminal the prompt was cut with nothing + saying so and no way to see the rest.""" + rows = max(1, max_y - y0 - 2) + self.pane_scroll = max(0, min(self.pane_scroll, + max(0, len(lines) - rows))) + for i, ln in enumerate(lines[self.pane_scroll: + self.pane_scroll + rows]): + self._put(y0 + i, x, ln[:max_x - x - 3], ln.attr + if hasattr(ln, "attr") else 0) + if len(lines) > rows: + self.scrollbar(y0, rows, max_x - 2, len(lines), self.pane_scroll) + pos = (f"{self.pane_scroll + 1}-" + f"{min(self.pane_scroll + rows, len(lines))}" + f"/{len(lines)} [j/k] scrolls") + self._put(max_y - 2, max(0, max_x - len(pos) - 3), pos, + self.curses.A_DIM) + + def design_rows(self): + if self._design is None: + try: + self._design = dz.load() + except Exception: + self._design = [] + mode = DESIGN_SORTS[self.design_sort] + keys = { + "arm": lambda r: r["arm"], + "always": lambda r: r["always_bytes"], + "ondemand": lambda r: r["ondemand_bytes"], + } + return sorted(self._design, key=keys[mode], reverse=self.design_desc) + + def view_design(self, rows, body, max_x): + C = self.curses + if not rows: + self._put(3, 3, "no arms materialized — run `make trees` or " + "point ARMSDIR at a built arms directory", + C.A_DIM) + return 0, 0 + d = dz.deltas(rows) + self._put(3, 1, "arms — instruction surface measured off disk, not " + "described", C.A_BOLD) + self._put(4, 1, f"{'arm':<5}{'name':<22}{'always':>9}{'vs a1':>9}" + f"{'on demand':>11}{'sha':>10} role", C.A_DIM) + avail = max(1, body - 3) + self.cursor = max(0, min(self.cursor, len(rows) - 1)) + for i, r in enumerate(rows[:avail]): + y = 5 + i + base = C.color_pair(5) if i == self.cursor else 0 + self._put(y, 1, f"{r['arm']:<5}", base) + self._put(y, 6, f"{r['name']:<22}", base) + if not r["present"]: + self._put(y, 28, f"{'not built':>9}", C.A_DIM) + continue + self._put(y, 28, f"{r['always_bytes']:>9,}", C.color_pair(2)) + self._put(y, 37, f"{d[r['arm']]:>+9,}", C.A_DIM) + self._put(y, 46, f"{r['ondemand_bytes']:>11,}", + C.color_pair(3) if r["ondemand_bytes"] else C.A_DIM) + self._put(y, 57, f"{r['sha8']:>10}", C.A_DIM) + self._put(y, 69, r["role"][:max(0, max_x - 71)], C.A_DIM) + y = 5 + min(len(rows), avail) + 1 + self._put(y, 1, "ground rules — [space] on an arm for its purpose " + "and files", C.A_BOLD) + y += 1 + width = max(len(k) for k, _ in dz.GROUND_RULES) + 2 + for k, v in dz.GROUND_RULES: + if y >= body + 2: + break + self._put(y, 3, k[:width - 1], 0) + for ln in self._wrap(v, max_x - width - 6)[:1]: + self._put(y, 3 + width, ln, C.A_DIM) + y += 1 + return 0, 0 + + def view_design_detail(self, r, max_y, max_x): + C = self.curses + self._put(3, 1, f"{r['arm']} — {r['name']}", C.A_BOLD) + self._put(4, 1, r["role"], C.A_DIM) + lines = [] + lines += self._wrap(r["purpose"], max_x - 8) + [""] + lines.append(f"always loaded {r['always_bytes']:,} B " + f"{', '.join(r['always_files']) or '(none)'}") + lines.append(f"on demand {r['ondemand_bytes']:,} B " + f"{len(r['ondemand_files'])} file(s)") + if r["ondemand_files"]: + lines += [" " + f for f in r["ondemand_files"][:12]] + lines.append(f"surface sha {r['sha8'] or '—'}") + lines += ["", "ground rules this run is bound by", ""] + for k, v in dz.GROUND_RULES: + lines.append(k) + lines += [" " + x for x in self._wrap(v, max_x - 12)] + lines.append("") + self._pane(lines, 6, max_y, max_x) + + @staticmethod + def _wrap(text, width): + out, cur = [], "" + for word in str(text).split(): + if len(cur) + len(word) + 1 > width and cur: + out.append(cur) + cur = word + else: + cur = f"{cur} {word}".strip() + if cur: + out.append(cur) + return out def view_cells(self, cs, body, max_x): C = self.curses @@ -396,7 +1333,12 @@ def view_cells(self, cs, body, max_x): f"{'probes':>8}{'redun':>7}{'abnd':>6} failing", C.A_DIM) avail = max(1, body - 2) + self.cursor = max(0, min(self.cursor, max(0, len(cs) - 1))) self.scroll = max(0, min(self.scroll, max(0, len(cs) - avail))) + if self.cursor < self.scroll: + self.scroll = self.cursor + elif self.cursor >= self.scroll + avail: + self.scroll = self.cursor - avail + 1 for i, c in enumerate(cs[self.scroll:self.scroll + avail]): y = 4 + i idx = self.scroll + i @@ -523,12 +1465,23 @@ def view_cost(self, cs, body, max_x): def view_calib(self, body, max_x): C = self.curses if not self.proxy_rows: - self._put(3, 3, "no proxy log loaded — pass --proxy runs/" - "proxy.jsonl, or run under bench/with-proxy.sh", - C.A_DIM) - self._put(5, 3, "Without it the adapter's token counts are " + # The old text pointed at bench/with-proxy.sh, which does not + # exist and never did, and omitted the reason this tab is empty + # for most runs: the gate cannot be measured in parallel at all. + self._put(3, 3, "no proxy log for this run.", C.A_DIM) + self._put(5, 3, "The proxy now runs by default and writes " + "alongside the results — runs/probe.jsonl " + "pairs with", C.A_DIM) + self._put(6, 3, "runs/probe.proxy.jsonl, which this view finds " + "on its own. A run started before that change, " + "or one", C.A_DIM) + self._put(7, 3, "started with ADH_NO_PROXY=1, has no log to " + "find.", C.A_DIM) + self._put(9, 3, "Without it the adapter's token counts are " "unverified: design §3.2 makes the proxy " - "authoritative, not the adapter.", C.A_DIM) + "authoritative,", C.A_DIM) + self._put(10, 3, "not the adapter, and the 2% H4 agreement " + "gate has nothing to compare.", C.A_DIM) return 0, 0 cal = sd.calibration(self.rows, self.proxy_rows) if not cal: @@ -576,23 +1529,55 @@ def view_detail(self, c, max_y, max_x): f"{c['model']} · {c['adapter']}", C.A_BOLD) arm_name, arm_role = sd.ARMS.get(c["arm"], ("?", "")) self._put(4, 1, f"arm {c['arm']} — {arm_name}: {arm_role}", C.A_DIM) + self._put(6, 3, f"{'':<26}{'median':>14}{'mean':>14}{'p90':>14}" + f"{'p90/med':>10}", C.A_DIM) + # median, mean and p90 side by side. The analysis compares medians; + # the p90 column is what says whether the median is representative, + # and the ratio is the fastest way to see a cell whose tail is + # carrying the cost. + dist = [ + ("tok_in", c["tok"], c["avg_tok"], c["p90_tok"], "{:,.0f}"), + ("calls", c["calls"], c["avg_calls"], c["p90_calls"], "{:,.1f}"), + ("tool calls", c["tools"], c["avg_tools"], c["p90_tools"], + "{:,.1f}"), + ("probes to first edit", c["probes"], c["avg_probes"], + c["p90_probes"], "{:,.1f}"), + ("duration s", c["dur_s"], c["avg_dur"], c["p90_dur"], "{:,.0f}"), + ] + y = 7 + for name, med, avg, p90, fmt in dist: + ratio = (p90 / med) if med else 0 + self._put(y, 3, f"{name:<26}", C.A_DIM) + self._put(y, 29, f"{fmt.format(med):>14}") + self._put(y, 43, f"{fmt.format(avg):>14}", C.A_DIM) + self._put(y, 57, f"{fmt.format(p90):>14}", + C.color_pair(4) if ratio >= 2 else 0) + self._put(y, 71, f"{ratio:>9.1f}x" if ratio else f"{'—':>10}", + C.color_pair(4) if ratio >= 2 else C.A_DIM) + y += 1 + y += 1 lines = [ ("trials", f"{c['trials']}"), ("pass@1", f"{c['pass_rate']:.0f}% (± {c['spread']:.0f})"), - ("median tok_in", f"{c['tok']:,.0f}"), ("median tok_in | passed", f"{c['tok_won']:,.0f}"), - ("median calls", f"{c['calls']:.0f}"), - ("probes to first edit", f"{c['probes']:.0f}"), + ("median tok_in | worked", f"{c['tok_worked']:,.0f}" + + (" (abandoned trials excluded — " + "they never pass and spend a " + "fraction of a real attempt)" + if c["abandoned"] else "")), + ("ungradeable (harness)", f"{c['ungradeable']}/{c['trials']}"), ("redundant reads", f"{c['redundant']:.0f}"), - ("subagent calls", f"{c['subagent_calls']}"), + ("subagents dispatched", f"{c['n_subagents']:.0f} " + f"(median), {c['subagent_calls']} " + f"child calls total"), + ("subagent tok_in", f"{c['subagent_tok']:,.0f} (median)"), ("abandoned trials", f"{c['abandoned']}/{c['trials']}"), - ("median duration", sd.fmt_duration(c["dur_s"])), ("failing checks", ", ".join(c["fails"]) or "—"), ] for i, (k, v) in enumerate(lines): - self._put(6 + i, 3, f"{k:<26}", C.A_DIM) - self._put(6 + i, 29, v) - y = 6 + len(lines) + 1 + self._put(y + i, 3, f"{k:<26}", C.A_DIM) + self._put(y + i, 29, v) + y = y + len(lines) + 1 self._put(y, 1, "per-trial", C.A_DIM) for i, r in enumerate(c["rows"][:max(0, max_y - y - 4)]): m = r.get("metrics") or {} @@ -672,16 +1657,156 @@ def handle_key(self, key) -> bool: return True return False - if key in (ord("q"), ord("Q"), 27): + # One rule everywhere: [q]/[esc] back out one level, [Q] quits from + # any depth. `q` used to mean both, so from inside a nested view + # there was no way to leave without pressing it repeatedly and no + # way to know whether the next press would exit the program. + # [Q] quit from anywhere · [q] back one level · [esc] back to the + # root list regardless of depth. q alone used to mean both "back" + # and "quit", so from a nested view there was no way to leave + # without pressing it repeatedly and no way to know which press + # would exit the program. + if key == ord("Q"): + return True + if key == 27: + if self.detail or self.act_open: + self.detail = self.act_open = False + return False + return True + if key == ord("q"): if self.detail: self.detail = False return False return True - if key == 9: # tab + # The live view nests: run list -> run detail with an activity + # list -> one expanded event. Each level consumes its own keys so + # [q] backs out one step instead of quitting from three levels deep. + if VIEWS[self.view] == "live" and not self.detail: + if key in (ord("l"), C.KEY_RIGHT): + self.live_section = (self.live_section + 1) % 3 + return False + if key in (ord("h"), C.KEY_LEFT): + self.live_section = (self.live_section - 1) % 3 + return False + if key in (ord("j"), C.KEY_DOWN, ord("k"), C.KEY_UP): + step = 1 if key in (ord("j"), C.KEY_DOWN) else -1 + self._move_live(step) + return False + if key in (ord(" "), 10, 13): + # Each table opens its own kind of detail: a live run, a + # graded result, or the cell's distribution. Left/right at + # this level move between sections, so descending is + # [space]/[enter] only -- binding right to both would make + # section-switching unreachable. + if self.live_section == 0 and self.live: + self.detail = True + self.act_cursor = self.act_scroll = 0 + self.act_sel, self.act_follow = "", True + self.act_open = False + elif self.live_section == 1 and self.cells or self.live_section == 2 and self.rows: + self.detail = True + return False + # Only the running section has an activity list under it. Letting + # this block run for the graded and summary details created a level + # with nothing in it: [space] appeared to do nothing and then took + # two backs to leave. + if (VIEWS[self.view] == "live" and self.detail + and self.live_section != 0): + if key == ord("Q"): + return True + if key in (ord(" "), 10, 13, ord("q"), 27, C.KEY_LEFT): + self.detail = False + return False + return False + if VIEWS[self.view] == "live" and self.detail: + if self.act_open: + if key == ord("Q"): + return True # Q quits from any depth + if key == 27: # esc: all the way out + self.act_open = self.detail = False + self.act_scroll = 0 + return False + if key == C.KEY_LEFT: # left: up one level + self.act_open = False + self.act_scroll = 0 + return False + if key in (ord(" "), 10, 13, ord("q")): + self.act_open = False + self.act_scroll = 0 + elif key in (ord("j"), C.KEY_DOWN): + self.act_scroll += 1 + elif key in (ord("k"), C.KEY_UP): + self.act_scroll = max(0, self.act_scroll - 1) + elif key == C.KEY_NPAGE: + self.act_scroll += 15 + elif key == C.KEY_PPAGE: + self.act_scroll = max(0, self.act_scroll - 15) + return False + if key in (ord("j"), C.KEY_DOWN, ord("k"), C.KEY_UP, + C.KEY_NPAGE, C.KEY_PPAGE): + step = {ord("j"): 1, C.KEY_DOWN: 1, ord("k"): -1, + C.KEY_UP: -1, C.KEY_NPAGE: 15, + C.KEY_PPAGE: -15}[key] + # Any deliberate move pins the selection; until then it + # follows the newest event. + self.act_follow = False + self.act_cursor = max(0, self.act_cursor + step) + self.act_sel = "" # resolved from the index next draw + return False + if key in (ord(" "), 10, 13, C.KEY_RIGHT): + # Expanding pins the selection unconditionally. Reading one + # event while the list follows the newest is how the thing + # you opened disappears mid-sentence. + self.act_open, self.act_scroll = True, 0 + self.act_follow = False + return False + if key == C.KEY_LEFT: # left: back to the run list + self.detail = False + self.act_cursor = self.act_scroll = 0 + return False + if key == ord("Q"): + return True + if key in (ord("q"), 27): # one level == root from here + self.detail = False + self.act_cursor = self.act_scroll = 0 + return False + if key == 9: # tab: next view self.view = (self.view + 1) % len(VIEWS) - self.scroll = 0 + self.scroll = self.cursor = 0 + elif key in (C.KEY_BTAB, 353): # shift-tab: previous + # 353 is the literal code some terminals send when terminfo + # carries no kcbt entry, so both are accepted rather than + # leaving the key dead on a terminal that reports it raw. + self.view = (self.view - 1) % len(VIEWS) + self.scroll = self.cursor = 0 + elif key == ord("S"): + if VIEWS[self.view] == "tasks": + self.task_desc = not self.task_desc + elif VIEWS[self.view] == "design": + self.design_desc = not self.design_desc + elif VIEWS[self.view] == "live" and self.live_section == 2: + self.graded_desc = not self.graded_desc + elif VIEWS[self.view] == "live" and self.live_section == 1: + self.sum_desc = not self.sum_desc + else: + # cells / arms / cost had no direction at all: [S] was + # advertised in the footer and did nothing outside live. + self.sort_desc = not self.sort_desc elif key == ord("s"): - self.sort = (self.sort + 1) % len(SORTS) + if VIEWS[self.view] == "tasks": + self.task_sort = (self.task_sort + 1) % len(TASK_SORTS) + self.cursor = self.scroll = 0 + elif VIEWS[self.view] == "design": + self.design_sort = (self.design_sort + 1) % len(DESIGN_SORTS) + self.cursor = self.scroll = 0 + elif VIEWS[self.view] == "live" and self.live_section == 2: + self.graded_sort = (self.graded_sort + 1) % len(GRADED_SORTS) + self.graded_scroll = 0 + elif VIEWS[self.view] == "live" and self.live_section == 1: + self.sum_sort = (self.sum_sort + 1) % len(SUM_SORTS) + self.sum_scroll = 0 + else: + self.sort = (self.sort + 1) % len(SORTS) elif key == ord("/"): self.editing, self.buf = True, self.pattern elif key == ord("F"): @@ -698,24 +1823,33 @@ def handle_key(self, key) -> bool: self.scroll = 0 elif key == ord(" "): self.detail = not self.detail + self.pane_scroll = 0 + if self.detail: + # Open on the newest event; that is what "what is it doing" + # means for a run still in flight. + self.act_cursor = self.act_scroll = 0 + self.act_sel, self.act_follow = "", True + self.act_open = False elif key in (ord("j"), C.KEY_DOWN): if VIEWS[self.view] == "live": self.live_cursor += 1 self._anchor_live() + elif self.detail and VIEWS[self.view] in ("tasks", "design"): + self.pane_scroll += 1 else: - self.cursor += 1 - self.scroll += 1 + self._move_rows(1) elif key in (ord("k"), C.KEY_UP): if VIEWS[self.view] == "live": self.live_cursor = max(0, self.live_cursor - 1) self._anchor_live() + elif self.detail and VIEWS[self.view] in ("tasks", "design"): + self.pane_scroll = max(0, self.pane_scroll - 1) else: - self.cursor = max(0, self.cursor - 1) - self.scroll = max(0, self.scroll - 1) + self._move_rows(-1) elif key == C.KEY_NPAGE: - self.scroll += 10 + self._move_rows(10) elif key == C.KEY_PPAGE: - self.scroll = max(0, self.scroll - 10) + self._move_rows(-10) return False @@ -751,6 +1885,9 @@ def main(): # of a run that just started, and watching it fill is the whole reason # to point the viewer at one. Only the unscoped case -- nothing named, # nothing on disk -- is a "you have not run anything yet" error. + # Find the run's own proxy log rather than making someone name it. + if not proxy: + proxy = sd.paired_proxy_log(files) or None if not files and not sd.load_rows(): print("no results found — run the suite first " "(make all), or pass FILES=", file=sys.stderr) diff --git a/src/adherence/metrics.py b/src/adherence/metrics.py index 6a05ab0..0260f35 100644 --- a/src/adherence/metrics.py +++ b/src/adherence/metrics.py @@ -183,7 +183,8 @@ def tool_calls(transcript) -> int: schema.TASK, schema.PROBE)) -def abandoned(transcript, expects_edit: bool = True) -> bool: +def abandoned(transcript, expects_edit: bool = True, + completed: bool = True, observed_edits: int = -1) -> bool: """Design §5 control 3: flag trials that terminate with fewer than 2 tool calls, or with no edit where an edit was the job. An arm whose token advantage comes from giving up sooner has to be caught here, @@ -193,11 +194,29 @@ def abandoned(transcript, expects_edit: bool = True) -> bool: answered by a report and nothing else — s04's right answer is to STOP and edit nothing at all. Flagging those as abandoned would put a red column on compliant behaviour, and a flag that cries wolf is a flag - nobody reads by the time the PR-derived tasks arrive.""" + nobody reads by the time the PR-derived tasks arrive. + + `completed=False` means the harness stopped the trial -- a deadline, a + crashed adapter -- and the flag is withheld. Giving up and being cut + off look identical in a transcript that ends early, and conflating + them puts a give-up flag on the harness's own decision. Observed on a + 45-minute trial killed at its ceiling with 27 MB of events: recorded + as abandoned, which is the opposite of what happened.""" + if not completed: + return False if tool_calls(transcript) < 2: return True if not expects_edit: return False + # `observed_edits` is git's answer, and git outranks the transcript + # here. EDIT events are emitted only for tools that name a file in + # their input; an agent that writes through the shell -- a heredoc, + # `sed -i` -- changes files that git sees and the transcript does not. + # Measured: a trial with 26 calls, 35 tool calls and every check + # passing was flagged as having given up, because it did its editing + # in bash. -1 means nobody looked, so fall back to the transcript. + if observed_edits >= 0: + return observed_edits == 0 return not any(e.get("type") == schema.EDIT for e in transcript) @@ -216,7 +235,8 @@ def turns_until_first_compaction(transcript) -> int | None: def compute(transcript, floor: int = 0, duration_s: float | None = None, - expects_edit: bool = True) -> dict: + expects_edit: bool = True, completed: bool = True, + observed_edits: int = -1) -> dict: """All per-run cost metrics. `floor` is the per-arm harness floor from E5's calibration: the input @@ -270,7 +290,26 @@ def compute(transcript, floor: int = 0, duration_s: float | None = None, "compactions": sum(1 for e in transcript if e.get("type") == schema.COMPACTION), "turns_until_first_compaction": turns_until_first_compaction(transcript), - "abandoned": abandoned(transcript, expects_edit), + "abandoned": abandoned(transcript, expects_edit, completed, + observed_edits), + # Dispatches the stream saw, against subagent sessions the + # transcript actually carries. They must match. When they do not, + # the child walker missed a session and every subagent number in + # this row is an under-report -- measured at 26 calls recorded + # against 66 the proxy handled, with two whole sessions absent. + # Recorded rather than corrected, because the proxy is the + # authority on the real figure (§3.2) and a silently patched + # transcript would hide the disagreement that says so. + "task_dispatches": sum(1 for e in transcript + if e.get("type") == schema.TASK), + "subagent_capture_gap": max( + 0, sum(1 for e in transcript if e.get("type") == schema.TASK) + - len({e.get("agent") for e in subagent_calls(transcript)})), + # What git saw, beside what the transcript claims. A gap between + # them means the agent edited through a tool the stream does not + # mark as an edit, which silently empties first_edit, probe_trail + # and every routing metric built on them. + "observed_edits": observed_edits, # §7: total_tokens is the only figure quotable as a saving; # per-agent numbers are diagnostics. Both are reported, and the # total leads. @@ -323,13 +362,33 @@ def proxy_totals(rows: list[dict]) -> dict: def split_by_mark(rows: list[dict]) -> dict: - """Group proxy records by the trial mark the runner wrote before each - trial. Records before any mark land under ''.""" + """Group proxy records by trial. + + Two mechanisms, and the per-record one wins wherever it is present: + + `run_id` is carried on the request path, so it is correct no matter how + many trials are in flight. `mark` is a single piece of proxy state set + between trials -- fine serially, meaningless under --jobs>1, where two + trials interleave into whichever label was set last. + + Records with neither land under ''.""" out, cur = {}, "" for r in rows: if r.get("type") == "mark": cur = r.get("label", "") out.setdefault(cur, []) continue - out.setdefault(cur, []).append(r) + key = trial_key(r.get("run_id") or r.get("mark") or cur) + out.setdefault(key, []).append(r) return out + + +def trial_key(run_id: str) -> str: + """`scenario|arm|trial`, the unit a result row is joined on. + + A run id carries a fourth field so two out-dirs for the same trial + (a retry, a re-run) stay distinct as processes. Calibration joins on + the trial, so the suffix is dropped here rather than at every call + site that would otherwise have to know about it.""" + parts = (run_id or "").split("|") + return "|".join(parts[:3]) if len(parts) >= 3 else (run_id or "") diff --git a/src/adherence/mkpr.py b/src/adherence/mkpr.py index 569c3ad..c0b1278 100644 --- a/src/adherence/mkpr.py +++ b/src/adherence/mkpr.py @@ -172,6 +172,51 @@ def warm_at(d: str, env: dict, eco: dict | None = None) -> None: capture_output=True, text=True, timeout=1800) +# A compiler complaining that the test names something that does not exist. +# Go's wording; the shape generalizes and each ecosystem can add its own. +_MISSING_SYMBOL = ( + re.compile(r"undefined:\s*([A-Za-z_][\w.]*)"), + re.compile(r"unknown field (\w+) in struct literal"), + re.compile(r"has no field or method (\w+)"), + re.compile(r"undeclared name:\s*(\w+)"), +) + + +def names_the_test_requires(output: str) -> list[str]: + """Identifiers the PR's own tests reference that do not exist yet. + + The fail-before check proves a task goes red at the parent commit. It + does not ask *why*, and there are two very different reasons: + + behaviour the tests compile and assert the wrong answer. An agent + that implements the behaviour passes, whatever it names + its internals. This is a real task. + compile the tests do not build, because they reference a symbol + the fix introduces -- `undefined: findCopilotBinaryFunc`, + `unknown field IssueType in struct literal`. Passing + requires independently choosing the maintainers' exact + identifier. That measures name-guessing, not the work, + and no arm can do it reliably. + + Measured on the validation grid: all four compile-class scenarios + scored 0%, 5%, 0%, 0% -- none within the [0.25, 0.80] calibration band + -- while every in-band and every ceiling scenario was behaviour-class. + A perfect separation, and the dominant cause of the floor problem the + registration names as its biggest schedule risk. + + This is why SWE-bench-style grading works for bug fixes and not for + feature PRs: a bug fix's tests call API that already exists, while a + feature PR's tests call API the PR is adding.""" + found, seen = [], set() + for pat in _MISSING_SYMBOL: + for m in pat.finditer(output or ""): + name = m.group(1) + if name not in seen: + seen.add(name) + found.append(name) + return found + + def verify(mirror: Path, parent: str, merge: str, test_files: list[str], pkgs: list[str], env: dict, eco: dict) -> tuple[bool, str]: """Fail-before / pass-after, without a model. @@ -194,6 +239,19 @@ def verify(mirror: Path, parent: str, merge: str, test_files: list[str], capture_output=True, text=True, timeout=900) if red.returncode == 0: return False, "tests already pass at the parent commit" + invented = names_the_test_requires(red.stdout + red.stderr) + if invented: + # Not a broken task -- a task the UNIT grader cannot judge + # fairly. It is handed to the CLI grader instead, which + # compares against the merge commit's own binary at the + # command line, where the contract is the one the PR body + # already gave the agent. Recorded so the choice is on the + # record rather than in a heuristic nobody can see. + return True, ("cli-graded: the PR's tests name " + + ", ".join(invented[:4]) + + ", which the fix introduces, so they cannot " + "compile against any implementation that " + "chose different identifiers") # The full post-fix state. Checking out `merge -- .` instead # would restore files the merge has but NOT remove files it # deleted, so a PR that deletes or renames a source file leaves @@ -328,14 +386,23 @@ def drop(n, why): drop(n, why) continue + # Which grader can judge this task fairly. `verify` says so: a + # task whose tests do not compile at the parent because they name + # what the fix introduces cannot be judged by those tests against + # an agent that chose different identifiers. + cli_graded = why.startswith("cli-graded:") keep({ "pr": n, "ecosystem": kind, "title": pr["title"], "base_commit": pr["base"]["sha"], "merge_commit": pr["merge_commit_sha"], "prompt": prompt, "test_files": tests, "test_cmd": eco["test_cmd"](pkgs), "code_files": code, "additions": pr.get("additions", 0), "verified": why, + "grader": "cli" if cli_graded else "unit", + "invented_symbols": (names_the_test_requires(why) + if cli_graded else []), }) - print(f" kept #{n} (+{pr.get('additions',0)}) {pr['title'][:56]}", + print(f" kept #{n} (+{pr.get('additions',0)}) " + f"[{'cli' if cli_graded else 'unit'}] {pr['title'][:52]}", file=sys.stderr) gt_f.close() diff --git a/src/adherence/mkscenarios.py b/src/adherence/mkscenarios.py index ec958de..4964ecc 100644 --- a/src/adherence/mkscenarios.py +++ b/src/adherence/mkscenarios.py @@ -20,6 +20,31 @@ import sys from pathlib import Path +# Every PR-derived prompt gets the same imperative, because a PR body is +# not reliably a task. Measured on the validation grid: prompts whose body +# describes the SOLUTION ("Use int64 for GitHub database IDs", "## Summary +# - skip the target-agent prompt when ...") drew the reply "I see you've +# shared a PR description ... What would you like me to do with this?" -- +# one inference call, no tool calls, six seconds, 8 of 14 abandoned trials. +# Prompts whose body describes a PROBLEM ("gh pr checks prints a blank +# summary when ...") were attempted normally. +# +# Worse than noise: the rate was arm-dependent (a1 12%, a2 1%), so the +# instruction surface was being credited for whether the agent guessed the +# prompt was a task at all -- a confound sitting directly on the primary +# outcome, and one that makes the quitting arm look CHEAPER because a +# refusal costs one call. +# +# The imperative names no file, package, symbol or API. It says only that +# the description is work to be done, which every arm receives identically. +TASK_PREAMBLE = ( + "Implement the change described below in this repository.\n" + "Write the code required to make it work. Do not write or modify " + "tests; they already exist and will be run against your work.\n" + "Work from the description alone -- locating the relevant code is " + "part of the task.\n\n" +) + GRADE_SHIM = '''"""Generated by adherence.mkscenarios — do not edit. Every PR-derived task shares one grader so the registered protocol has a @@ -61,6 +86,19 @@ def main(): f"the grid indistinguishable from a real one. Re-run mkpr " f"without --no-verify.") + # A task whose unit tests cannot compile AND which exposes no command + # surface is gradeable by nothing. It used to enter the suite anyway + # and pass on diff_coverage alone. + from adherence.cligrade import command_path + ungradeable = [t for t in tasks + if t.get("grader") == "cli" + and not command_path(t.get("code_files") or [])] + if ungradeable: + for t in ungradeable: + print(f" dropped #{t['pr']}: cli-graded but no pkg/cmd surface; " + f"neither grader can judge it", file=sys.stderr) + tasks = [t for t in tasks if t not in ungradeable] + prefix = args.prefix or args.fixture.replace("/", "-") out_root = Path(args.out) ids = [] @@ -71,6 +109,7 @@ def main(): (d / "grade.py").write_text(GRADE_SHIM) (d / "scenario.yaml").write_text( "# Generated by adherence.mkscenarios from a merged PR.\n" + "# A fixed imperative precedes the PR body; see mkscenarios.\n" "# The prompt has had file paths stripped: locating the work is\n" "# the thing under measurement. The agent never sees the tests.\n" f"category: pr\n" @@ -79,7 +118,7 @@ def main(): f"base_commit: {t['base_commit']}\n" f"timeout: {args.timeout}\n" f"expects_edit: 1\n" - f'prompt: "{yaml_escape(t["prompt"])}"\n') + f'prompt: "{yaml_escape(TASK_PREAMBLE + t["prompt"])}"\n') # The grader needs the test files and command, and the agent must # not: it lands as a dotfile the runner commits into the baseline # so it never appears as an agent edit, and it names no source path @@ -88,6 +127,14 @@ def main(): "pr": t["pr"], "merge_commit": t["merge_commit"], "test_files": t["test_files"], "test_cmd": t["test_cmd"], "code_files": t["code_files"], + # The mirror, so a CLI-graded task can build the PR's own + # binary as its oracle. + "repo": args.mirror, + # "unit" when the PR's tests compile against any correct + # implementation; "cli" when they name symbols the PR + # introduces, which no agent can be expected to guess. + "grader": t.get("grader", "unit"), + "invented_symbols": t.get("invented_symbols", []), }, indent=2) + "\n") ids.append(sid) diff --git a/src/adherence/prgrade.py b/src/adherence/prgrade.py index 18a61fd..1d84ed1 100644 --- a/src/adherence/prgrade.py +++ b/src/adherence/prgrade.py @@ -55,17 +55,33 @@ def grade(sandbox, transcript, final) -> list[Check]: changed = set(git_changed_files(sandbox)) real = set(task.get("code_files", [])) + cli_graded = task.get("grader") == "cli" # --- did it do the job the PR did --------------------------------- + # Under the unit grader this is a real failure: the PR's own tests are + # the verdict, and an agent that touched none of the PR's files did not + # do the work. Under the CLI grader it is REPORTED, not failed -- + # `cli.surface` already proves the shipped behaviour flag-for-flag + # against the PR's own binary, and insisting on the same file set + # would smuggle back the "match the maintainers' structure" demand + # that Amendment 2 exists to remove. hit = sorted(changed & real) coverage = len(hit) / len(real) if real else 0.0 - checks.append(ok("pr.diff_coverage", - f"touched {len(hit)}/{len(real)} of the real diff's " - f"files ({coverage:.0%}): {hit[:6]}") - if hit else - bad("pr.diff_coverage", - f"touched none of the {len(real)} files the real PR " - f"changed; agent touched {sorted(changed)[:6]}")) + if hit: + checks.append(ok("pr.diff_coverage", + f"touched {len(hit)}/{len(real)} of the real diff's " + f"files ({coverage:.0%}): {hit[:6]}")) + elif cli_graded: + checks.append(skip("pr.diff_coverage", + f"touched none of the {len(real)} files the real " + f"PR changed. Reported, not failed: this task is " + f"graded at the command line, where a different " + f"file layout with the same behaviour is a valid " + f"implementation")) + else: + checks.append(bad("pr.diff_coverage", + f"touched none of the {len(real)} files the real " + f"PR changed; agent touched {sorted(changed)[:6]}")) extra = sorted(changed - real) checks.append(skip("pr.scope", @@ -97,6 +113,32 @@ def grade(sandbox, transcript, final) -> list[Check]: "no probes recorded before the first edit; the " "adapter may not emit probe events")) + # --- CLI-boundary grading, when the unit tests cannot be fair ------ + # A feature PR's tests reference symbols the PR introduces, so they do + # not compile against an agent that named its internals differently -- + # measured: 0-5% on every such task, on runs that had already passed + # diff_coverage. mkpr marks those tasks, and they are graded against + # what the PR publicly promises instead: the merge commit's own binary + # is the oracle, and the comparison is flag-for-flag at the command + # line. Nothing is added to the prompt; the grader stops asking a + # question the agent was never given the means to answer. + if task.get("grader") == "cli": + from adherence.cligrade import grade_cli + repo = task.get("repo") or "" + merge = task.get("merge_commit", "") + if not repo or not merge: + checks.append(skip("cli.surface", + "task marked cli-graded but carries no repo " + "mirror or merge commit")) + return checks + mirror = Path(repo) + if not mirror.is_absolute(): + from adherence import REPO_ROOT + mirror = REPO_ROOT / mirror + checks += grade_cli(sandbox, mirror, merge, + task.get("code_files") or []) + return checks + # --- the maintainers' own tests decide it ------------------------- test_files = task.get("test_files") or [] merge = task.get("merge_commit", "") diff --git a/src/adherence/probe.py b/src/adherence/probe.py index 8622f63..e9666be 100644 --- a/src/adherence/probe.py +++ b/src/adherence/probe.py @@ -18,6 +18,8 @@ import sys from collections import defaultdict +from adherence.suitedata import is_ungradeable + BAND = (0.25, 0.80) MIN_USABLE = 6 # registered stopping rule in docs/EVAL.md @@ -42,12 +44,25 @@ def main() -> int: keep, ceiling, floor, broken = [], [], [], [] for sid, rs in sorted(by.items()): - rate = sum(1 for r in rs if r["all_pass"]) / len(rs) - adapter_fail = sum(1 for r in rs - if any(c["name"] == "adapter" and c["status"] == "fail" - for c in r["checks"])) - row = (sid, rate, len(rs), adapter_fail) - if adapter_fail == len(rs): + # Two defects lived here, and together they kept a dead scenario. + # + # The adapter check reports a ceiling hit as `ungradeable`, not + # `fail`, so matching on "fail" counted zero harness faults for a + # run that had three -- the summary printed "0 harness" while three + # trials had been killed at 2700s. + # + # And the rate divided by every trial, so those three counted as + # model failures: cli-cli-13057 read 40% (2 of 5) and landed inside + # the band, when its gradeable trials were 2 of 2 and it belongs + # outside. The registration is explicit -- "harness faults are not + # model failures" -- and this is the surface that prints the go/no-go + # verdict, so it was the worst of the three places to get it wrong. + broken_n = sum(1 for r in rs if is_ungradeable(r)) + graded = [r for r in rs if not is_ungradeable(r)] + rate = (sum(1 for r in graded if r["all_pass"]) / len(graded) + if graded else 0.0) + row = (sid, rate, len(graded), broken_n) + if not graded: broken.append(row) elif rate > BAND[1]: ceiling.append(row) diff --git a/src/adherence/proxy.py b/src/adherence/proxy.py index 7903063..ac97232 100644 --- a/src/adherence/proxy.py +++ b/src/adherence/proxy.py @@ -33,6 +33,7 @@ import threading import time import urllib.error +import urllib.parse import urllib.request from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -64,7 +65,12 @@ def record(self, rec: dict) -> int: self.seq += 1 rec["type"] = "call" rec["seq"] = seq - rec["mark"] = self.current_mark + # A run id carried on the request path beats the global mark and + # is the only one of the two that survives concurrency: the mark + # is one piece of shared state, so two trials in flight write + # into whichever label happened to be set last. `mark` stays for + # serial runs and for reading old logs. + rec.setdefault("mark", self.current_mark) self._write(rec) return seq @@ -123,6 +129,27 @@ def _fingerprint(body: dict) -> dict: } +# Disconnects are normal traffic, not faults. opencode opens keep-alive +# connections and drops them when a session ends, and http.server's default +# response is to dump a full traceback per connection -- which buried a +# live run's own output under hundreds of ConnectionResetErrors and made +# the proxy look like it was failing while it was working correctly. +_BENIGN = (ConnectionResetError, BrokenPipeError, ConnectionAbortedError, + TimeoutError) + + +class QuietServer(ThreadingHTTPServer): + daemon_threads = True + # Long generations hold a connection open; the default would give up. + timeout = None + + def handle_error(self, request, client_address): + exc = sys.exc_info()[1] + if isinstance(exc, _BENIGN): + return # the client hung up; nothing to say + super().handle_error(request, client_address) + + class Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" recorder: Recorder = None # set on the server class @@ -132,8 +159,42 @@ class Handler(BaseHTTPRequestHandler): def log_message(self, fmt, *a): # quiet; the JSONL is the log pass + def handle_one_request(self): + """Treat a hung-up client as the end of the conversation. + + Without this the reset propagates out of the handler and + socketserver logs it as an unhandled error. It also ends the + thread mid-request, which is what made a selftest sender die + silently on 3.10 and get reported as cross-attribution.""" + try: + super().handle_one_request() + except _BENIGN: + self.close_connection = True + # ---- control plane ------------------------------------------------- + def _split_run(self) -> str: + """Pull a `/__run/` prefix off the request path. + + This is how a trial identifies itself under --jobs>1. The runner + gives each trial a provider baseURL carrying its own run id, so the + attribution arrives with the request instead of being inferred from + proxy state that two concurrent trials would fight over. Nothing in + an inference request body identifies its trial -- measured; the + sandbox path is not even in the system prompt -- so the path is the + only place left to put it. + + Returns the id and rewrites self.path to what upstream should see. + """ + if not self.path.startswith("/__run/"): + return "" + rest = self.path[len("/__run/"):] + rid, sep, tail = rest.partition("/") + if not sep: + return "" + self.path = "/" + tail + return urllib.parse.unquote(rid) + def _control(self, body: bytes) -> bool: if not self.path.startswith("/__proxy/"): return False @@ -177,6 +238,9 @@ def _proxy(self, method: str) -> None: body = self.rfile.read(n) if n else b"" if self._control(body): return + # Strip before anything reads self.path: upstream must never see + # the routing prefix, and every record below should carry the id. + run_id = self._split_run() parsed, fp = None, {} if body: @@ -213,6 +277,7 @@ def _proxy(self, method: str) -> None: resp, status = e, e.code except Exception as e: # upstream down/refused self.recorder.record({**fp, "status": 0, "error": str(e), + **({"mark": run_id} if run_id else {}), "duration_ms": int((time.time() - t0) * 1000)}) self._json(502, {"error": f"proxy upstream: {e}"}) return @@ -226,6 +291,9 @@ def _proxy(self, method: str) -> None: rec = {**fp, "status": status, "path": self.path, "duration_ms": int((time.time() - t0) * 1000), "finish_reason": finish} + if run_id: + rec["mark"] = run_id + rec["run_id"] = run_id rec.update(usage or {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cached_tokens": 0, "usage_missing": True}) @@ -316,7 +384,7 @@ def main(): Handler.upstream = args.upstream.rstrip("/") Handler.inject_usage = not args.no_inject_usage - srv = ThreadingHTTPServer((args.host, args.port), Handler) + srv = QuietServer((args.host, args.port), Handler) srv.daemon_threads = True print(f"proxy: {args.host}:{args.port} -> {Handler.upstream} " f"log={args.log}", flush=True) diff --git a/src/adherence/runner.py b/src/adherence/runner.py index 2ef1f11..7e3ec95 100755 --- a/src/adherence/runner.py +++ b/src/adherence/runner.py @@ -274,6 +274,88 @@ def install_reapers() -> None: signal.signal(sig, _reap_all) +def per_trial_bench_config(out_dir: Path, run_id: str) -> Path | None: + """A copy of the harness config whose baseURL names this trial. + + The proxy is authoritative for token counts (H4), but it could only + attribute calls to a trial via a single global mark -- so under + --jobs>1 the runner skipped marking entirely rather than write an + attribution it knew was wrong, and the gate could not be measured in + parallel at all. That was a real limitation of the design, recorded as + "the second cost of parallelism". + + It does not have to be. Nothing in an inference request body identifies + its trial, but the request *path* is ours: routing this trial through + `/__run//v1` makes attribution arrive with the call. + The proxy strips the prefix before forwarding, so upstream sees exactly + what it saw before. + + Returns None when no proxy is in use, leaving the adapter on whatever + config it would otherwise have picked.""" + base = os.environ.get("ADH_PROXY") + src = os.environ.get("ADH_BENCH_CONFIG") or str(ROOT / "bench" / + "opencode-bench.json") + if not base or not Path(src).is_file(): + return None + try: + cfg = json.loads(Path(src).read_text()) + except (OSError, json.JSONDecodeError): + return None + + root = base.rsplit("/v1", 1)[0].rstrip("/") + tagged = f"{root}/__run/{urllib.parse.quote(run_id, safe='')}/v1" + changed = False + for prov in (cfg.get("provider") or {}).values(): + opts = prov.get("options") if isinstance(prov, dict) else None + if isinstance(opts, dict) and opts.get("baseURL"): + opts["baseURL"] = tagged + changed = True + if not changed: + return None + dest = out_dir / "opencode-bench.json" + dest.write_text(json.dumps(cfg, indent=2)) + return dest + + +def salvage(adapter: Path, sandbox: Path, model: str, prompt_file: Path, + out_dir: Path, target_agent: str, env: dict) -> int: + """Convert a killed trial's event stream into a transcript. + + The adapter writes its transcript at the end, so killing it at a + deadline discards everything: a trial that ran 45 minutes and left + 27,623,328 bytes of events on disk was recorded as `calls=0, tok=0`. + For a cost experiment that is not a missing row, it is a measurement + thrown away. + + The trial stays `ungradeable` either way -- the harness did not let it + finish, so it is not a model result and exclusion criterion 1 still + drops it. What salvage buys is the ability to say what an arm was + spending when the ceiling stopped it, which is exactly the question a + ceiling makes urgent. + + Returns the number of transcript events recovered.""" + stream = out_dir / "stdout.txt" + if not stream.is_file() or stream.stat().st_size == 0: + return 0 + if (out_dir / "transcript.jsonl").is_file(): + return 0 # the adapter got there first + salvage_env = dict(env) + salvage_env["ADH_SALVAGE"] = "1" + try: + subprocess.run( + [str(adapter), str(sandbox), model, str(prompt_file), + str(out_dir), target_agent], + capture_output=True, text=True, timeout=300, env=salvage_env, + **({"start_new_session": True} if os.name == "posix" else {})) + except (OSError, subprocess.SubprocessError): + return 0 + try: + return sum(1 for line in (out_dir / "transcript.jsonl").read_text() + .splitlines() if line.strip()) + except OSError: + return 0 + + def wait_or_kill(proc, out_dir: Path, hard_s: int, idle_s: int): """Wait for the adapter, killing it only for a reason worth killing for. @@ -297,18 +379,45 @@ def wait_or_kill(proc, out_dir: Path, hard_s: int, idle_s: int): adapter's trap a window to convert whatever the stream captured into a transcript, then SIGKILL for anything that ignored it.""" stream = out_dir / "stdout.txt" + # A subagent runs in its OWN opencode session, and the root stream + # carries none of its events -- so a trial that dispatches one and + # waits looks completely idle on stdout.txt for as long as the child + # works. Observed live: parent 0 calls, subagent 5 calls and 206,689 + # tokens, forty tool calls deep, with the stream silent for 3m27s. + # Watching only the stream would kill that run as hung, and it would do + # it *more* to the arms that route to subagents -- which are the arms + # under test. Progress is either file moving. + store = out_dir / "xdg" / "data" / "opencode" / "opencode.db" + # WAL mode: the main .db neither grows nor restamps between + # checkpoints, so summing it alone made a busy subagent look idle. + wal = Path(str(store) + "-wal") t0 = time.time() - last_size, last_change = -1, t0 + last_size, last_change, real_bytes = -1, t0, 0 + + def _activity(): + """(fingerprint, bytes). The fingerprint only has to CHANGE when + work happens; the byte count is what a human reads, so they are + kept apart -- printing the fingerprint as a byte count produced + '1,785,879,733 bytes' for a 27 MB stream.""" + import contextlib + fp = 0 + size = 0 + for f in (stream, store, wal): + with contextlib.suppress(OSError): + st = f.stat() + # size AND mtime: a WAL overwritten in place can hold its + # size while still taking writes. + fp += st.st_size + int(st.st_mtime) + size += st.st_size + return fp, size + while True: try: return proc.communicate(timeout=2.0)[1], None except subprocess.TimeoutExpired: pass now = time.time() - try: - size = stream.stat().st_size if stream.exists() else 0 - except OSError: - size = last_size + size, real_bytes = _activity() if size != last_size: last_size, last_change = size, now @@ -316,12 +425,12 @@ def wait_or_kill(proc, out_dir: Path, hard_s: int, idle_s: int): elapsed = now - t0 why = None if idle_s and idle > idle_s and last_size >= 0: - why = (f"adapter idle {int(idle)}s (no stream activity; limit " - f"{idle_s}s) after {int(elapsed)}s and {last_size:,} " - f"bytes of events") + why = (f"adapter idle {int(idle)}s (no event-stream and no " + f"session-store activity; limit {idle_s}s) after " + f"{int(elapsed)}s and {real_bytes:,} bytes") elif hard_s and elapsed > hard_s: why = (f"adapter hit the hard ceiling of {hard_s}s while still " - f"active ({last_size:,} bytes of events, last advanced " + f"active ({real_bytes:,} bytes of events, last advanced " f"{int(idle)}s ago)") if not why: continue @@ -645,6 +754,9 @@ def run_one(scen_dir: Path, adapter: Path, model: str, keep: bool, # start_new_session puts the adapter in its own process group so a # deadline can kill everything it spawned. See kill_process_group. env["ADH_RUN_ID"] = run_id + cfg = per_trial_bench_config(out_dir, run_id) + if cfg: + env["ADH_BENCH_CONFIG"] = str(cfg) proc = subprocess.Popen( [str(adapter), str(sandbox), model, str(prompt_file), str(out_dir), target_agent], @@ -657,6 +769,15 @@ def run_one(scen_dir: Path, adapter: Path, model: str, keep: bool, _LIVE_PROCS.discard(proc) adapter_ok = why is None and proc.returncode == 0 adapter_err = why or (err or "")[-2000:] + if not adapter_ok: + # A killed trial still did its work; the harness just took away the + # step that writes it down. Re-invoke the adapter in salvage mode to + # convert whatever the stream captured. Best-effort and bounded -- + # the run is already over and nothing downstream may block on it. + saved = salvage(adapter, sandbox, model, prompt_file, out_dir, + target_agent, env) + if saved: + adapter_err += f"; salvaged {saved} transcript event(s)" duration = time.time() - t0 transcript = gradelib.load_transcript(out_dir / "transcript.jsonl") @@ -709,7 +830,12 @@ def run_one(scen_dir: Path, adapter: Path, model: str, keep: bool, metrics=metrics.compute(transcript, floor=floor, duration_s=round(duration, 1), expects_edit=bool(int( - meta.get("expects_edit", 1)))), + meta.get("expects_edit", 1))), + # A trial the harness stopped cannot be + # said to have given up. + completed=adapter_ok, + observed_edits=len( + gradelib.git_changed_files(sandbox))), provenance=provenance(scen_dir, meta, arm, arms_dir, adapter, harness), fixture=str(meta.get("fixture", "")), purpose=purpose, @@ -844,9 +970,11 @@ def emit(out, item, r, errs): + (f" [out_dir={r['out_dir']}]" if r.get("out_dir") else "")) if args.jobs > 1 and os.environ.get("ADH_PROXY"): - print("WARNING: --jobs>1 with a recording proxy — per-trial proxy " - "attribution is unavailable and trial marks are skipped. Run " - "the H4 calibration serially.", file=sys.stderr) + print("note: --jobs>1 with a recording proxy. Trial marks are " + "skipped (they are one piece of shared state and two trials " + "would interleave), but each trial routes through its own " + "/__run/ path, so per-trial attribution is exact and the " + "H4 gate is measurable in parallel.", file=sys.stderr) install_reapers() with open(args.out, "a") as out: @@ -869,10 +997,23 @@ def emit(out, item, r, errs): # Threads, not processes: every trial is dominated by waiting # on a subprocess and on the endpoint, so the GIL is not the # constraint and the shared output file needs no IPC. - from concurrent.futures import ThreadPoolExecutor + from concurrent.futures import ThreadPoolExecutor, as_completed lock = threading.Lock() with ThreadPoolExecutor(max_workers=args.jobs) as pool: - for item, r, errs in pool.map(one, work): + futures = [pool.submit(one, it) for it in work] + # as_completed, NOT map: map yields in SUBMISSION order, so + # one slow trial buffers every later result inside the + # executor. Observed on the probe -- a trial finished, its + # out-dir was cleaned up, and its row stayed unwritten + # behind two 20-minute runs: invisible to the live view, + # missing from the progress count, and lost outright if the + # runner were killed. A finished trial is written now. + for fut in as_completed(futures): + try: + item, r, errs = fut.result() + except Exception as e: # noqa: BLE001 + print(f" trial raised: {e}", file=sys.stderr) + continue with lock: emit(out, item, r, errs) else: diff --git a/src/adherence/screen_repos.py b/src/adherence/screen_repos.py index e45753c..8afa30c 100644 --- a/src/adherence/screen_repos.py +++ b/src/adherence/screen_repos.py @@ -108,6 +108,111 @@ def has_path(repo: str, path: str) -> bool: return rc == 0 +# Directories whose presence means a repository can be graded on observed +# BEHAVIOUR rather than on internal API. This is the single most valuable +# thing the screen can look for, and it was not looked for: on cli/cli, 30 +# of 34 extracted tasks were unit-test-only and none acceptance-only, which +# left no way to grade a feature PR except by comparing CLI flags -- and +# that ceilings at 100%, so it discriminates between arms not at all. +BEHAVIOURAL_DIRS = ("acceptance", "e2e", "integration", "features", + "functional", "system-test", "tests/e2e") + +# Title prefixes that mark a bug fix. Fixes are worth far more than +# features here: their tests call API that already EXISTS, so an agent that +# implements the behaviour passes whatever it names its internals. A +# feature PR's tests name what the PR adds, which no agent can be expected +# to guess -- 18 of 34 cli/cli tasks landed in that bucket. +FIX_PREFIXES = ("fix", "bug", "bugfix", "hotfix", "patch") +FEATURE_PREFIXES = ("feat", "feature", "add") + +# A PR this large costs hours per trial and, measured, does not +# discriminate: cli-cli-13057 spans 13 files, takes 45 minutes, times out +# on 3 of 5 trials and passes both times it finishes. +BIG_PR_FILES = 8 + + +def pr_sample(repo: str, limit: int = 60) -> list[dict]: + """Recent merged post-cutoff PRs with the fields the new checks need.""" + code, out = gh([ + "pr", "list", "--repo", repo, "--state", "merged", "--limit", + str(limit), "--json", "title,files,mergedAt,labels"]) + if code != 0 or not out.strip(): + return [] + try: + prs = json.loads(out) + except json.JSONDecodeError: + return [] + return [p for p in prs if (p.get("mergedAt") or "") >= CUTOFF] + + +def _bucket(files: list[str]) -> str: + """What a PR actually changes. A raw merge count is not supply: + measured on cli/cli, 174 post-cutoff merges contained 32 usable tasks, + because 47.7% were CI and agent-config and 24.1% dependency bumps.""" + if not files: + return "empty" + if all(f.startswith(".github/") or "workflow" in f or "/skills/" in f + for f in files): + return "agent-config" + if all(f.endswith((".mod", ".sum", ".lock", ".toml", ".json")) + or "vendor/" in f for f in files): + return "deps" + src = [f for f in files if f.endswith((".go", ".rs", ".py", ".ts", ".js"))] + if not src: + return "docs-other" + tests = [f for f in src if "_test." in f or f.startswith("tests/") + or "/test" in f] + return "source+tests" if tests else "source-no-tests" + + +def task_supply(repo: str) -> dict: + """The checks this session's failures argued for, all from the API.""" + prs = pr_sample(repo) + if not prs: + return {"sampled": 0} + buckets, fixes, feats, big, behavioural = {}, 0, 0, 0, 0 + sizes = [] + for pr in prs: + files = [f.get("path", "") for f in (pr.get("files") or [])] + b = _bucket(files) + buckets[b] = buckets.get(b, 0) + 1 + title = (pr.get("title") or "").lower() + head = title.split(":")[0].split("(")[0].strip() + if head in FIX_PREFIXES or title.startswith("fix "): + fixes += 1 + elif head in FEATURE_PREFIXES: + feats += 1 + if b == "source+tests": + sizes.append(len(files)) + if len(files) > BIG_PR_FILES: + big += 1 + if any(f.startswith(BEHAVIOURAL_DIRS) or "/e2e/" in f + or "/acceptance/" in f for f in files): + behavioural += 1 + usable = buckets.get("source+tests", 0) + sizes.sort() + return { + "sampled": len(prs), + "buckets": buckets, + "usable": usable, + "usable_rate": round(usable / len(prs), 3), + "fix_prs": fixes, + "feature_prs": feats, + # Titles are the only cheap signal, and most repositories do not + # use conventional prefixes -- on cli/cli just 3 of 60 were + # classifiable, so a fix_rate of 1.0 there rests on three PRs. + # The denominator is reported so the rate is never read as + # stronger than its sample. + "fix_classifiable": fixes + feats, + "fix_rate": (round(fixes / (fixes + feats), 3) + if fixes + feats >= 8 else None), + "behavioural_tests": behavioural, + "behavioural_rate": round(behavioural / max(1, usable), 3), + "median_files": sizes[len(sizes) // 2] if sizes else 0, + "oversized": big, + } + + def screen(repo: str) -> dict: meta = gh_json(f"repos/{repo}") if not meta: @@ -118,6 +223,8 @@ def screen(repo: str) -> dict: instr = [f for f in INSTRUCTION_FILES if has_path(repo, f)] owners = any(has_path(repo, p) for p in CODEOWNERS_PATHS) prs = post_cutoff_prs(repo) + supply = task_supply(repo) + has_behavioural = any(has_path(repo, d) for d in BEHAVIOURAL_DIRS) fails = [] if lic not in PERMISSIVE: @@ -128,6 +235,14 @@ def screen(repo: str) -> dict: if len(dirs) < MIN_SUBSYSTEMS: fails.append(f"only {len(dirs)} top-level subsystems " f"(need {MIN_SUBSYSTEMS})") + # Raw merge count is not supply. Applied to the sample rather than the + # whole history, so it is an estimate and says so. + if supply.get("sampled") and supply["usable"] < 6: + fails.append( + f"only {supply['usable']}/{supply['sampled']} sampled PRs are " + f"source+tests ({supply['usable_rate']:.0%}); at that rate " + f"{prs} merges yield about {int(prs * supply['usable_rate'])} " + f"tasks") total_bytes = sum(langs.values()) or 1 mixed = sum(1 for v in langs.values() if v / total_bytes >= 0.10) @@ -143,10 +258,20 @@ def screen(repo: str) -> dict: "codeowners": owners, "languages": sorted(langs, key=langs.get, reverse=True)[:4], "mixed_languages": mixed, + "behavioural_suite": has_behavioural, + **{f"supply_{k}": v for k, v in supply.items()}, "fails": fails, "passes": not fails, - # Preference score, used only to order the survivors. - "score": (2 * bool(instr)) + bool(owners) + (mixed >= 2), + # Preference score, used only to order the survivors. The three new + # terms outweigh the old ones because they are what decided whether + # cli/cli could produce a usable task, and the old ones were all + # satisfied by a repo that then produced one in-band scenario in + # twelve. + "score": (2 * bool(instr)) + bool(owners) + (mixed >= 2) + + 3 * bool(has_behavioural) + + 2 * ((supply.get("fix_rate") or 0) >= 0.5) + + 2 * (supply.get("usable_rate", 0) >= 0.35) + + (supply.get("median_files", 99) <= 4), } diff --git a/src/adherence/selftest.py b/src/adherence/selftest.py index ff72819..d37cea3 100755 --- a/src/adherence/selftest.py +++ b/src/adherence/selftest.py @@ -10,6 +10,7 @@ """ from __future__ import annotations +import collections import random import shutil import subprocess @@ -621,6 +622,15 @@ def ev(**kw): f"half-written last line must be counted, not " f"raised") + # A liveness check must not be able to affect what it observes. + # os.kill(pid, 0) is a probe on POSIX and a real Ctrl-C on Windows, + # where signal.CTRL_C_EVENT == 0 -- checking our own pid took the + # whole selftest down with a KeyboardInterrupt raised inside an + # unrelated subprocess several checks later. + if lv._alive(_os.getpid()) and _os.name != "posix": + problems.append("_alive() answered on a non-POSIX platform; " + "os.kill(pid, 0) is not an existence probe there") + # No marker: scenario is recoverable from the path, arm is not. runs = lv.snapshot(tmp=tmp) if len(runs) != 1: @@ -641,8 +651,27 @@ def ev(**kw): runs = lv.snapshot(tmp=tmp) if not runs or runs[0]["arm"] != "a3" or runs[0]["trial"] != 2: problems.append("a marked run did not report its own arm/trial") - elif runs[0]["budget"] is None: - problems.append("budget not computed despite a known timeout") + elif runs[0]["budget"] is not None: + problems.append( + f"budget {runs[0]['budget']} reported for a run in state " + f"{runs[0]['state']!r}. The adapter deadline governs a trial " + f"that is still generating; a finished one is not racing " + f"anything and its clock read 104% before this was fixed") + + # ...and while a trial IS running, the budget is a real fraction and + # never exceeds the whole. + real_busy = lv.busy_out_dirs + lv.busy_out_dirs = lambda: {str(d)} + try: + runs = lv.snapshot(tmp=tmp) + finally: + lv.busy_out_dirs = real_busy + if not runs or runs[0]["state"] != "running": + problems.append("a run whose out-dir is held by a live process " + "must read as running") + elif runs[0]["budget"] is None or not 0 <= runs[0]["budget"] <= 1: + problems.append(f"budget {runs[0]['budget']} outside [0,1] for a " + f"running trial") return problems @@ -680,9 +709,12 @@ def check_process_hygiene() -> list[str]: grandchild = int((proc.stdout.readline() or "0").strip()) except ValueError: grandchild = 0 - time.sleep(0.3) - leaked = kill_process_group(proc, grace=2.0) - time.sleep(0.3) + # Generous: this runs alongside whatever else the machine is + # doing, and a check that fails under load reports a defect that + # is really CPU contention. + time.sleep(0.5) + leaked = kill_process_group(proc, grace=5.0) + time.sleep(0.5) if leaked: problems.append(f"{leaked} process(es) survived the group kill") if grandchild: @@ -713,6 +745,295 @@ def check_process_hygiene() -> list[str]: return problems +def check_proxy_attribution() -> list[str]: + """The H4 gate must be measurable with trials running concurrently. + + The proxy could only attribute a call to a trial through one global + mark, so under --jobs>1 the runner skipped marking rather than write an + attribution it knew was wrong -- and the gate, which decides whether + any cost number in this suite can be trusted, was simply unavailable + for every parallel run. The registration called that "the second cost + of parallelism". + + It is not a cost, it is a missing key. Each trial now routes through + `/__run//v1`, so attribution arrives with the request. + This drives two trials at one proxy concurrently and asserts no call + lands in the wrong bucket -- and that upstream never sees the prefix, + because a proxy that alters the request is not measuring it.""" + import json as _json + import tempfile as _tf + import threading as _th + import urllib.request as _ur + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + from pathlib import Path as _P + + from adherence import metrics as _M + from adherence import proxy as _px + problems = [] + seen_paths = set() + + class Up(BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_POST(self): + seen_paths.add(self.path) + n = int(self.headers.get("Content-Length") or 0) + self.rfile.read(n) + time.sleep(0.05) # force the requests to overlap + body = _json.dumps({ + "choices": [{"message": {"content": "ok"}, + "finish_reason": "stop"}], + "usage": {"prompt_tokens": 100, "completion_tokens": 5, + "total_tokens": 105}}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + up = ThreadingHTTPServer(("127.0.0.1", 0), Up) + _th.Thread(target=up.serve_forever, daemon=True).start() + tmp = _tf.mkdtemp(prefix="self-proxy-") + log = _P(tmp) / "proxy.jsonl" + _px.Handler.recorder = _px.Recorder(str(log)) + _px.Handler.upstream = f"http://127.0.0.1:{up.server_address[1]}" + px = ThreadingHTTPServer(("127.0.0.1", 0), _px.Handler) + _th.Thread(target=px.serve_forever, daemon=True).start() + port = px.server_address[1] + + # Count what actually got through. The invariant under test is "every + # call the proxy handled is attributed to the trial that made it", not + # "exactly N requests succeeded" -- and a transient client-side error + # (seen on 3.10's http.client under keep-alive) would otherwise be + # reported as cross-attribution, which is a wrong diagnosis for a + # networking hiccup. + sent = collections.Counter() + + def fire(run_id, n): + for _ in range(n): + req = _ur.Request( + f"http://127.0.0.1:{port}/__run/{run_id}/v1/chat/completions", + data=_json.dumps({"model": "m", "messages": [], + # tools present, or is_auxiliary() would + # class these as title-generation overhead + "tools": [{"type": "function"}]}).encode(), + headers={"Content-Type": "application/json"}, method="POST") + try: + _ur.urlopen(req, timeout=30).read() + except Exception: + continue # counted by omission, not fatal + sent[run_id] += 1 + + try: + a, b = "s01|a1|0|aaa", "s02|a3|2|bbb" + ts = [_th.Thread(target=fire, args=(a, 4)), + _th.Thread(target=fire, args=(b, 6))] + for t in ts: + t.start() + for t in ts: + t.join() + + # The proxy writes its log row AFTER the response is on the wire + # (see proxy._pump_body), so a client that has already read its body + # can be a few milliseconds ahead of the recorder. Wait for the log + # to settle instead of racing it: this is write latency, not + # attribution, and asserting straight through it is what made CI + # flaky on the slower runner. + deadline = time.time() + 5.0 + rows = [] + while True: + rows = [_json.loads(x) for x in log.read_text().splitlines() + if x.strip()] + if len(rows) >= sum(sent.values()) or time.time() > deadline: + break + time.sleep(0.05) + groups = _M.split_by_mark(rows) + + # The invariant is "no call lands in the wrong bucket", NOT "exactly + # N requests succeeded". Those came apart on 3.10, where the client + # can also raise on a keep-alive connection AFTER the proxy has + # handled and logged the call: `sent` undercounts, the log does not, + # and comparing them reported a networking hiccup as + # cross-attribution -- a red CI run for the opposite of the defect + # under test. Assert attribution directly instead. + want = {a: 4, b: 6} + expected = {_M.trial_key(k): k for k in want} + if len(rows) < 6: + problems.append(f"only {len(rows)} of 10 calls reached the proxy; " + f"it is not serving") + for key in groups: + if key not in expected: + problems.append( + f"{key}: calls attributed to a trial that never ran. " + f"Concurrent trials cross-attributed, which is what the " + f"single global mark did and the whole point of the run id") + for key, rid in expected.items(): + got = len(groups.get(key, [])) + if got < sent[rid]: + problems.append( + f"{key}: {sent[rid]} call(s) the client confirmed but only " + f"{got} attributed -- calls are being lost or misfiled") + if got > want[rid]: + problems.append( + f"{key}: {got} calls attributed but only {want[rid]} were " + f"ever sent to it; it is absorbing another trial's traffic") + first = _M.trial_key(a) + tot = _M.proxy_totals(groups.get(first, [])) + # Tied to what the proxy recorded, not to what the client confirmed: + # the question here is whether usage survives attribution. + if tot["tok_in_billed"] != 100 * len(groups.get(first, [])): + problems.append(f"tok_in_billed {tot['tok_in_billed']} does not " + f"match {len(groups.get(first, []))} attributed " + f"calls x 100 -- usage is not surviving attribution") + if seen_paths != {"/v1/chat/completions"}: + problems.append(f"upstream saw {seen_paths}; the routing prefix " + f"must be stripped, or the proxy is altering " + f"the request it claims to be measuring") + finally: + px.shutdown() + up.shutdown() + shutil.rmtree(tmp, ignore_errors=True) + return problems + + +def check_cursor_anchoring() -> list[str]: + """A cursor over a live list must follow the item, not the row. + + Both live tables grow at the top: runs start and finish, and activity + events arrive newest-first. A positional cursor therefore walks + backwards one row per arrival -- and an expanded view silently swaps to + a different tool call while it is being read. Observed twice, once per + table, before either was anchored to an id.""" + from adherence.matrix_tui import SuiteTui + problems = [] + + class Fake(SuiteTui): + def __init__(self): # no curses + self.act_cursor = 0 + self.act_sel = "" + self.act_follow = True + + def ev(ids): + return [{"id": i} for i in ids] + + t = Fake() + t._anchor_act(ev(["c", "b", "a"])) + if t.act_sel != "c": + problems.append("a fresh cursor must follow the newest event") + t._anchor_act(ev(["d", "c", "b", "a"])) + if t.act_sel != "d": + problems.append("while following, a new event must take the cursor") + + t.act_follow, t.act_cursor, t.act_sel = False, 2, "" + t._anchor_act(ev(["d", "c", "b", "a"])) + if t.act_sel != "b": + problems.append(f"pinning resolved to {t.act_sel!r}, expected 'b'") + t._anchor_act(ev(["f", "e", "d", "c", "b", "a"])) + if t.act_sel != "b" or t.act_cursor != 4: + problems.append( + f"after two new events the selection moved to {t.act_sel!r} at " + f"index {t.act_cursor}; it must stay on 'b' and slide to 4") + + t._anchor_act(ev(["z", "y", "x"])) + if t.act_sel not in {"z", "y", "x"}: + problems.append("a selection that rolled out of the window must " + "re-anchor, not leave the cursor dangling") + if t._anchor_act([]) is None and t.act_sel != "": + problems.append("an empty list must clear the selection") + return problems + + +def check_cli_grader() -> list[str]: + """Three CLI checks passing must read as success everywhere. + + A CLI-graded task's verdict is `cli.surface` against the PR's own + binary. Anything that quietly keeps a structural demand alive -- a + failing diff_coverage, say -- would drag all_pass to False and put the + task back where Amendment 2 found it: unpassable for reasons the agent + was never told.""" + from adherence.cligrade import command_path + problems = [] + + def all_pass(cs): + return (all(c["status"] == "pass" for c in cs + if c["status"] != "ungradeable") + and any(c["status"] == "pass" for c in cs)) + + def P(n): + return {"name": n, "status": "pass", "evidence": ""} + + def S(n): + return {"name": n, "status": "ungradeable", "evidence": ""} + + def F(n): + return {"name": n, "status": "fail", "evidence": ""} + + cases = [ + ("three CLI passes", [P("cli.builds"), P("cli.surface"), + P("cli.extra_surface")], True), + ("plus advisory checks", [S("pr.diff_coverage"), S("pr.scope"), + S("pr.route"), P("cli.builds"), + P("cli.surface"), P("cli.extra_surface")], + True), + ("a missing flag fails", [P("cli.builds"), F("cli.surface"), + P("cli.extra_surface")], False), + ("a broken build fails", [F("cli.builds")], False), + ] + for label, cs, want in cases: + if all_pass(cs) is not want: + problems.append(f"{label}: all_pass={all_pass(cs)}, expected " + f"{want}") + + # The command battery must be derived, never authored. + if command_path(["pkg/cmd/issue/create/create.go"]) != ["issue", "create"]: + problems.append("command path is not derived from the PR's own file " + "paths; an experimenter choosing which commands to " + "test is an experimenter choosing the result") + if command_path(["api/queries_issue.go"]): + problems.append("a PR touching no command must yield no CLI battery") + return problems + + +def check_abandoned() -> list[str]: + """`abandoned` must mean the agent gave up, and nothing else. + + Two ways it lied. It fired on a trial the HARNESS killed -- a 45-minute + run cut off at its ceiling with 27 MB of events was recorded as having + given up, the opposite of what happened. And it fired on a trial that + edited through the shell: EDIT events exist only for tools naming a + file in their input, so an agent using a heredoc or `sed -i` changes + files git can see and the transcript cannot. Measured on a trial with + 26 calls, 35 tool calls and every check passing.""" + problems = [] + t = [schema.capability(task_events=True)] + [ + schema.command("sed -i s/a/b/ f.go") for _ in range(5)] + + cases = [ + ("harness killed it", dict(completed=False), False), + ("edited via the shell, git saw 3", + dict(completed=True, observed_edits=3), False), + ("touched nothing, git agrees", + dict(completed=True, observed_edits=0), True), + ("no git answer, fall back to the transcript", + dict(completed=True, observed_edits=-1), True), + ] + for label, kw, want in cases: + got = metrics.abandoned(t, True, **kw) + if got is not want: + problems.append(f"{label}: abandoned={got}, expected {want}") + + # A real give-up still has to be caught, or the flag is useless. + quit_early = [schema.capability(task_events=True), + schema.probe("read", "a.go", 10)] + if not metrics.abandoned(quit_early, True, completed=True, + observed_edits=0): + problems.append("a trial that made one probe and stopped was not " + "flagged; an arm whose token advantage comes from " + "giving up sooner has to be caught here") + return problems + + def check_analysis() -> list[str]: """The pre-specified analysis (docs/EVAL.md) must detect a planted effect, stay silent when there is none, and REFUSE when a precondition @@ -825,6 +1146,10 @@ def dataset(ratio, floor=9500, passed=True, arms=("a1", "a2", "a3")): ("harness fault is not a model failure", check_harness_exclusion), ("live run reader", check_live), ("process hygiene", check_process_hygiene), + ("live cursor anchoring", check_cursor_anchoring), + ("cli grader verdicts", check_cli_grader), + ("abandoned means gave up", check_abandoned), + ("proxy attribution under concurrency", check_proxy_attribution), ("pre-specified analysis", check_analysis), ("stdlib test runner", check_test_runner), ("fixture noise filter", check_noise_filter), diff --git a/src/adherence/stoprun.py b/src/adherence/stoprun.py new file mode 100644 index 0000000..f062136 --- /dev/null +++ b/src/adherence/stoprun.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Stop a run and reclaim everything it left behind. Deletes no results. + + python3 -m adherence.stoprun # plan only + python3 -m adherence.stoprun --yes # do it + +Restarting a run cleanly is a five-step ritual with three ways to get it +wrong, and every one of them has already happened here: + + * `pkill -f adherence.runner` kills the runner and ORPHANS its opencode + grandchildren, which keep holding the GPU. Two of those left running + turned a --jobs 3 probe into a five-way contention for three slots, + visible only as unexplained slowness. + * the paired proxy log gets forgotten, so the next run's calibration + reads against the last run's calls. + * the temp sandboxes stay behind, and the live view keeps rendering a + torn-down run as though it were still going. + +So the awkward part is one command. + +**What this deliberately does NOT do is delete results.** Bundling "kill +the processes" with "remove the data" into a single verb is how a habit +formed for the safe purpose ends up performing the destructive one, and a +`--force` flag exists to be used. Clearing a results file stays a manual +`rm`: explicit, visible, and already a thing you know how to undo by not +typing it. This command is safe to run whenever, which is the point -- +a reclaim step nobody hesitates over is a reclaim step that actually gets +run. +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +from adherence import REPO_ROOT +from adherence.live import sweep +from adherence.runner import strays + +RUNNER_MATCH = "adherence.runner" + + +def runner_groups() -> list[tuple[int, int, str]]: + """(pid, pgid, argv) for every live runner.""" + try: + out = subprocess.run(["ps", "-eo", "pid,pgid,args"], + capture_output=True, text=True, timeout=10).stdout + except (OSError, subprocess.SubprocessError): + return [] + found = [] + for line in out.splitlines()[1:]: + parts = line.split(None, 2) + if len(parts) < 3: + continue + pid, pgid, args = parts + if RUNNER_MATCH not in args or " -eo " in args: + continue + if "python" not in args.split()[0] and "/python" not in args: + continue + try: + found.append((int(pid), int(pgid), args[:110])) + except ValueError: + continue + return found + + +def purpose_counts(path: Path) -> dict: + counts: dict[str, int] = {} + if not path.is_file(): + return counts + try: + for line in path.read_text().splitlines(): + if not line.strip(): + continue + try: + r = json.loads(line) + except json.JSONDecodeError: + continue + if "all_pass" in r: + counts[r.get("purpose", "(unlabelled)")] = counts.get( + r.get("purpose", "(unlabelled)"), 0) + 1 + except OSError: + pass + return counts + + +def stop(groups, grace: float = 15.0) -> list[str]: + """SIGTERM the whole group, then SIGKILL what ignored it.""" + notes = [] + if os.name != "posix": + import contextlib + for pid, _pgid, _a in groups: + with contextlib.suppress(OSError): + os.kill(pid, 9) + return ["no process groups on this platform; killed the runner only"] + import signal + pgids = sorted({g[1] for g in groups}) + for sig in (signal.SIGTERM, signal.SIGKILL): + alive = [] + for pgid in pgids: + try: + os.killpg(pgid, sig) + alive.append(pgid) + except ProcessLookupError: + continue + except OSError as e: + notes.append(f"pgid {pgid}: {e}") + if not alive: + break + deadline = time.time() + (grace if sig == signal.SIGTERM else 5.0) + while time.time() < deadline: + still = [] + for pgid in alive: + try: + os.killpg(pgid, 0) + still.append(pgid) + except ProcessLookupError: + pass + if not still: + return notes + time.sleep(0.3) + pgids = alive + notes.append(f"escalating to SIGKILL for {pgids}") + return notes + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--yes", action="store_true", + help="actually do it; without this only the plan prints") + ap.add_argument("--keep-tmp", action="store_true", + help="leave sandboxes and out-dirs in place") + ap.add_argument("--out", default="", + help="results file to REPORT on (never deleted)") + args = ap.parse_args() + + out = Path(args.out) if args.out else None + if out is not None and not out.is_absolute(): + out = REPO_ROOT / out + + groups = runner_groups() + left = strays() + + print("plan" if not args.yes else "stopping") + print(f" stop {len(groups)} runner(s)" + + (f", process group(s) {sorted({g[1] for g in groups})}" + if groups else "")) + for _pid, _pgid, a in groups[:4]: + print(f" {a}") + print(f" reclaim {len(left)} unowned harness process(es)") + if not args.keep_tmp: + print(" sweep sandboxes and out-dirs from finished/killed runs") + print(" delete nothing. results are never touched by this command") + + if out is not None: + counts = purpose_counts(out) + if counts: + print(f"\n{out} holds {sum(counts.values())} row(s): {counts}") + print(" clear it yourself when you mean to:") + print(f" rm {out}" + + (f" {str(out)[:-len('.jsonl')]}.proxy.jsonl" + if out.name.endswith(".jsonl") else "")) + if counts.get("experiment"): + print(" NOTE: some of those rows are marked " + "purpose=experiment — registered data, not " + "regenerable.") + + if not args.yes: + print("\nnothing done. re-run with --yes") + return 0 + + if groups: + for note in stop(groups): + print(f" ! {note}") + # Give the group a moment to die before deciding what is a stray. + time.sleep(1.0) + remaining = strays() + if remaining: + print(f" ! {len(remaining)} process(es) still running and unowned:") + for pid, d in remaining[:6]: + print(f" {pid} {d}") + print(" kill them before starting a new run, or its timings will " + "reflect contention it did not cause") + + if not args.keep_tmp: + try: + n, freed = sweep() + print(f" swept {n} director{'y' if n == 1 else 'ies'}, " + f"{freed / 1e6:.0f} MB") + except RuntimeError as e: + print(f" ! sweep skipped: {e}") + + print("\nstopped and reclaimed. results untouched.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/adherence/suitedata.py b/src/adherence/suitedata.py index 044a787..acdf44a 100644 --- a/src/adherence/suitedata.py +++ b/src/adherence/suitedata.py @@ -87,6 +87,9 @@ def load_rows(paths=None): r.setdefault("arm", "-") r.setdefault("metrics", {}) r["_src"] = os.path.basename(p) + # File order is arrival order for an append-only log, + # which is the only "most recent" a viewer can know. + r["_seq"] = len(rows) rows.append(r) except (OSError, json.JSONDecodeError): continue @@ -100,11 +103,48 @@ def newest_mtime(paths=None): # ---------- cells ---------- +def is_ungradeable(row) -> bool: + """Did the harness fail to produce something judgeable? + + One definition, used by both the `ung` column and the pass rate, so a + row cannot be shown as ungradeable and simultaneously counted as a + failure in the rate beside it. Mirrors analyze.harness_excluded -- + the registered analysis stays authoritative; this exists so the + viewer cannot disagree with it.""" + if row.get("schema_errors"): + return True + return any(c.get("name") == "adapter" and c.get("status") != "pass" + for c in row.get("checks") or []) + + def _med(vals): vals = [v for v in vals if v is not None] return st.median(vals) if vals else 0 +def _avg(vals): + vals = [v for v in vals if v is not None] + return sum(vals) / len(vals) if vals else 0 + + +def _p90(vals): + """90th percentile, nearest-rank. + + Reported next to the median rather than instead of it. A cost + comparison is made on medians because between-scenario variance is + enormous, but a median hides the runs that actually hurt: the trial + that explored for forty calls, the one that timed out. The gap between + the two is the thing to look at -- a treatment that halves the median + and doubles the tail has not made anything cheaper.""" + vals = sorted(v for v in vals if v is not None) + if not vals: + return 0 + # Nearest-rank, not interpolation: with 5-7 trials per cell an + # interpolated p90 invents a value no trial produced. + k = max(1, math.ceil(0.9 * len(vals))) + return vals[k - 1] + + def load_cells(paths=None, pattern=None): """One cell per (model, adapter, arm, scenario), trials aggregated. @@ -122,7 +162,22 @@ def load_cells(paths=None, pattern=None): won = [r for r in rs if r["all_pass"]] toks = [(r["metrics"] or {}).get("tok_in_billed", 0) for r in rs] calls = [(r["metrics"] or {}).get("calls", 0) for r in rs] - passes = [1.0 if r["all_pass"] else 0.0 for r in rs] + # Ungradeable rows are EXCLUDED from the rate, not counted against + # it. They carry all_pass=False because nothing graded them, so + # averaging over every row scores a harness fault as a model that + # got it wrong -- which the registration forbids in as many words + # ("Harness faults are not model failures", docs/EVAL.md). + # + # This disagreed with the registered analysis, which has always + # applied the exclusion. Measured on the 120-run validation grid: + # cli-cli-13057 hit the adapter's hard ceiling on 3 of 5 trials and + # read 40% here against 100% there -- and 40% is inside the + # calibration band while 100% is outside it, so the two surfaces + # disagreed about whether that scenario could discriminate at all. + # A viewer that can move a scenario in or out of band is not a + # display bug. + gradeable = [r for r in rs if not is_ungradeable(r)] + passes = [1.0 if r["all_pass"] else 0.0 for r in gradeable] fails = sorted({c["name"] for r in rs for c in r["checks"] if c["status"] == "fail"}) cells.append({ @@ -148,6 +203,40 @@ def load_cells(paths=None, pattern=None): "dur_s": _med([r.get("duration_s", 0) for r in rs]), "fails": fails, "rows": rs, + # Averages and tails, alongside the medians above. The median + # is what the analysis compares; the p90 is what says whether + # the median is telling the whole story. + "avg_tok": _avg(toks), + "p90_tok": _p90(toks), + # Cost with the give-ups removed. An abandoned trial spends a + # fraction of a real attempt, so an arm that quits more often + # looks CHEAPER on the unconditioned median -- measured at 23% + # understatement for a1 on the validation grid, against the + # very arm the treatment is compared to. The registered + # analysis conditions on success; every surface that shows a + # raw median has to be able to say so too. + "tok_worked": _med([(r["metrics"] or {}).get("tok_in_billed", 0) + for r in rs + if not (r["metrics"] or {}).get("abandoned")]), + "avg_calls": _avg(calls), + "p90_calls": _p90(calls), + "avg_dur": _avg([r.get("duration_s", 0) for r in rs]), + "p90_dur": _p90([r.get("duration_s", 0) for r in rs]), + "tools": _med([(r["metrics"] or {}).get("tool_calls", 0) + for r in rs]), + "avg_tools": _avg([(r["metrics"] or {}).get("tool_calls", 0) + for r in rs]), + "p90_tools": _p90([(r["metrics"] or {}).get("tool_calls", 0) + for r in rs]), + "avg_probes": _avg([(r["metrics"] or {}).get( + "probes_to_first_edit", 0) for r in rs]), + "p90_probes": _p90([(r["metrics"] or {}).get( + "probes_to_first_edit", 0) for r in rs]), + "n_subagents": _med([(r["metrics"] or {}).get("n_subagents", 0) + for r in rs]), + "subagent_tok": _med([(r["metrics"] or {}).get("subagent_tok_in", 0) + for r in rs]), + "ungradeable": sum(1 for r in rs if is_ungradeable(r)), }) cells.sort(key=lambda c: (c["arm"], c["scenario"])) if pattern: @@ -245,6 +334,20 @@ def pareto_front(cells, cost="ktok", quality="pass_rate"): # ---------- proxy / calibration ---------- +def paired_proxy_log(paths): + """The proxy log belonging to these results, if one exists. + + isolate.sh writes runs/probe.jsonl -> runs/probe.proxy.jsonl, so the + pairing is derivable and the viewer should not need to be told. Asking + the user to pass --proxy for a file the harness just wrote next to the + results is the kind of step that guarantees the H4 tab stays empty.""" + for p in paths or []: + cand = p[:-len(".jsonl")] + ".proxy.jsonl" if p.endswith(".jsonl") else "" + if cand and os.path.exists(cand): + return cand + return "" + + def load_proxy(path): rows = [] try: @@ -270,6 +373,15 @@ def calibration(rows, proxy_rows): by_mark = M.split_by_mark(proxy_rows) out = [] for r in rows: + # A trial the harness stopped has a transcript truncated by + # construction -- or, before salvage existed, none at all. Compared + # against the proxy's complete record it reads as a 100% accounting + # error, which is not what H4 measures and buries the disagreements + # that are real. Excluded here for the same reason exclusion + # criterion 1 drops it from every pass rate. + if any(c.get("name") == "adapter" and c.get("status") != "pass" + for c in r.get("checks", [])): + continue mark = f"{r['scenario']}|{r.get('arm', '-')}|{r['trial']}" pr = by_mark.get(mark) if pr is None: diff --git a/src/adherence/tasks.py b/src/adherence/tasks.py new file mode 100644 index 0000000..c090cbd --- /dev/null +++ b/src/adherence/tasks.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""What each scenario asks for, and how it will be judged. + + python3 -m adherence.tasks # one-shot text listing + +Orientation, not results. An operator watching a grid sees scenario ids +and pass rates and has no way to tell what `cli-cli-13418` actually wants, +which files the answer lives in, or why one task is judged by unit tests +and the next at the command line. That gap is where a harness artifact +gets mistaken for a model failure -- the whole reason Amendment 2 exists. + +Everything here is static: it reads the scenario directory, never a run. +Stdlib only. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from adherence import REPO_ROOT +from adherence.runner import load_yamlish + +# What each grader actually decides, in the terms an operator needs when +# reading a verdict rather than in the terms the code is written in. +GRADER_MEANING = { + "unit": ("the PR's own unit tests", + "The agent's tree is built and the PR's `_test.go` files are " + "applied on top, then the affected packages run. The tests " + "compile against any correct implementation, so internal " + "naming does not matter."), + "cli": ("the PR's own binary, flag-for-flag", + "These tests name symbols the PR introduces, so they cannot " + "compile against an implementation that chose different " + "identifiers -- passing them would require guessing the " + "maintainers' names. The merge commit is built as an oracle " + "and the agent's binary is compared to it at the command " + "line, where the contract is the one the PR body already " + "gave the agent (Amendment 2)."), +} + + +def load(root: Path | None = None) -> list[dict]: + """One record per scenario, sorted by id.""" + root = root or REPO_ROOT + out = [] + for d in sorted((root / "scenarios").glob("*")): + y = d / "scenario.yaml" + if not y.is_file(): + continue + try: + meta = load_yamlish(y) + except (OSError, ValueError): + continue + task = {} + t = d / "task.json" + if t.is_file(): + try: + task = json.loads(t.read_text()) + except (OSError, json.JSONDecodeError): + task = {} + prompt = str(meta.get("prompt", "")) + title = next((ln.strip() for ln in prompt.splitlines() if ln.strip()), + "(no title)") + out.append({ + "id": d.name, + "category": str(meta.get("category", "")), + "fixture": str(meta.get("fixture", "")), + "base_commit": str(meta.get("base_commit", ""))[:12], + "timeout": int(meta.get("timeout", 0) or 0), + "title": title, + "prompt": prompt, + "prompt_lines": len(prompt.splitlines()), + "pr": task.get("pr", ""), + "grader": task.get("grader", "unit" if task else ""), + "invented": task.get("invented_symbols") or [], + "code_files": task.get("code_files") or [], + "test_files": task.get("test_files") or [], + "test_cmd": task.get("test_cmd", ""), + "dirs": sorted({str(Path(p).parent) + for p in (task.get("code_files") or [])}), + }) + return out + + +def summarize(rows) -> dict: + return { + "n": len(rows), + "unit": sum(1 for r in rows if r["grader"] == "unit"), + "cli": sum(1 for r in rows if r["grader"] == "cli"), + "synthetic": sum(1 for r in rows if not r["pr"]), + } + + +def main(): + rows = load() + if not rows: + print("no scenarios found") + return 0 + s = summarize(rows) + print(f"{s['n']} scenarios — {s['unit']} unit-graded, {s['cli']} " + f"cli-graded, {s['synthetic']} synthetic\n") + print(f"{'scenario':<20}{'PR':>7}{'grader':>8}{'files':>7}{'dirs':>6} what it asks for") + for r in rows: + print(f"{r['id']:<20}{str(r['pr'] or '—'):>7}{r['grader'] or '—':>8}" + f"{len(r['code_files']):>7}{len(r['dirs']):>6} {r['title'][:70]}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())