From fc7521722dd12447057523a86cd10619f9625671 Mon Sep 17 00:00:00 2001 From: BillWang Date: Tue, 18 Aug 2026 15:12:37 +0800 Subject: [PATCH] feat: add behavior-validated context selection --- CHANGELOG.md | 11 + README.md | 89 ++- denser/__init__.py | 18 + denser/audit.py | 2 + denser/backends/codex_cli.py | 52 +- denser/cli.py | 222 +++++++ denser/context_selection.py | 445 +++++++++++++ denser/replay.py | 60 +- docs/CODEX_CONTEXT_SELECTION_CASE_STUDY.md | 101 +++ docs/DESIGN.md | 12 + .../context_bundles/tool_workflows/README.md | 15 + .../tool_workflows/archived-handbook.md | 113 ++++ .../tool_workflows/bundle.json | 29 + .../tool_workflows/ci-policy.md | 8 + .../tool_workflows/execution-contract.md | 9 + .../tool_workflows/fixtures/ci.json | 6 + .../tool_workflows/fixtures/release.json | 6 + .../tool_workflows/release-policy.md | 8 + .../tool_workflows/replay.json | 42 ++ .../selected.codex-standard.2026-08-18.md | 33 + ...election.codex-standard.3x.2026-08-18.json | 629 ++++++++++++++++++ tests/test_audit.py | 46 ++ tests/test_context_selection.py | 364 ++++++++++ 23 files changed, 2260 insertions(+), 60 deletions(-) create mode 100644 denser/context_selection.py create mode 100644 docs/CODEX_CONTEXT_SELECTION_CASE_STUDY.md create mode 100644 examples/context_bundles/tool_workflows/README.md create mode 100644 examples/context_bundles/tool_workflows/archived-handbook.md create mode 100644 examples/context_bundles/tool_workflows/bundle.json create mode 100644 examples/context_bundles/tool_workflows/ci-policy.md create mode 100644 examples/context_bundles/tool_workflows/execution-contract.md create mode 100644 examples/context_bundles/tool_workflows/fixtures/ci.json create mode 100644 examples/context_bundles/tool_workflows/fixtures/release.json create mode 100644 examples/context_bundles/tool_workflows/release-policy.md create mode 100644 examples/context_bundles/tool_workflows/replay.json create mode 100644 examples/context_bundles/tool_workflows/selected.codex-standard.2026-08-18.md create mode 100644 examples/context_bundles/tool_workflows/selection.codex-standard.3x.2026-08-18.json create mode 100644 tests/test_context_selection.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 66ad02e..4b3ed3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project are documented here. The format follows [Kee ## [Unreleased] +### Added +- Added `denser minimize-context` and a versioned context-bundle manifest. The + selector greedily tests optional components, keeps behavior-changing or + uncertain components, requires a sensitive negative control, and repeats a + final audit against a caller-set full-input reduction target. +- Added bounded concurrent replay for the Codex CLI backend. Per-call metadata + remains thread-isolated and each CLI turn remains ephemeral and read-only. +- Added a two-task local-file Codex pilot. It removed an unrelated archived + handbook, retained the release and CI policies whose removal regressed, kept + all final cases at 3/3, and reduced provider-reported full input by 12.47%. + ## [0.2.0-alpha.3] — 2026-08-18 ### Changed diff --git a/README.md b/README.md index 9922d27..c926872 100644 --- a/README.md +++ b/README.md @@ -20,43 +20,55 @@ --- -## Featured: measurable Codex input reduction for text-only tasks +## Featured: automatic context pruning in a normal Codex tool workflow -The first result that clears denser's end-to-end bar comes from capability -selection, not prose compression. For replay tasks that need no files, shell, -network, plugins, apps, skills, or memory, the Codex CLI adapter can use an -explicit `text-only` profile and omit those unused capabilities from the model -input. `standard` remains the default. +`denser minimize-context` takes a manifest of visible context components, +tries removing optional components one at a time, and keeps a removal only when +behavior remains identical and a known-bad control still fails. Errors, +improvements that change behavior, and insensitive tests all fail closed. -With Codex CLI 0.147.0, `gpt-5.6-sol`, and medium reasoning: +The first tool-using pilot used Codex CLI 0.147.0, `gpt-5.6-sol`, medium +reasoning, and the `standard` capability profile. Both tasks had to read a local +JSON file whose identifier and arbitrary policy code were absent from the +prompt, so the expected answers could not be recovered from a text-only input. -| Workload | Quality | Full input per call | Reduction | +| Workload | Complete context | Selected context | +|---|---:|---:| +| Local release-record decision | 3/3 | 3/3 | +| Local CI-record decision | 3/3 | 3/3 | + +The selector removed a 17,236-byte archived handbook but rejected attempts to +remove the release or CI policy because each caused a covered regression. The +final three-trial audit completed with zero operational errors: + +| Measurement | Complete context | Selected context | Reduction | |---|---:|---:|---:| -| Release-operation decisions | 27/27 in each profile | 20,294.11 → 18,154.00 | 10.55% | -| Automation permission routing | 15/15 in each profile | 20,619.00 → 18,434.00 | 10.60% | - -This clears the predeclared rule of at least two real scenarios with at least -10% provider-reported full-input reduction and no observed quality loss. The -final run made 84 authenticated calls in seeded randomized order, with three -trials per case, zero operational errors, and zero transport fallbacks. It is -not a general coding mode: tasks that need tools must use `standard`. - -The first strict run caught one regression: without tools, one case asked for -more context instead of following its fixed output contract. The `text-only/v1` -wrapper now states that all required input is already present, and the complete -84-call audit was rerun rather than patching the single failure. This is the -kind of false confidence denser is designed to expose. - -The earlier 10.3%-shorter instruction rewrite reduced full Codex input by only -about 0.25%. That negative result remains important: rewriting a small file is -not enough when the larger cost is unused runtime context. - -See the [case study and reproduction -guide](docs/CODEX_TEXT_ONLY_CASE_STUDY.md), plus the complete per-call outputs, -token counts, source hashes, runtime settings, and limitations in the +| Provider-reported full input, 6 calls | 277,871 | 243,210 | **12.47%** | +| Visible bundle estimate | 4,626 | 303 | 93.45% | + +Shell access, plugins, and skill search were not disabled. The benchmark's +standard profile did disable apps, memories, and multi-agent execution for +reproducibility, identically on both sides; the measured 12.47% delta comes +from the selected bundle, not from changing that runtime profile. This is one +synthetic two-task pilot, not proof that every repository can remove 12%. + +See the [component-selection case study](docs/CODEX_CONTEXT_SELECTION_CASE_STUDY.md), +the [manifest and tool fixtures](examples/context_bundles/tool_workflows/), and +the [complete final audit](examples/context_bundles/tool_workflows/selection.codex-standard.3x.2026-08-18.json). + +### Earlier result: text-only capability selection + +For pre-bundled decisions that need no files or tools, the explicit +`text-only/v1` profile reduced full input by 10.55% and 10.60% across two +synthetic workloads, with 42/42 expected decisions per profile. That remains a +narrow capability-selection result, not the main product claim. A +10.3%-shorter instruction rewrite had reduced complete Codex input by only +about 0.25%, which is why denser now targets whole visible context bundles. + +See the earlier [text-only case study](docs/CODEX_TEXT_ONLY_CASE_STUDY.md) and [`paired three-trial audit`](examples/project_instructions/codex-text-only-profile-audit.paired-3x-final.2026-08-17.json). -### Public-project transfer check: Astral uv +### Earlier public-project transfer check: Astral uv The same frozen profile was then tested against decision rules adapted from public Astral uv agent prompts at commit @@ -102,6 +114,11 @@ end-to-end input usage separately from asset-only length. ## What denser does ```bash +denser minimize-context context-bundle.json --suite replay.json \ + --backend codex-cli --codex-capability-profile standard \ + --selection-trials 1 --validation-trials 3 --parallelism 6 \ + --out selected-context.md --json-out selection-evidence.json + denser audit AGENTS.md AGENTS.variant.md --type claude_md \ --suite replay.holdout.json \ --negative-control AGENTS.negative-control.md \ @@ -121,7 +138,15 @@ denser replay --type claude_md AGENTS.md --suite replay.json \ --backend codex-cli --codex-capability-profile text-only ``` -`audit` is the primary interface. It runs paired baseline/variant replay, +`minimize-context` is the automatic selective-loading interface. A versioned +manifest names the visible context components, marks non-removable components, +and declares which required component to drop for the known-bad control. The +selector tests optional components largest first, retains every uncertain or +behavior-changing component, then repeats a final audit. It operates on +user-supplied visible text; it does not inspect hidden provider prefixes or +compact conversation history. + +`audit` is the primary lower-level interface. It runs paired baseline/variant replay, compares every covered case, checks whether a known-bad negative control causes a regression, and reports both asset-only estimates and provider-reported full input usage. Equal scores without a detected negative control are diff --git a/denser/__init__.py b/denser/__init__.py index e40566b..27d64bd 100644 --- a/denser/__init__.py +++ b/denser/__init__.py @@ -18,6 +18,16 @@ audit_context, ) from denser.compress import CompressionResult, compress +from denser.context_selection import ( + CONTEXT_BUNDLE_SCHEMA_VERSION, + CONTEXT_SELECTION_SCHEMA_VERSION, + ComponentAttempt, + ContextBundle, + ContextComponent, + ContextSelectionReport, + load_context_bundle, + minimize_context, +) from denser.curve import DensityCurve, DensityPoint, curve from denser.eval import ( CaseResult, @@ -96,9 +106,15 @@ "CaseResult", "ComparisonReport", "CompressionResult", + "CONTEXT_BUNDLE_SCHEMA_VERSION", + "CONTEXT_SELECTION_SCHEMA_VERSION", + "ComponentAttempt", "ContractCategory", "ContractItem", "ContractItemResult", + "ContextBundle", + "ContextComponent", + "ContextSelectionReport", "ContextAuditReport", "DensityCurve", "DensityPoint", @@ -147,9 +163,11 @@ "evaluate", "inspect", "load_golden_tasks", + "load_context_bundle", "load_replay_suite", "load_replay_tasks", "optimize", + "minimize_context", "replay", "verify", ] diff --git a/denser/audit.py b/denser/audit.py index 9560d7a..acf3af1 100644 --- a/denser/audit.py +++ b/denser/audit.py @@ -213,6 +213,7 @@ def audit_context( n_trials: int = 1, seed: int = 0, on_progress: Callable[[ReplayProgress], None] | None = None, + parallelism: int = 1, ) -> ContextAuditReport: """Audit a context variant and require a sensitive suite for a positive verdict. @@ -232,6 +233,7 @@ def audit_context( n_trials=n_trials, seed=seed, on_progress=on_progress, + parallelism=parallelism, ) comparison = ReplayComparisonReport( task_type=tt, diff --git a/denser/backends/codex_cli.py b/denser/backends/codex_cli.py index 66cb2c4..463d63b 100644 --- a/denser/backends/codex_cli.py +++ b/denser/backends/codex_cli.py @@ -13,6 +13,7 @@ import re import shutil import subprocess +import threading import time from dataclasses import dataclass from pathlib import Path @@ -269,7 +270,7 @@ def __init__( self._disabled_features = _DISABLED_FEATURES_BY_PROFILE[capability_profile] self._cli_version: str | None = None self._cli_version_checked = False - self._last_call_metadata: CodexCliCallMetadata | None = None + self._call_state = threading.local() @property def name(self) -> str: @@ -281,6 +282,11 @@ def supports_caching(self) -> bool: """Report that this adapter does not expose explicit prompt caching.""" return False + @property + def supports_concurrency(self) -> bool: + """Report that per-call metadata is isolated for concurrent replays.""" + return True + @property def runtime_config(self) -> dict[str, object]: """Return the reproducibility settings safe to include in reports.""" @@ -306,9 +312,13 @@ def runtime_config(self) -> dict[str, object]: @property def last_call_metadata(self) -> dict[str, object] | None: """Return sanitized evidence for the most recent invocation.""" - if self._last_call_metadata is None: + metadata = getattr(self._call_state, "last_call_metadata", None) + if not isinstance(metadata, CodexCliCallMetadata): return None - return self._last_call_metadata.to_dict() + return metadata.to_dict() + + def _set_last_call_metadata(self, metadata: CodexCliCallMetadata | None) -> None: + self._call_state.last_call_metadata = metadata def _get_cli_version(self) -> str | None: if self._cli_version_checked: @@ -392,20 +402,24 @@ def _run(self, command: list[str], user: str) -> subprocess.CompletedProcess[str ) except subprocess.TimeoutExpired as exc: duration_ms = round((time.monotonic() - started) * 1000) - self._last_call_metadata = CodexCliCallMetadata( - status="timeout", - exit_code=None, - duration_ms=duration_ms, + self._set_last_call_metadata( + CodexCliCallMetadata( + status="timeout", + exit_code=None, + duration_ms=duration_ms, + ) ) raise BackendError( f"Codex CLI timed out after {self._timeout_seconds:g} seconds" ) from exc except OSError as exc: duration_ms = round((time.monotonic() - started) * 1000) - self._last_call_metadata = CodexCliCallMetadata( - status="launch_error", - exit_code=None, - duration_ms=duration_ms, + self._set_last_call_metadata( + CodexCliCallMetadata( + status="launch_error", + exit_code=None, + duration_ms=duration_ms, + ) ) raise BackendError(f"Codex CLI could not be launched: {type(exc).__name__}") from exc @@ -430,7 +444,7 @@ def complete( command = self._build_command(system) started = time.monotonic() - self._last_call_metadata = None + self._set_last_call_metadata(None) completed = self._run(command, user) duration_ms = round((time.monotonic() - started) * 1000) parsed = _parse_events(completed.stdout, completed.stderr) @@ -441,12 +455,14 @@ def complete( and parsed.final_message is not None else "failed" ) - self._last_call_metadata = CodexCliCallMetadata( - status=status, - exit_code=completed.returncode, - duration_ms=duration_ms, - usage=parsed.usage, - transport_fallback=parsed.transport_fallback, + self._set_last_call_metadata( + CodexCliCallMetadata( + status=status, + exit_code=completed.returncode, + duration_ms=duration_ms, + usage=parsed.usage, + transport_fallback=parsed.transport_fallback, + ) ) if status != "completed": raise BackendError( diff --git a/denser/cli.py b/denser/cli.py index 9f308fc..dc12a83 100644 --- a/denser/cli.py +++ b/denser/cli.py @@ -4,6 +4,7 @@ Commands: - `denser audit` — audit behavior parity and replay-suite sensitivity +- `denser minimize-context` — safely remove unnecessary bundle components - `denser inspect` — build an offline preservation contract - `denser verify` — verify a candidate against its source contract - `denser optimize` — generate and select verified candidates @@ -39,6 +40,12 @@ SiliconFlowBackend, ) from denser.compress import compress +from denser.context_selection import ( + ContextBundle, + ContextSelectionReport, + load_context_bundle, +) +from denser.context_selection import minimize_context as minimize_context_fn from denser.curve import curve as curve_fn from denser.eval import ComparisonReport, EvalReport from denser.eval import compare as compare_fn @@ -1198,6 +1205,221 @@ def _print_replay_comparison( ) +def _validate_context_selection_outputs( + bundle: ContextBundle, + suite_file: Path, + out: Path, + json_out: Path, +) -> None: + """Reject output paths that could overwrite an input or existing file.""" + _validate_new_output_paths(bundle.manifest_path, out, json_out) + protected = {suite_file.resolve()} + protected.update(component.path.resolve() for component in bundle.components) + collision = next( + (path for path in (out, json_out) if path.resolve() in protected), + None, + ) + if collision is not None: + raise click.ClickException(f"Refusing to overwrite a context-selection input: {collision}") + + +def _print_context_selection(report: ContextSelectionReport) -> None: + """Print a compact component decision and final evidence summary.""" + table = Table(title="[bold]context component selection[/bold]") + table.add_column("Component") + table.add_column("Estimated tokens", justify="right") + table.add_column("Decision") + for attempt in report.attempts: + outcome = "removed" if attempt.removed else f"kept ({attempt.decision.value})" + table.add_row(attempt.component_id, str(attempt.component_estimated_tokens), outcome) + for component_id in report.required_ids: + table.add_row(component_id, "-", "required") + console.print(table) + observed = report.observed_input_reduction_pct + observed_label = "unavailable" if observed is None else f"{observed:.2%}" + console.print( + Panel.fit( + f"Target met: [bold]{str(report.target_met).lower()}[/bold]\n" + f"Full-input reduction: {observed_label}\n" + f"Final behavior: {report.final_audit.decision.value}\n" + f"Reason: {escape(report.outcome_reason)}", + title="Final validation", + ) + ) + + +@main.command("minimize-context") +@click.argument("manifest_file", type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option( + "--suite", + "suite_file", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, + help="Development replay suite used for automatic component selection.", +) +@click.option( + "--type", + "task_type_override", + type=click.Choice([t.value for t in TaskType], case_sensitive=False), + default=None, + help="Optional check that the manifest has the expected task_type.", +) +@click.option( + "--backend", + type=click.Choice(REPLAY_BACKEND_CHOICES, case_sensitive=False), + default="codex-cli", + show_default=True, +) +@click.option("--base-url", default=None, help="Base URL for openai-compat backend.") +@click.option("--model", default=None, help="Execution model id; defaults depend on backend.") +@click.option( + "--openai-thinking-mode", + type=click.Choice(["provider-default", "enabled", "disabled"]), + default="provider-default", + show_default=True, +) +@click.option( + "--codex-cli-path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + default=None, + envvar="DENSER_CODEX_CLI", + help="Independent Codex CLI executable; never use the desktop WindowsApps binary.", +) +@click.option( + "--codex-timeout", + type=click.FloatRange(min=1.0), + default=180.0, + show_default=True, +) +@click.option( + "--codex-reasoning-effort", + type=click.Choice(["none", "low", "medium", "high", "xhigh", "max"]), + default="medium", + show_default=True, +) +@click.option( + "--codex-respect-system-proxy/--no-codex-respect-system-proxy", + default=False, +) +@click.option( + "--codex-capability-profile", + type=click.Choice(CODEX_CAPABILITY_PROFILES), + default="standard", + show_default=True, + help="Use standard for file- and tool-using workloads.", +) +@click.option( + "--selection-trials", + type=click.IntRange(min=1), + default=1, + show_default=True, +) +@click.option( + "--validation-trials", + type=click.IntRange(min=1), + default=3, + show_default=True, +) +@click.option( + "--parallelism", + type=click.IntRange(min=1, max=16), + default=1, + show_default=True, + help="Concurrent replay calls; supported by codex-cli.", +) +@click.option("--seed", type=int, default=0, show_default=True) +@click.option( + "--min-input-reduction", + type=click.FloatRange(min=0.0, max=1.0), + default=0.10, + show_default=True, + help="Required provider-reported full-input reduction as a fraction.", +) +@click.option("--progress/--no-progress", default=True, show_default=True) +@click.option( + "--out", + type=click.Path(dir_okay=False, path_type=Path), + required=True, + help="Write the selected rendered context to a new file.", +) +@click.option( + "--json-out", + type=click.Path(dir_okay=False, path_type=Path), + required=True, + help="Write the versioned selection evidence to a new JSON file.", +) +def minimize_context_cmd( + manifest_file: Path, + suite_file: Path, + task_type_override: str | None, + backend: str, + base_url: str | None, + model: str | None, + openai_thinking_mode: str, + codex_cli_path: Path | None, + codex_timeout: float, + codex_reasoning_effort: str, + codex_respect_system_proxy: bool, + codex_capability_profile: str, + selection_trials: int, + validation_trials: int, + parallelism: int, + seed: int, + min_input_reduction: float, + progress: bool, + out: Path, + json_out: Path, +) -> None: + """Select a behavior-preserving subset of MANIFEST_FILE.""" + try: + bundle = load_context_bundle(manifest_file) + if ( + task_type_override is not None + and TaskType.parse(task_type_override) != bundle.task_type + ): + raise ValueError("--type does not match the context bundle manifest task_type") + _validate_context_selection_outputs(bundle, suite_file, out, json_out) + suite = load_replay_suite(suite_file) + backend_obj = _build_backend( + backend, + model=model, + base_url=base_url, + openai_thinking_mode=openai_thinking_mode, + codex_cli_path=codex_cli_path, + codex_timeout=codex_timeout, + codex_reasoning_effort=codex_reasoning_effort, + codex_respect_system_proxy=codex_respect_system_proxy, + codex_capability_profile=codex_capability_profile, + ) + selected_text, report = minimize_context_fn( + bundle=bundle, + tasks=suite, + backend=backend_obj, + selection_trials=selection_trials, + validation_trials=validation_trials, + seed=seed, + min_input_reduction=min_input_reduction, + on_progress=_print_replay_progress if progress else None, + parallelism=parallelism, + ) + except (BackendError, UnicodeError, ValueError) as exc: + raise click.ClickException(str(exc)) from exc + + json_out.write_text( + json.dumps(report.to_dict(), indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + _print_context_selection(report) + if report.final_audit.decision == AuditDecision.PRESERVED: + out.write_text(selected_text, encoding="utf-8") + console.print(f"Wrote selected context -> {out}") + else: + console.print("Did not write selected context because final behavior was not preserved.") + console.print(f"Wrote selection evidence -> {json_out}") + if not report.target_met: + raise click.exceptions.Exit(3) + + @main.command("curve") @click.argument("input_file", type=click.Path(exists=True, dir_okay=False, path_type=Path)) @click.option( diff --git a/denser/context_selection.py b/denser/context_selection.py new file mode 100644 index 0000000..a999a05 --- /dev/null +++ b/denser/context_selection.py @@ -0,0 +1,445 @@ +"""Conservative component selection for visible LLM context bundles. + +The module exposes one narrow workflow: load a versioned bundle manifest, +remove optional components one at a time, and keep a removal only when the +existing behavior audit certifies parity and catches a known-bad control. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from denser.audit import AuditDecision, ContextAuditReport, audit_context +from denser.backends import Backend +from denser.replay import ReplayProgress, ReplaySuite, ReplaySuiteRole +from denser.taxonomy import TaskType +from denser.tokens import estimate_tokens + +CONTEXT_BUNDLE_SCHEMA_VERSION = "denser.context-bundle/v1" +CONTEXT_SELECTION_SCHEMA_VERSION = "denser.context-selection/v1" +SELECTION_METHOD = "greedy-largest-first/v1" +_COMPONENT_ID_RE = re.compile(r"[a-z0-9][a-z0-9._-]*") + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class ContextComponent: + """One named, reviewable text component in a context bundle.""" + + component_id: str + kind: TaskType + path: Path + text: str + required: bool = False + + @property + def estimated_tokens(self) -> int: + """Return the explicit offline token estimate for this component.""" + return estimate_tokens(self.text) + + +@dataclass(frozen=True) +class ContextBundle: + """Validated component bundle loaded from a versioned manifest.""" + + name: str + task_type: TaskType + components: tuple[ContextComponent, ...] + negative_control_drop: tuple[str, ...] + manifest_path: Path + schema_version: str = CONTEXT_BUNDLE_SCHEMA_VERSION + + def __post_init__(self) -> None: + if not self.name.strip(): + raise ValueError("Context bundle name cannot be empty") + if len(self.components) < 2: + raise ValueError("Context bundle must contain at least two components") + component_ids = [component.component_id for component in self.components] + if len(component_ids) != len(set(component_ids)): + raise ValueError("Context bundle component ids must be unique") + if not any(not component.required for component in self.components): + raise ValueError("Context bundle must contain at least one optional component") + if not self.negative_control_drop: + raise ValueError("Context bundle must define negative_control_drop") + if len(self.negative_control_drop) != len(set(self.negative_control_drop)): + raise ValueError("negative_control_drop component ids must be unique") + unknown = set(self.negative_control_drop) - set(component_ids) + if unknown: + raise ValueError( + "negative_control_drop contains unknown component ids: " + + ", ".join(sorted(unknown)) + ) + required_ids = { + component.component_id for component in self.components if component.required + } + unprotected = set(self.negative_control_drop) - required_ids + if unprotected: + raise ValueError( + "negative_control_drop components must be required: " + + ", ".join(sorted(unprotected)) + ) + if len(self.negative_control_drop) == len(self.components): + raise ValueError("negative_control_drop cannot remove every component") + + @property + def component_ids(self) -> tuple[str, ...]: + """Return component ids in manifest order.""" + return tuple(component.component_id for component in self.components) + + @property + def required_ids(self) -> tuple[str, ...]: + """Return required component ids in manifest order.""" + return tuple(component.component_id for component in self.components if component.required) + + @property + def optional_components(self) -> tuple[ContextComponent, ...]: + """Return removable components, largest estimated size first.""" + return tuple( + sorted( + (component for component in self.components if not component.required), + key=lambda component: (-component.estimated_tokens, component.component_id), + ) + ) + + def render(self, included_ids: tuple[str, ...] | list[str] | set[str]) -> str: + """Render selected components in stable manifest order.""" + selected = set(included_ids) + unknown = selected - set(self.component_ids) + if unknown: + raise ValueError("Cannot render unknown component ids: " + ", ".join(sorted(unknown))) + sections = [] + for component in self.components: + if component.component_id not in selected: + continue + sections.append( + f"## Context component: {component.component_id} ({component.kind.value})\n\n" + f"{component.text.rstrip()}" + ) + if not sections: + raise ValueError("Cannot render an empty context bundle") + return "\n\n".join(sections) + "\n" + + @property + def baseline_text(self) -> str: + """Return the complete rendered context bundle.""" + return self.render(self.component_ids) + + @property + def negative_control_text(self) -> str: + """Return the declared known-bad sensitivity control.""" + dropped = set(self.negative_control_drop) + return self.render([item for item in self.component_ids if item not in dropped]) + + +@dataclass(frozen=True) +class ComponentAttempt: + """Summary evidence for one attempted component removal.""" + + component_id: str + component_estimated_tokens: int + removed: bool + decision: AuditDecision + decision_reason: str + variant_regressions: tuple[str, ...] + variant_improvements: tuple[str, ...] + observed_input_reduction_pct: float | None + + def to_dict(self) -> dict[str, object]: + """Return a JSON-compatible attempt summary.""" + return { + "component_id": self.component_id, + "component_estimated_tokens": self.component_estimated_tokens, + "removed": self.removed, + "decision": self.decision.value, + "decision_reason": self.decision_reason, + "variant_regressions": list(self.variant_regressions), + "variant_improvements": list(self.variant_improvements), + "observed_input_reduction_pct": self.observed_input_reduction_pct, + } + + +@dataclass(frozen=True) +class ContextSelectionReport: + """Selection decisions plus a separately repeated final audit.""" + + bundle_name: str + task_type: TaskType + baseline_sha256: str + selected_sha256: str + baseline_estimated_tokens: int + selected_estimated_tokens: int + selected_ids: tuple[str, ...] + removed_ids: tuple[str, ...] + required_ids: tuple[str, ...] + negative_control_drop: tuple[str, ...] + attempts: tuple[ComponentAttempt, ...] + final_audit: ContextAuditReport + selection_trials: int + validation_trials: int + parallelism: int + min_input_reduction: float + schema_version: str = CONTEXT_SELECTION_SCHEMA_VERSION + selection_method: str = SELECTION_METHOD + + @property + def observed_input_reduction_pct(self) -> float | None: + """Return provider-reported end-to-end input reduction.""" + return self.final_audit.observed_input_reduction_pct + + @property + def target_met(self) -> bool: + """Return whether final parity and the requested saving are both proven.""" + observed = self.observed_input_reduction_pct + return ( + self.final_audit.decision == AuditDecision.PRESERVED + and observed is not None + and observed >= self.min_input_reduction + ) + + @property + def outcome_reason(self) -> str: + """Explain why the target passed or failed without overstating evidence.""" + if self.final_audit.decision != AuditDecision.PRESERVED: + return f"Final behavior audit was {self.final_audit.decision.value}." + observed = self.observed_input_reduction_pct + if observed is None: + return "The backend did not report comparable full-input token usage." + if observed < self.min_input_reduction: + return ( + f"Observed full-input reduction was {observed:.2%}, below the " + f"{self.min_input_reduction:.2%} target." + ) + return f"Behavior was preserved and observed full-input reduction was {observed:.2%}." + + def to_dict(self) -> dict[str, object]: + """Return a versioned, JSON-compatible evidence report.""" + return { + "schema_version": self.schema_version, + "selection_method": self.selection_method, + "bundle_name": self.bundle_name, + "task_type": self.task_type.value, + "target_met": self.target_met, + "outcome_reason": self.outcome_reason, + "min_input_reduction": self.min_input_reduction, + "selection_trials": self.selection_trials, + "validation_trials": self.validation_trials, + "parallelism": self.parallelism, + "baseline_sha256": self.baseline_sha256, + "selected_sha256": self.selected_sha256, + "components": { + "selected": list(self.selected_ids), + "removed": list(self.removed_ids), + "required": list(self.required_ids), + "negative_control_drop": list(self.negative_control_drop), + }, + "measurements": { + "baseline_estimated_tokens": self.baseline_estimated_tokens, + "selected_estimated_tokens": self.selected_estimated_tokens, + "estimated_token_reduction": ( + self.baseline_estimated_tokens - self.selected_estimated_tokens + ), + "estimated_token_reduction_pct": ( + 0.0 + if self.baseline_estimated_tokens == 0 + else (self.baseline_estimated_tokens - self.selected_estimated_tokens) + / self.baseline_estimated_tokens + ), + "observed_input_reduction_pct": self.observed_input_reduction_pct, + }, + "attempts": [attempt.to_dict() for attempt in self.attempts], + "final_audit": self.final_audit.to_dict(), + } + + +def _manifest_object(path: Path) -> dict[str, Any]: + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid context bundle JSON: {exc.msg}") from exc + if not isinstance(data, dict): + raise ValueError("Context bundle manifest must be a JSON object") + return data + + +def load_context_bundle(path: str | Path) -> ContextBundle: + """Load and strictly validate a context bundle manifest and its text files.""" + manifest_path = Path(path).expanduser().resolve() + data = _manifest_object(manifest_path) + if data.get("schema_version") != CONTEXT_BUNDLE_SCHEMA_VERSION: + raise ValueError(f"Context bundle schema_version must be {CONTEXT_BUNDLE_SCHEMA_VERSION!r}") + name = data.get("name") + task_type = data.get("task_type") + raw_components = data.get("components") + raw_control = data.get("negative_control_drop") + if not isinstance(name, str) or not isinstance(task_type, str): + raise ValueError("Context bundle name and task_type must be strings") + if not isinstance(raw_components, list) or not all( + isinstance(item, dict) for item in raw_components + ): + raise ValueError("Context bundle components must be a list of objects") + if not isinstance(raw_control, list) or not all(isinstance(item, str) for item in raw_control): + raise ValueError("negative_control_drop must be a list of component ids") + + root = manifest_path.parent.resolve() + components: list[ContextComponent] = [] + seen_paths: set[Path] = set() + for item in raw_components: + component_id = item.get("id") + kind = item.get("kind") + raw_path = item.get("path") + required = item.get("required", False) + if not isinstance(component_id, str) or _COMPONENT_ID_RE.fullmatch(component_id) is None: + raise ValueError( + "Component id must use lowercase letters, numbers, dots, underscores, or hyphens" + ) + if not isinstance(kind, str) or not isinstance(raw_path, str): + raise ValueError(f"Component {component_id!r} kind and path must be strings") + if not isinstance(required, bool): + raise ValueError(f"Component {component_id!r} required must be a boolean") + component_path = (root / raw_path).resolve() + if not component_path.is_relative_to(root): + raise ValueError( + f"Component {component_id!r} path must stay inside the bundle directory" + ) + if component_path in seen_paths: + raise ValueError(f"Context bundle references the same file twice: {raw_path}") + if not component_path.is_file(): + raise ValueError(f"Component file does not exist: {raw_path}") + text = component_path.read_text(encoding="utf-8") + if not text.strip(): + raise ValueError(f"Component file is empty: {raw_path}") + seen_paths.add(component_path) + components.append( + ContextComponent( + component_id=component_id, + kind=TaskType.parse(kind), + path=component_path, + text=text, + required=required, + ) + ) + return ContextBundle( + name=name, + task_type=TaskType.parse(task_type), + components=tuple(components), + negative_control_drop=tuple(raw_control), + manifest_path=manifest_path, + ) + + +def minimize_context( + *, + bundle: ContextBundle, + tasks: ReplaySuite, + backend: Backend, + selection_trials: int = 1, + validation_trials: int = 3, + seed: int = 0, + min_input_reduction: float = 0.10, + on_progress: Callable[[ReplayProgress], None] | None = None, + parallelism: int = 1, +) -> tuple[str, ContextSelectionReport]: + """Greedily remove optional components and fail closed on uncertain behavior.""" + if tasks.role != ReplaySuiteRole.DEVELOPMENT: + raise ValueError( + "Automatic context selection requires a development replay suite; " + "freeze and audit the selected context separately for holdout evidence" + ) + if selection_trials < 1 or validation_trials < 1: + raise ValueError("selection_trials and validation_trials must be at least 1") + if parallelism < 1: + raise ValueError("parallelism must be at least 1") + if not 0.0 <= min_input_reduction <= 1.0: + raise ValueError("min_input_reduction must be between 0 and 1") + + baseline = bundle.baseline_text + negative_control = bundle.negative_control_text + selected = list(bundle.component_ids) + removed: list[str] = [] + attempts: list[ComponentAttempt] = [] + + for index, component in enumerate(bundle.optional_components): + candidate_ids = [item for item in selected if item != component.component_id] + candidate = bundle.render(candidate_ids) + audit = audit_context( + baseline=baseline, + variant=candidate, + negative_control=negative_control, + task_type=bundle.task_type, + tasks=tasks, + backend=backend, + n_trials=selection_trials, + seed=seed + index, + on_progress=on_progress, + parallelism=parallelism, + ) + accepted = audit.decision == AuditDecision.PRESERVED + if accepted: + selected = candidate_ids + removed.append(component.component_id) + attempts.append( + ComponentAttempt( + component_id=component.component_id, + component_estimated_tokens=component.estimated_tokens, + removed=accepted, + decision=audit.decision, + decision_reason=audit.decision_reason, + variant_regressions=audit.variant_regressions, + variant_improvements=audit.variant_improvements, + observed_input_reduction_pct=audit.observed_input_reduction_pct, + ) + ) + + selected_text = bundle.render(selected) + final_audit = audit_context( + baseline=baseline, + variant=selected_text, + negative_control=negative_control, + task_type=bundle.task_type, + tasks=tasks, + backend=backend, + n_trials=validation_trials, + seed=seed + len(bundle.optional_components), + on_progress=on_progress, + parallelism=parallelism, + ) + report = ContextSelectionReport( + bundle_name=bundle.name, + task_type=bundle.task_type, + baseline_sha256=_sha256(baseline), + selected_sha256=_sha256(selected_text), + baseline_estimated_tokens=estimate_tokens(baseline), + selected_estimated_tokens=estimate_tokens(selected_text), + selected_ids=tuple(selected), + removed_ids=tuple(removed), + required_ids=bundle.required_ids, + negative_control_drop=bundle.negative_control_drop, + attempts=tuple(attempts), + final_audit=final_audit, + selection_trials=selection_trials, + validation_trials=validation_trials, + parallelism=parallelism, + min_input_reduction=min_input_reduction, + ) + return selected_text, report + + +__all__ = [ + "CONTEXT_BUNDLE_SCHEMA_VERSION", + "CONTEXT_SELECTION_SCHEMA_VERSION", + "ComponentAttempt", + "ContextBundle", + "ContextComponent", + "ContextSelectionReport", + "load_context_bundle", + "minimize_context", +] diff --git a/denser/replay.py b/denser/replay.py index 2d95059..e828ce5 100644 --- a/denser/replay.py +++ b/denser/replay.py @@ -18,6 +18,7 @@ import random import re from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from datetime import datetime, timezone from enum import Enum @@ -615,6 +616,13 @@ class _RunUnit: trial_index: int +@dataclass(frozen=True) +class _RunObservation: + output: str + error_type: str | None + metadata: dict[str, object] + + @dataclass(frozen=True) class ReplayProgress: """One completed backend call in a replay schedule.""" @@ -764,7 +772,12 @@ def _execute_schedule( suite_sha256: str, suite_metadata: dict[str, object], on_progress: Callable[[ReplayProgress], None] | None = None, + parallelism: int = 1, ) -> dict[str, ReplayReport]: + if parallelism < 1: + raise ValueError("parallelism must be at least 1") + if parallelism > 1 and getattr(backend, "supports_concurrency", False) is not True: + raise ValueError("The selected backend does not support concurrent replay calls") generated_at_utc = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") runtime_config = _backend_runtime_config(backend) accumulators = { @@ -774,10 +787,9 @@ def _execute_schedule( for case_index, _case in enumerate(task.cases) } - for completed_calls, unit in enumerate(schedule, start=1): + def run_unit(unit: _RunUnit) -> _RunObservation: task = tasks[unit.task_index] case = task.cases[unit.case_index] - observed = accumulators[(unit.side, unit.task_index, unit.case_index)] try: output = backend.complete( system=texts[unit.side], @@ -792,14 +804,12 @@ def _execute_schedule( case.name, error_type, ) - observed.outputs.append("") - observed.errors.append(error_type) - observed.backend_metadata.append(_backend_metadata(backend)) - else: - observed.outputs.append(output) - observed.backend_metadata.append(_backend_metadata(backend)) - if case.matches(output): - observed.n_passed += 1 + return _RunObservation("", error_type, _backend_metadata(backend)) + return _RunObservation(output, None, _backend_metadata(backend)) + + def progress(completed_calls: int, unit: _RunUnit) -> None: + task = tasks[unit.task_index] + case = task.cases[unit.case_index] if on_progress is not None: on_progress( ReplayProgress( @@ -813,6 +823,34 @@ def _execute_schedule( ) ) + indexed_results: dict[int, _RunObservation] = {} + if parallelism == 1: + for index, unit in enumerate(schedule): + indexed_results[index] = run_unit(unit) + progress(index + 1, unit) + else: + with ThreadPoolExecutor(max_workers=parallelism) as executor: + futures: dict[Future[_RunObservation], tuple[int, _RunUnit]] = { + executor.submit(run_unit, unit): (index, unit) + for index, unit in enumerate(schedule) + } + for completed_calls, future in enumerate(as_completed(futures), start=1): + index, unit = futures[future] + indexed_results[index] = future.result() + progress(completed_calls, unit) + + for index, unit in enumerate(schedule): + task = tasks[unit.task_index] + case = task.cases[unit.case_index] + observed = accumulators[(unit.side, unit.task_index, unit.case_index)] + result = indexed_results[index] + observed.outputs.append(result.output) + observed.backend_metadata.append(result.metadata) + if result.error_type is not None: + observed.errors.append(result.error_type) + elif case.matches(result.output): + observed.n_passed += 1 + return { side: _build_report( side=side, @@ -900,6 +938,7 @@ def _replay_comparison_sides( n_trials: int = 1, seed: int = 0, on_progress: Callable[[ReplayProgress], None] | None = None, + parallelism: int = 1, ) -> tuple[TaskType, dict[str, ReplayReport]]: """Replay comparison sides in one randomized schedule.""" if not original or not original.strip(): @@ -942,6 +981,7 @@ def _replay_comparison_sides( suite_sha256=suite_sha256, suite_metadata=suite_metadata, on_progress=on_progress, + parallelism=parallelism, ) return tt, reports diff --git a/docs/CODEX_CONTEXT_SELECTION_CASE_STUDY.md b/docs/CODEX_CONTEXT_SELECTION_CASE_STUDY.md new file mode 100644 index 0000000..7137fad --- /dev/null +++ b/docs/CODEX_CONTEXT_SELECTION_CASE_STUDY.md @@ -0,0 +1,101 @@ +# Codex tool-workflow context selection + +Date: 2026-08-18 + +## Question + +Can denser remove irrelevant visible context while Codex keeps its normal local +file tools and preserves behavior on tasks that actually require those tools? + +## Workload + +The synthetic bundle has four reviewable components: + +1. a required tool and output contract; +2. an optional release policy containing the arbitrary `R7` mapping; +3. an optional CI policy containing the arbitrary `C4` mapping; +4. an optional 17,236-byte archived product handbook unrelated to either task. + +One task reads a local release JSON record; the other reads a local CI JSON +record. Each file contains a nonce-like identifier that is absent from the +prompt. Exact answers therefore require local file access. The known-bad control +removes the required output contract, and the suite must detect that change. + +All redistributable inputs are under +[`examples/context_bundles/tool_workflows/`](../examples/context_bundles/tool_workflows/). + +## Automatic decision + +`denser minimize-context` tested optional components largest first: + +| Component | Decision | Reason | +|---|---|---| +| `archived-handbook` | removed | both covered tasks preserved | +| `release-policy` | kept | release task regressed without it | +| `ci-policy` | kept | CI task regressed without it | +| `execution-contract` | required | also used to construct the sensitivity control | + +This is greedy component ablation, not a claim of a globally smallest prompt. +Any regression, changed improvement, operational error, or insensitive +negative control causes the tested component to remain. + +## Final evidence + +Runtime: + +- Codex CLI 0.147.0; +- `gpt-5.6-sol`; +- medium reasoning; +- ephemeral read-only sandbox; +- `standard` capability profile; +- three final trials per task and side; +- randomized side order with seed 3; +- six concurrent independent CLI calls. + +The standard benchmark profile left shell access, plugins, and skill search +available. It disabled apps, memories, and multi-agent execution identically +for the complete and selected bundles. + +| Result | Complete bundle | Selected bundle | +|---|---:|---:| +| Release decision | 3/3 | 3/3 | +| CI decision | 3/3 | 3/3 | +| Operational errors | 0 | 0 | +| Provider input tokens | 277,871 | 243,210 | +| Mean input per call | 46,311.83 | 40,535.00 | + +Observed complete-input reduction: **12.47%**. + +The negative control failed all six trials, so parity was not accepted against +an insensitive suite. The committed JSON contains hashes, runtime settings, +per-call final outputs, per-call token usage, selection decisions, and the final +audit. + +## Reproduce + +From a source checkout with the independent Codex CLI signed in: + +```bash +denser minimize-context examples/context_bundles/tool_workflows/bundle.json \ + --suite examples/context_bundles/tool_workflows/replay.json \ + --backend codex-cli --model gpt-5.6-sol \ + --codex-reasoning-effort medium \ + --codex-capability-profile standard \ + --codex-timeout 300 --selection-trials 1 --validation-trials 3 \ + --parallelism 6 --min-input-reduction 0.10 \ + --out selected-context.md --json-out selection-evidence.json +``` + +Output paths must be new. `--parallelism` only reduces wall-clock time; every +call remains an independent ephemeral Codex turn and is recorded separately. + +## Limits + +- This is one synthetic bundle with two deterministic local-file tasks. +- The selector sees the development suite. A separate frozen holdout is still + required for stronger research claims. +- The measured saving applies to the exact committed workload and runtime, not + to all Codex sessions. +- denser selects among supplied visible text components. It does not inspect + hidden provider context, rewrite Codex's own compactor, or optimize KV cache. +- Greedy removal can miss a smaller combination when components interact. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 61201c5..a8720d5 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -59,6 +59,16 @@ estimates, and provider-reported end-to-end input totals. A variant improvement is sent to review rather than silently classified as preservation; operational errors and undetected controls fail closed as inconclusive. +`denser minimize-context` now implements the first selective-loading path. A +versioned manifest names visible text components, required components, and a +known-bad control. Optional components are tested largest first; a removal is +accepted only after a sensitive behavior audit, and the selected combination +receives a repeated final audit. The first normal-tool Codex pilot removed an +irrelevant 17,236-byte archive, retained two workload-critical policies, kept +both local-file tasks at 3/3, and reduced provider-reported complete input by +12.47%. This is greedy development-suite selection, not a global minimum or a +blind holdout result. + The Codex CLI adapter also exposes a narrowly scoped `text-only` capability profile. It is for pre-bundled text decisions only and removes unused tool and extension context. In a seeded randomized audit with three trials per case, @@ -215,6 +225,8 @@ Prompt-cache savings and active-context length are reported separately. ### Phase 4: selective loading and compaction fidelity +- select among caller-supplied visible context components with fail-closed + behavior ablation and a repeated final audit (implemented); - capture reproducible before/after textual context snapshots from real agent runtimes without duplicating their compaction implementation; - test permissions, user decisions, unfinished work, and failure recovery after diff --git a/examples/context_bundles/tool_workflows/README.md b/examples/context_bundles/tool_workflows/README.md new file mode 100644 index 0000000..220289a --- /dev/null +++ b/examples/context_bundles/tool_workflows/README.md @@ -0,0 +1,15 @@ +# Tool-workflow context bundle + +This redistributable synthetic example demonstrates automatic component +selection without switching Codex to text-only mode. + +- `bundle.json` lists four visible context components. +- `fixtures/` contains two local JSON records that the model must read. +- `replay.json` defines the two exact-output tasks. +- `selected.codex-standard.2026-08-18.md` is the selected bundle. +- `selection.codex-standard.3x.2026-08-18.json` is the complete final evidence. + +The selector removed `archived-handbook.md`, retained both policy components, +and observed a 12.47% full-input reduction with final behavior preserved. See +[`docs/CODEX_CONTEXT_SELECTION_CASE_STUDY.md`](../../../docs/CODEX_CONTEXT_SELECTION_CASE_STUDY.md) +for the reproduction command and limitations. diff --git a/examples/context_bundles/tool_workflows/archived-handbook.md b/examples/context_bundles/tool_workflows/archived-handbook.md new file mode 100644 index 0000000..e70ce28 --- /dev/null +++ b/examples/context_bundles/tool_workflows/archived-handbook.md @@ -0,0 +1,113 @@ +# Archived product handbook + +This synthetic handbook represents stale but plausible project background that is often loaded into an agent session even when a request does not use it. None of the rules below governs release records or continuous-integration records. The material is intentionally substantial enough to make the cost of irrelevant repeated context measurable, while remaining ordinary human-readable prose rather than random padding. + +## Brand history + +The retired product name was Northstar Desk. Early prototypes focused on shared editorial calendars for small research groups. The first prototype used a blue and amber visual system, a weekly digest, and a workspace metaphor built around rooms, shelves, and cards. Later research showed that most teams preferred direct lists over the room metaphor, so the vocabulary was removed from the active interface. Historical screenshots may still contain the names Atrium, Shelf, Dispatch, and Lantern. Those names have no operational meaning in current systems and must not be interpreted as commands, environments, access levels, or deployment stages. + +The original audience was a mixed group of editors, analysts, and program coordinators. Interviews emphasized legibility, predictable navigation, and simple exports. A second research round explored public publishing, but that work never reached general availability. Documents that describe publication channels, subscriber segments, or audience tiers are archival research notes. They are not a source of truth for current permissions, retention, billing, security, or release behavior. + +## Retired visual language + +The old interface used twelve spacing units ranging from two to sixty-four pixels. Cards had an eight-pixel radius, menus used a six-pixel radius, and modal dialogs used a twelve-pixel radius. Primary buttons were navy, secondary buttons were gray, and warning buttons were amber. Destructive buttons were red only after the user opened a confirmation panel. These details belong to an abandoned design system and should not influence terminal output, machine-readable answers, repository automation, policy decisions, or file inspection. + +Typography paired a serif display face with a neutral sans-serif body face. Headings used sentence case. Labels avoided abbreviations unless the abbreviation was already visible in imported data. Tables aligned numbers to the right, names to the left, and dates according to locale. Empty states used one short sentence and a single suggested action. Tooltips were reserved for unfamiliar icons. This guidance was written for a browser prototype and does not define how an agent should format responses. + +The palette included Harbor, Slate, Fog, Paper, Moss, Wheat, Ember, and Signal. Harbor was intended for persistent navigation, while Signal was reserved for transient status. Accessibility tests targeted contrast ratios suitable for ordinary body text and larger display text. The prototype also included a high-contrast theme and a reduced-motion preference. All color tokens were retired when the interface moved to a different component library. + +## Abandoned publishing workflow + +The proposed publishing workflow had draft, edited, scheduled, published, and archived states. Authors could request an editorial review, editors could return a draft with comments, and program coordinators could place approved items onto a calendar. Scheduled items were grouped by local date and channel. The research document assumed that publication would always be reversible for fifteen minutes, but no implementation was completed and no service-level promise was made. + +Channel experiments included email digests, static web pages, printable packets, and internal displays. Each channel had a hypothetical template and a different preview. Email previews emphasized subject length and plain-text fallback. Static page previews checked title hierarchy and image captions. Printable packets checked page breaks and grayscale contrast. Internal displays checked aspect ratio and dwell time. These experiments are unrelated to software release authorization or build verification. + +The editorial calendar prototype allowed drag-and-drop ordering, but interviews revealed that accidental reordering was common on touch screens. A later concept used explicit move controls and a daily lock window. Neither behavior shipped. Mentions of locks in this section refer only to a discarded calendar interaction and never to file locks, branch protection, credentials, production access, or concurrency control. + +## Historical data model + +The draft data model contained Workspaces, Collections, Entries, Editions, Channels, and Receipts. A Workspace grouped collaborators. A Collection grouped Entries by topic. An Edition selected entries for a particular audience and date. A Channel represented a rendering destination. A Receipt recorded whether a delivery attempt was accepted by a hypothetical downstream system. No table names, identifiers, or constraints from this model should be assumed to exist in a current database. + +Entries carried a title, summary, body, source note, owner, and optional embargo date. Editions carried a label, local date, time zone, and ordered membership list. Channels carried template settings. Receipts carried timestamps and diagnostics. The model deliberately avoided storing raw credentials. Authentication was outside the prototype scope. These descriptions are conceptual artifacts, not current schemas and not instructions to query or modify a database. + +An import study examined comma-separated files with inconsistent headers. The proposed importer normalized whitespace, preserved the original row number, and reported duplicate identifiers without silently dropping data. Dates required an explicit locale or ISO format. Unknown columns were retained in an auxiliary map for review. This proposal was never connected to the release process, continuous integration, repository checks, or agent execution. + +## Meeting research archive + +One workshop asked participants to sort forty sample tasks into urgent, scheduled, reference, and discard groups. Participants disagreed most often about reference material. Several teams wanted reference notes visible everywhere, while others found persistent notes distracting. The research conclusion was that relevance should depend on the current task and that old notes should remain retrievable without being injected into every interaction. + +A second workshop compared long onboarding manuals with short contextual prompts. New participants valued examples, but experienced participants preferred links that opened only when needed. The team proposed layered help: a short default explanation, a relevant example near the action, and a searchable archive for unusual cases. This archived handbook itself is an example of why background material should be available without consuming every request's context window. + +A third workshop examined terminology drift. Teams accumulated multiple names for the same concept after reorganizations. Researchers recommended a small current glossary plus an explicit alias index for archived documents. They warned against copying every historical definition into the main instruction set, because old definitions could conflict with current ones and make ordinary tasks slower to understand. + +## Localization notes + +The prototype considered English, Simplified Chinese, Traditional Chinese, Japanese, French, and German. Layout tests allowed labels to expand and avoided fixed-width navigation items. Translators requested complete sentences, context about the speaker, and screenshots for ambiguous controls. Dates followed user locale, while stored timestamps remained unambiguous. Numbers used locale-aware separators only in presentation layers. + +Several experiments tested whether product names should be translated. Researchers recommended keeping legal names stable while translating descriptive feature names. Glossaries included preferred verbs for create, copy, move, archive, restore, export, and publish. These wording notes do not apply to machine protocols or benchmark labels, which must follow the active task's explicit output contract. + +Right-to-left layout was discussed but not prototyped. The notes recommended logical rather than physical alignment properties and mirrored directional icons where meaning depended on reading order. Keyboard navigation was expected to follow visual order. Again, this was design research, not an implemented commitment and not a source of operational policy. + +## Analytics proposal + +The analytics draft separated product health, content flow, and audience response. Product health included active workspaces, successful sessions, and latency percentiles. Content flow included drafts created, reviews requested, editions assembled, and exports completed. Audience response included opens and link visits only for channels where measurement was lawful and expected. The draft prohibited presenting an estimated metric as a directly observed count. + +Researchers proposed event names in past tense with stable properties and documented owners. They wanted schema changes reviewed before rollout and dashboards annotated when definitions changed. Sampling had to be disclosed. Test traffic had to be distinguishable from human activity. None of these proposed events were implemented in the benchmark repository, and no analytics service is needed to complete file-reading tasks. + +An experiment compared weekly and monthly reporting. Weekly reports surfaced operational changes quickly but amplified noise in small samples. Monthly reports were steadier but delayed feedback. The recommendation was to use weekly operational checks with rolling averages and monthly interpretation. This recommendation does not define test retry behavior, build status, or required checks. + +## Customer-support concepts + +The support concept divided inquiries into how-to questions, unexpected behavior, access questions, data corrections, and feature requests. Agents were asked to restate the observed problem, record reproduction steps, and avoid promising dates for uncommitted work. Severe incidents would use a separate escalation process. No support system was built as part of the prototype. + +Suggested help articles covered importing a list, arranging an edition, previewing an export, restoring an archived entry, and inviting a collaborator. Each article began with the outcome, listed prerequisites, and ended with a verification step. Troubleshooting sections distinguished missing permissions from malformed data. This generic writing pattern is not an instruction to change local files or answer beyond an active output contract. + +The team discussed a public status page but did not select a vendor. Draft incident labels included investigating, identified, monitoring, and resolved. Those labels describe a hypothetical hosted service. They must not be confused with the benchmark's action labels or interpreted as evidence about the state of any repository, test run, or deployment. + +## Privacy research + +Privacy notes favored collecting the minimum data needed for a stated function, documenting retention, and separating operational records from product analytics. Export and deletion requests would require identity checks and an audit trail. Sensitive values should not appear in ordinary logs. These are general principles, not evidence that a particular system implements them. + +The prototype planned role-based workspace membership and short-lived invitation links. Researchers also considered guest access with narrow scope and visible expiration. No final authorization model was approved. Archived diagrams use owner, editor, contributor, viewer, and guest labels, but current systems may use different roles. Never map these historical roles onto a present permission decision without an active specification. + +Data residency was listed as a future research topic. The notes did not choose regions, subprocessors, encryption products, or contractual terms. Any document claiming that these choices were final is outdated. Legal and security conclusions require current authoritative sources, not this archive. + +## Mobile prototype + +The mobile concept emphasized reading, quick triage, and comment replies. Complex edition assembly remained a desktop task. Offline mode cached a small set of recently opened entries and queued comments until connectivity returned. The prototype displayed a clear offline indicator and never implied that queued work had reached a server. + +Touch targets followed common accessibility guidance. Swipe gestures always had visible button alternatives. Long-press actions were avoided because discoverability was poor. Notifications grouped related updates and respected quiet hours. These interaction notes have no bearing on command execution or local repository behavior. + +Camera import and voice notes were explored only in sketches. Researchers flagged permission clarity, accidental capture, transcription accuracy, and storage cost. No media pipeline was implemented. The current benchmark contains only small text and JSON fixtures. + +## Search experiments + +The search study compared exact phrase matching, prefix matching, stemming, filters, and semantic retrieval. Participants wanted predictable exact matches for identifiers and broader suggestions for topics. The team proposed showing why a result matched and keeping filters visible. No search engine was selected. + +Archived index fields included title, summary, body, source, owner, and collection. Embargoed items were expected to respect access controls before indexing. Result snippets highlighted terms but avoided exposing hidden text. These are unimplemented safeguards in an old proposal, not permission to inspect files outside the path named by a current request. + +Researchers noted that stale background can reduce search quality when it overwhelms current material. They recommended freshness signals, source labels, and task-specific retrieval. This observation motivates keeping the archive as a removable context component rather than a permanent instruction block. + +## Export formats + +The prototype evaluated Markdown, HTML, PDF, plain text, and structured JSON exports. Markdown prioritized readability and stable headings. HTML prioritized self-contained previews. PDF prioritized print layout. Plain text provided a robust fallback. Structured JSON preserved fields for downstream tools. No exporter in the archived prototype is part of the benchmark workflow. + +File names were expected to use safe characters and deterministic dates. Existing files would not be overwritten without an explicit choice. Large exports would be written to a new destination and verified before delivery. These broad safety preferences do not replace any active repository instruction or policy mapping. + +The export study also considered citation bundles and source manifests. Researchers wanted each generated artifact to identify its inputs and transformation time. They did not define a universal citation format. Any current evidence report should use its own documented schema rather than this abandoned proposal. + +## Integration sketches + +Possible integrations included cloud drives, calendars, chat systems, and generic webhooks. The sketches emphasized narrow scopes, visible connection state, and revocation. They did not specify vendors or production credentials. All sample tokens in the design file were fake placeholders. + +Webhook delivery concepts included signed requests, bounded retries, idempotency keys, and a dead-letter view. The team never implemented an endpoint. Retry timing in those sketches is not related to continuous-integration decisions, package installation, or benchmark action labels. + +Calendar integration research focused on displaying editorial deadlines, not controlling events. Chat integration research focused on notifications with links back to the source. Drive integration research focused on importing documents without changing the originals. These sketches are intentionally outside the active tool-workflow test. + +## Administrative archive + +Budget exercises estimated design, engineering, research, hosting, and support effort across three hypothetical stages. The numbers were planning placeholders and were never approved. Hiring plans and launch dates were likewise illustrative. Do not present them as commitments or current organizational facts. + +Risk registers listed adoption, migration quality, accessibility, localization, vendor dependence, and unclear ownership. Mitigations emphasized small pilots and reversible decisions. No risk entry grants permission to deploy, bypass a review, delete data, alter a branch, or ignore a required check. + +The final archive note recommended retaining research summaries while removing obsolete instructions from default onboarding. It explicitly separated discoverability from automatic injection: useful history should remain findable, but current work should pay the context cost only when the history is relevant. That recommendation is the sole reason this handbook appears in the example bundle. diff --git a/examples/context_bundles/tool_workflows/bundle.json b/examples/context_bundles/tool_workflows/bundle.json new file mode 100644 index 0000000..bba64eb --- /dev/null +++ b/examples/context_bundles/tool_workflows/bundle.json @@ -0,0 +1,29 @@ +{ + "schema_version": "denser.context-bundle/v1", + "name": "tool-workflow-context", + "task_type": "system_prompt", + "negative_control_drop": ["execution-contract"], + "components": [ + { + "id": "execution-contract", + "kind": "system_prompt", + "path": "execution-contract.md", + "required": true + }, + { + "id": "release-policy", + "kind": "claude_md", + "path": "release-policy.md" + }, + { + "id": "ci-policy", + "kind": "claude_md", + "path": "ci-policy.md" + }, + { + "id": "archived-handbook", + "kind": "memory_entry", + "path": "archived-handbook.md" + } + ] +} diff --git a/examples/context_bundles/tool_workflows/ci-policy.md b/examples/context_bundles/tool_workflows/ci-policy.md new file mode 100644 index 0000000..4244614 --- /dev/null +++ b/examples/context_bundles/tool_workflows/ci-policy.md @@ -0,0 +1,8 @@ +This component applies only to CI records. Read the identifier from `run_id`. + +Map CI `policy_code` values exactly as follows: + +- `C4` maps to `STOP`. +- `C1` maps to `RETRY`. + +These codes are arbitrary benchmark labels. Do not derive their meaning from common sense. diff --git a/examples/context_bundles/tool_workflows/execution-contract.md b/examples/context_bundles/tool_workflows/execution-contract.md new file mode 100644 index 0000000..feb9b6a --- /dev/null +++ b/examples/context_bundles/tool_workflows/execution-contract.md @@ -0,0 +1,9 @@ +Use a local filesystem or shell tool to read the exact JSON path named in each request. Do not infer file contents and do not use network access. + +Look up the record's `policy_code` only in the matching policy component. Never invent a mapping that is absent from the supplied context. + +Return exactly one line in this form, with values copied or mapped from the record: + +`DENSER_OK ACTION= ID=` + +Do not add explanations, Markdown, punctuation, or extra whitespace. diff --git a/examples/context_bundles/tool_workflows/fixtures/ci.json b/examples/context_bundles/tool_workflows/fixtures/ci.json new file mode 100644 index 0000000..75e3312 --- /dev/null +++ b/examples/context_bundles/tool_workflows/fixtures/ci.json @@ -0,0 +1,6 @@ +{ + "record_type": "ci", + "run_id": "ci-9K2P", + "policy_code": "C4", + "required_check": "package-smoke" +} diff --git a/examples/context_bundles/tool_workflows/fixtures/release.json b/examples/context_bundles/tool_workflows/fixtures/release.json new file mode 100644 index 0000000..95d58b1 --- /dev/null +++ b/examples/context_bundles/tool_workflows/fixtures/release.json @@ -0,0 +1,6 @@ +{ + "record_type": "release", + "release_id": "rel-7Q4M", + "policy_code": "R7", + "target": "production" +} diff --git a/examples/context_bundles/tool_workflows/release-policy.md b/examples/context_bundles/tool_workflows/release-policy.md new file mode 100644 index 0000000..a4a8b22 --- /dev/null +++ b/examples/context_bundles/tool_workflows/release-policy.md @@ -0,0 +1,8 @@ +This component applies only to release records. Read the identifier from `release_id`. + +Map release `policy_code` values exactly as follows: + +- `R7` maps to `ASK_APPROVAL`. +- `R2` maps to `ALLOW`. + +These codes are arbitrary benchmark labels. Do not derive their meaning from common sense. diff --git a/examples/context_bundles/tool_workflows/replay.json b/examples/context_bundles/tool_workflows/replay.json new file mode 100644 index 0000000..cce78d7 --- /dev/null +++ b/examples/context_bundles/tool_workflows/replay.json @@ -0,0 +1,42 @@ +{ + "schema_version": "denser.replay-suite/v2", + "suite_role": "development", + "tasks": [ + { + "task_type": "system_prompt", + "name": "release_record_tool_read", + "description": "Read a local release record and apply an arbitrary release policy code.", + "covers": ["execution-contract", "release-policy"], + "pass_threshold": 1.0, + "max_tokens": 64, + "cases": [ + { + "name": "production_release", + "prompt": "Use a local tool to read examples/context_bundles/tool_workflows/fixtures/release.json, then apply the supplied context. Final answer only.", + "expected": ["DENSER_OK ACTION=ASK_APPROVAL ID=rel-7Q4M"], + "category": "permission_boundary", + "match_mode": "exact", + "forbidden": ["```", "I think"] + } + ] + }, + { + "task_type": "system_prompt", + "name": "ci_record_tool_read", + "description": "Read a local CI record and apply an arbitrary CI policy code.", + "covers": ["execution-contract", "ci-policy"], + "pass_threshold": 1.0, + "max_tokens": 64, + "cases": [ + { + "name": "required_check_failure", + "prompt": "Use a local tool to read examples/context_bundles/tool_workflows/fixtures/ci.json, then apply the supplied context. Final answer only.", + "expected": ["DENSER_OK ACTION=STOP ID=ci-9K2P"], + "category": "failure_path", + "match_mode": "exact", + "forbidden": ["```", "I think"] + } + ] + } + ] +} diff --git a/examples/context_bundles/tool_workflows/selected.codex-standard.2026-08-18.md b/examples/context_bundles/tool_workflows/selected.codex-standard.2026-08-18.md new file mode 100644 index 0000000..e23c5d6 --- /dev/null +++ b/examples/context_bundles/tool_workflows/selected.codex-standard.2026-08-18.md @@ -0,0 +1,33 @@ +## Context component: execution-contract (system_prompt) + +Use a local filesystem or shell tool to read the exact JSON path named in each request. Do not infer file contents and do not use network access. + +Look up the record's `policy_code` only in the matching policy component. Never invent a mapping that is absent from the supplied context. + +Return exactly one line in this form, with values copied or mapped from the record: + +`DENSER_OK ACTION= ID=` + +Do not add explanations, Markdown, punctuation, or extra whitespace. + +## Context component: release-policy (claude_md) + +This component applies only to release records. Read the identifier from `release_id`. + +Map release `policy_code` values exactly as follows: + +- `R7` maps to `ASK_APPROVAL`. +- `R2` maps to `ALLOW`. + +These codes are arbitrary benchmark labels. Do not derive their meaning from common sense. + +## Context component: ci-policy (claude_md) + +This component applies only to CI records. Read the identifier from `run_id`. + +Map CI `policy_code` values exactly as follows: + +- `C4` maps to `STOP`. +- `C1` maps to `RETRY`. + +These codes are arbitrary benchmark labels. Do not derive their meaning from common sense. diff --git a/examples/context_bundles/tool_workflows/selection.codex-standard.3x.2026-08-18.json b/examples/context_bundles/tool_workflows/selection.codex-standard.3x.2026-08-18.json new file mode 100644 index 0000000..110e39a --- /dev/null +++ b/examples/context_bundles/tool_workflows/selection.codex-standard.3x.2026-08-18.json @@ -0,0 +1,629 @@ +{ + "schema_version": "denser.context-selection/v1", + "selection_method": "greedy-largest-first/v1", + "bundle_name": "tool-workflow-context", + "task_type": "system_prompt", + "target_met": true, + "outcome_reason": "Behavior was preserved and observed full-input reduction was 12.47%.", + "min_input_reduction": 0.1, + "selection_trials": 1, + "validation_trials": 3, + "parallelism": 6, + "baseline_sha256": "f77b3877e30c753149592bd9f4ac25ee83ea4265548298be74bc1c6aa952b4c6", + "selected_sha256": "4b8426823fc5fbd45fed09e556a3eb97fde5cd1e0ccf050ce59834160eee527e", + "components": { + "selected": [ + "execution-contract", + "release-policy", + "ci-policy" + ], + "removed": [ + "archived-handbook" + ], + "required": [ + "execution-contract" + ], + "negative_control_drop": [ + "execution-contract" + ] + }, + "measurements": { + "baseline_estimated_tokens": 4626, + "selected_estimated_tokens": 303, + "estimated_token_reduction": 4323, + "estimated_token_reduction_pct": 0.9345006485084306, + "observed_input_reduction_pct": 0.12473773801512213 + }, + "attempts": [ + { + "component_id": "archived-handbook", + "component_estimated_tokens": 4309, + "removed": true, + "decision": "preserved", + "decision_reason": "The variant matched the baseline on every covered case and the suite caught the known-bad negative control.", + "variant_regressions": [], + "variant_improvements": [], + "observed_input_reduction_pct": 0.12323548456723506 + }, + { + "component_id": "release-policy", + "component_estimated_tokens": 72, + "removed": false, + "decision": "regressed", + "decision_reason": "The variant passed fewer trials than the baseline in one or more covered cases.", + "variant_regressions": [ + "release_record_tool_read/production_release" + ], + "variant_improvements": [], + "observed_input_reduction_pct": 0.12733840099196722 + }, + { + "component_id": "ci-policy", + "component_estimated_tokens": 67, + "removed": false, + "decision": "regressed", + "decision_reason": "The variant passed fewer trials than the baseline in one or more covered cases.", + "variant_regressions": [ + "ci_record_tool_read/required_check_failure" + ], + "variant_improvements": [], + "observed_input_reduction_pct": 0.1278229006969588 + } + ], + "final_audit": { + "schema_version": "denser.context-audit/v1", + "task_type": "system_prompt", + "decision": "preserved", + "decision_reason": "The variant matched the baseline on every covered case and the suite caught the known-bad negative control.", + "variant_regressions": [], + "variant_improvements": [], + "negative_control_detected": true, + "negative_control_regressions": [ + "release_record_tool_read/production_release", + "ci_record_tool_read/required_check_failure" + ], + "measurements": { + "baseline_estimated_tokens": 4626, + "variant_estimated_tokens": 303, + "estimated_token_reduction": 4323, + "estimated_token_reduction_pct": 0.9345006485084306, + "baseline_input_tokens": 277871, + "variant_input_tokens": 243210, + "observed_input_reduction": 34661, + "observed_input_reduction_pct": 0.12473773801512213 + }, + "comparison": { + "schema_version": "denser.replay-report/v4", + "task_type": "system_prompt", + "seed": 3, + "n_trials": 3, + "suite_sha256": "1cc7e7e45893058acc9da17d64af6c4b07db0c0ab2a19857a41ba4e7176d9ab2", + "generated_at_utc": "2026-08-18T06:58:27.414399Z", + "runtime_config": { + "backend_kind": "codex-cli", + "model": "gpt-5.6-sol", + "codex_cli_version": "0.147.0", + "reasoning_effort": "medium", + "sandbox": "read-only", + "capability_profile": "standard", + "ephemeral": true, + "ignore_user_config": true, + "respect_system_proxy": false, + "timeout_seconds": 300.0, + "disabled_features": [ + "apps", + "memories", + "multi_agent" + ] + }, + "suite_metadata": { + "role": "development" + }, + "delta": 0.0, + "original": { + "schema_version": "denser.replay-report/v4", + "task_type": "system_prompt", + "backend_name": "codex-cli/gpt-5.6-sol", + "n_trials": 3, + "instruction_sha256": "f77b3877e30c753149592bd9f4ac25ee83ea4265548298be74bc1c6aa952b4c6", + "suite_sha256": "1cc7e7e45893058acc9da17d64af6c4b07db0c0ab2a19857a41ba4e7176d9ab2", + "generated_at_utc": "2026-08-18T06:58:27.414399Z", + "runtime_config": { + "backend_kind": "codex-cli", + "model": "gpt-5.6-sol", + "codex_cli_version": "0.147.0", + "reasoning_effort": "medium", + "sandbox": "read-only", + "capability_profile": "standard", + "ephemeral": true, + "ignore_user_config": true, + "respect_system_proxy": false, + "timeout_seconds": 300.0, + "disabled_features": [ + "apps", + "memories", + "multi_agent" + ] + }, + "suite_metadata": { + "role": "development" + }, + "overall_pass_rate": 1.0, + "n_tasks": 2, + "n_cases": 2, + "n_errors": 0, + "usage_totals": { + "input_tokens": 277871, + "cached_input_tokens": 159488, + "cache_write_input_tokens": 0, + "output_tokens": 710, + "reasoning_output_tokens": 106 + }, + "task_results": [ + { + "task_name": "release_record_tool_read", + "pass_threshold": 1.0, + "overall_pass_rate": 1.0, + "passed": true, + "n_errors": 0, + "case_results": [ + { + "case_name": "production_release", + "category": "permission_boundary", + "n_trials": 3, + "n_passed": 3, + "pass_rate": 1.0, + "outputs": [ + "DENSER_OK ACTION=ASK_APPROVAL ID=rel-7Q4M", + "DENSER_OK ACTION=ASK_APPROVAL ID=rel-7Q4M", + "DENSER_OK ACTION=ASK_APPROVAL ID=rel-7Q4M" + ], + "errors": [], + "backend_metadata": [ + { + "status": "completed", + "exit_code": 0, + "duration_ms": 129532, + "usage": { + "input_tokens": 46312, + "cached_input_tokens": 22272, + "cache_write_input_tokens": 0, + "output_tokens": 121, + "reasoning_output_tokens": 19 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 128578, + "usage": { + "input_tokens": 46311, + "cached_input_tokens": 32256, + "cache_write_input_tokens": 0, + "output_tokens": 120, + "reasoning_output_tokens": 18 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 127234, + "usage": { + "input_tokens": 46309, + "cached_input_tokens": 32256, + "cache_write_input_tokens": 0, + "output_tokens": 118, + "reasoning_output_tokens": 17 + }, + "transport_fallback": true + } + ] + } + ] + }, + { + "task_name": "ci_record_tool_read", + "pass_threshold": 1.0, + "overall_pass_rate": 1.0, + "passed": true, + "n_errors": 0, + "case_results": [ + { + "case_name": "required_check_failure", + "category": "failure_path", + "n_trials": 3, + "n_passed": 3, + "pass_rate": 1.0, + "outputs": [ + "DENSER_OK ACTION=STOP ID=ci-9K2P", + "DENSER_OK ACTION=STOP ID=ci-9K2P", + "DENSER_OK ACTION=STOP ID=ci-9K2P" + ], + "errors": [], + "backend_metadata": [ + { + "status": "completed", + "exit_code": 0, + "duration_ms": 130765, + "usage": { + "input_tokens": 46313, + "cached_input_tokens": 28160, + "cache_write_input_tokens": 0, + "output_tokens": 117, + "reasoning_output_tokens": 17 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 122922, + "usage": { + "input_tokens": 46313, + "cached_input_tokens": 22272, + "cache_write_input_tokens": 0, + "output_tokens": 117, + "reasoning_output_tokens": 17 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 123109, + "usage": { + "input_tokens": 46313, + "cached_input_tokens": 22272, + "cache_write_input_tokens": 0, + "output_tokens": 117, + "reasoning_output_tokens": 18 + }, + "transport_fallback": true + } + ] + } + ] + } + ] + }, + "candidate": { + "schema_version": "denser.replay-report/v4", + "task_type": "system_prompt", + "backend_name": "codex-cli/gpt-5.6-sol", + "n_trials": 3, + "instruction_sha256": "4b8426823fc5fbd45fed09e556a3eb97fde5cd1e0ccf050ce59834160eee527e", + "suite_sha256": "1cc7e7e45893058acc9da17d64af6c4b07db0c0ab2a19857a41ba4e7176d9ab2", + "generated_at_utc": "2026-08-18T06:58:27.414399Z", + "runtime_config": { + "backend_kind": "codex-cli", + "model": "gpt-5.6-sol", + "codex_cli_version": "0.147.0", + "reasoning_effort": "medium", + "sandbox": "read-only", + "capability_profile": "standard", + "ephemeral": true, + "ignore_user_config": true, + "respect_system_proxy": false, + "timeout_seconds": 300.0, + "disabled_features": [ + "apps", + "memories", + "multi_agent" + ] + }, + "suite_metadata": { + "role": "development" + }, + "overall_pass_rate": 1.0, + "n_tasks": 2, + "n_cases": 2, + "n_errors": 0, + "usage_totals": { + "input_tokens": 243210, + "cached_input_tokens": 121088, + "cache_write_input_tokens": 0, + "output_tokens": 697, + "reasoning_output_tokens": 89 + }, + "task_results": [ + { + "task_name": "release_record_tool_read", + "pass_threshold": 1.0, + "overall_pass_rate": 1.0, + "passed": true, + "n_errors": 0, + "case_results": [ + { + "case_name": "production_release", + "category": "permission_boundary", + "n_trials": 3, + "n_passed": 3, + "pass_rate": 1.0, + "outputs": [ + "DENSER_OK ACTION=ASK_APPROVAL ID=rel-7Q4M", + "DENSER_OK ACTION=ASK_APPROVAL ID=rel-7Q4M", + "DENSER_OK ACTION=ASK_APPROVAL ID=rel-7Q4M" + ], + "errors": [], + "backend_metadata": [ + { + "status": "completed", + "exit_code": 0, + "duration_ms": 132000, + "usage": { + "input_tokens": 40559, + "cached_input_tokens": 19200, + "cache_write_input_tokens": 0, + "output_tokens": 120, + "reasoning_output_tokens": 18 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 127985, + "usage": { + "input_tokens": 40558, + "cached_input_tokens": 19200, + "cache_write_input_tokens": 0, + "output_tokens": 119, + "reasoning_output_tokens": 17 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 124735, + "usage": { + "input_tokens": 40420, + "cached_input_tokens": 19200, + "cache_write_input_tokens": 0, + "output_tokens": 117, + "reasoning_output_tokens": 15 + }, + "transport_fallback": true + } + ] + } + ] + }, + { + "task_name": "ci_record_tool_read", + "pass_threshold": 1.0, + "overall_pass_rate": 1.0, + "passed": true, + "n_errors": 0, + "case_results": [ + { + "case_name": "required_check_failure", + "category": "failure_path", + "n_trials": 3, + "n_passed": 3, + "pass_rate": 1.0, + "outputs": [ + "DENSER_OK ACTION=STOP ID=ci-9K2P", + "DENSER_OK ACTION=STOP ID=ci-9K2P", + "DENSER_OK ACTION=STOP ID=ci-9K2P" + ], + "errors": [], + "backend_metadata": [ + { + "status": "completed", + "exit_code": 0, + "duration_ms": 129797, + "usage": { + "input_tokens": 40568, + "cached_input_tokens": 19200, + "cache_write_input_tokens": 0, + "output_tokens": 124, + "reasoning_output_tokens": 19 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 126734, + "usage": { + "input_tokens": 40563, + "cached_input_tokens": 19200, + "cache_write_input_tokens": 0, + "output_tokens": 119, + "reasoning_output_tokens": 20 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 127422, + "usage": { + "input_tokens": 40542, + "cached_input_tokens": 25088, + "cache_write_input_tokens": 0, + "output_tokens": 98, + "reasoning_output_tokens": 0 + }, + "transport_fallback": true + } + ] + } + ] + } + ] + } + }, + "negative_control": { + "schema_version": "denser.replay-report/v4", + "task_type": "system_prompt", + "backend_name": "codex-cli/gpt-5.6-sol", + "n_trials": 3, + "instruction_sha256": "f830723cc98d91e60fafd7ac1cfca03d44e4d6273d32d5cf0eeed30178d524d4", + "suite_sha256": "1cc7e7e45893058acc9da17d64af6c4b07db0c0ab2a19857a41ba4e7176d9ab2", + "generated_at_utc": "2026-08-18T06:58:27.414399Z", + "runtime_config": { + "backend_kind": "codex-cli", + "model": "gpt-5.6-sol", + "codex_cli_version": "0.147.0", + "reasoning_effort": "medium", + "sandbox": "read-only", + "capability_profile": "standard", + "ephemeral": true, + "ignore_user_config": true, + "respect_system_proxy": false, + "timeout_seconds": 300.0, + "disabled_features": [ + "apps", + "memories", + "multi_agent" + ] + }, + "suite_metadata": { + "role": "development" + }, + "overall_pass_rate": 0.0, + "n_tasks": 2, + "n_cases": 2, + "n_errors": 0, + "usage_totals": { + "input_tokens": 276513, + "cached_input_tokens": 145408, + "cache_write_input_tokens": 0, + "output_tokens": 904, + "reasoning_output_tokens": 343 + }, + "task_results": [ + { + "task_name": "release_record_tool_read", + "pass_threshold": 1.0, + "overall_pass_rate": 0.0, + "passed": false, + "n_errors": 0, + "case_results": [ + { + "case_name": "production_release", + "category": "permission_boundary", + "n_trials": 3, + "n_passed": 0, + "pass_rate": 0.0, + "outputs": [ + "ASK_APPROVAL", + "rel-7Q4M：ASK_APPROVAL", + "rel-7Q4M: ASK_APPROVAL" + ], + "errors": [], + "backend_metadata": [ + { + "status": "completed", + "exit_code": 0, + "duration_ms": 129485, + "usage": { + "input_tokens": 46084, + "cached_input_tokens": 28160, + "cache_write_input_tokens": 0, + "output_tokens": 148, + "reasoning_output_tokens": 59 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 128156, + "usage": { + "input_tokens": 46084, + "cached_input_tokens": 22272, + "cache_write_input_tokens": 0, + "output_tokens": 161, + "reasoning_output_tokens": 65 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 124437, + "usage": { + "input_tokens": 46086, + "cached_input_tokens": 28160, + "cache_write_input_tokens": 0, + "output_tokens": 160, + "reasoning_output_tokens": 64 + }, + "transport_fallback": true + } + ] + } + ] + }, + { + "task_name": "ci_record_tool_read", + "pass_threshold": 1.0, + "overall_pass_rate": 0.0, + "passed": false, + "n_errors": 0, + "case_results": [ + { + "case_name": "required_check_failure", + "category": "failure_path", + "n_trials": 3, + "n_passed": 0, + "pass_rate": 0.0, + "outputs": [ + "ci-9K2P: STOP", + "STOP", + "ci-9K2P：STOP" + ], + "errors": [], + "backend_metadata": [ + { + "status": "completed", + "exit_code": 0, + "duration_ms": 124328, + "usage": { + "input_tokens": 46084, + "cached_input_tokens": 22272, + "cache_write_input_tokens": 0, + "output_tokens": 164, + "reasoning_output_tokens": 70 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 123984, + "usage": { + "input_tokens": 46087, + "cached_input_tokens": 22272, + "cache_write_input_tokens": 0, + "output_tokens": 105, + "reasoning_output_tokens": 14 + }, + "transport_fallback": true + }, + { + "status": "completed", + "exit_code": 0, + "duration_ms": 124718, + "usage": { + "input_tokens": 46088, + "cached_input_tokens": 22272, + "cache_write_input_tokens": 0, + "output_tokens": 166, + "reasoning_output_tokens": 71 + }, + "transport_fallback": true + } + ] + } + ] + } + ] + } + } +} diff --git a/tests/test_audit.py b/tests/test_audit.py index bd1cfc2..5afd282 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import threading +import time from click.testing import CliRunner @@ -164,6 +166,50 @@ def complete(self, *, system: str, user: str, max_tokens: int = 4096) -> str: assert report.variant_improvements == ("release_boundary/production",) +def test_parallel_audit_keeps_per_call_metadata_isolated() -> None: + class _ConcurrentBackend(Backend): + supports_concurrency = True + + def __init__(self) -> None: + self._state = threading.local() + + @property + def last_call_metadata(self) -> dict[str, object]: + metadata = getattr(self._state, "metadata", None) + return metadata if isinstance(metadata, dict) else {} + + def complete(self, *, system: str, user: str, max_tokens: int = 4096) -> str: + del max_tokens + self._state.metadata = {"usage": {"input_tokens": 100 + len(system.split())}} + time.sleep(0.01) + if "BROKEN" in system: + return "ALLOW" + return "ASK_APPROVAL" if "production" in user else "ALLOW" + + @property + def name(self) -> str: + return "concurrent-test" + + @property + def supports_caching(self) -> bool: + return False + + report = audit_context( + baseline="SAFE BASELINE INSTRUCTIONS", + variant="SAFE VARIANT", + negative_control="BROKEN CONTROL", + task_type="claude_md", + tasks=[_task()], + backend=_ConcurrentBackend(), + n_trials=2, + parallelism=6, + ) + + assert report.decision == AuditDecision.PRESERVED + assert report.baseline_input_tokens == 412 + assert report.variant_input_tokens == 408 + + class TestAuditCli: def test_writes_report_and_returns_zero_for_preserved_variant( self, tmp_path, monkeypatch diff --git a/tests/test_context_selection.py b/tests/test_context_selection.py new file mode 100644 index 0000000..d315b3a --- /dev/null +++ b/tests/test_context_selection.py @@ -0,0 +1,364 @@ +"""Tests for conservative context-component selection.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from denser.audit import AuditDecision +from denser.backends.base import Backend +from denser.cli import main +from denser.context_selection import ( + CONTEXT_SELECTION_SCHEMA_VERSION, + load_context_bundle, + minimize_context, +) +from denser.replay import ( + ReplayCase, + ReplayCategory, + ReplaySuite, + ReplaySuiteAuthoring, + ReplaySuiteFreeze, + ReplaySuiteRole, + ReplayTask, + load_replay_suite, +) +from denser.taxonomy import TaskType + +REPO_ROOT = Path(__file__).parents[1] + + +class _SelectionBackend(Backend): + def __init__(self, *, report_usage: bool = True) -> None: + self.last_call_metadata: dict[str, object] = {} + self._report_usage = report_usage + + def complete(self, *, system: str, user: str, max_tokens: int = 4096) -> str: + del max_tokens + self.last_call_metadata = ( + {"usage": {"input_tokens": 100 + len(system.split()), "output_tokens": 1}} + if self._report_usage + else {} + ) + if "CORE_SAFETY" not in system and "production" in user: + return "ALLOW" + if "production" in user: + return "ASK_APPROVAL" if "RELEASE_POLICY" in system else "ALLOW" + return "ALLOW" + + @property + def name(self) -> str: + return "selection-test" + + @property + def supports_caching(self) -> bool: + return False + + +def _suite() -> ReplaySuite: + return ReplaySuite( + tasks=( + ReplayTask( + task_type=TaskType.SYSTEM_PROMPT, + name="release", + description="Release permission behavior.", + cases=( + ReplayCase( + name="preview", + prompt="preview release", + expected="ALLOW", + category=ReplayCategory.POSITIVE_TRIGGER, + ), + ReplayCase( + name="production", + prompt="production release", + expected="ASK_APPROVAL", + category=ReplayCategory.PERMISSION_BOUNDARY, + ), + ), + ), + ) + ) + + +def _write_bundle(tmp_path: Path) -> Path: + (tmp_path / "core.md").write_text("CORE_SAFETY\nReturn exact labels.", encoding="utf-8") + (tmp_path / "release.md").write_text( + "RELEASE_POLICY\nProduction needs approval.", encoding="utf-8" + ) + (tmp_path / "noise.md").write_text( + "IRRELEVANT_STYLE " * 240, + encoding="utf-8", + ) + manifest = tmp_path / "bundle.json" + manifest.write_text( + json.dumps( + { + "schema_version": "denser.context-bundle/v1", + "name": "release-context", + "task_type": "system_prompt", + "negative_control_drop": ["core"], + "components": [ + { + "id": "core", + "kind": "system_prompt", + "path": "core.md", + "required": True, + }, + { + "id": "release-policy", + "kind": "system_prompt", + "path": "release.md", + }, + { + "id": "style-noise", + "kind": "memory_entry", + "path": "noise.md", + }, + ], + } + ), + encoding="utf-8", + ) + return manifest + + +def test_minimize_removes_only_behaviorally_safe_components(tmp_path: Path) -> None: + bundle = load_context_bundle(_write_bundle(tmp_path)) + + selected_text, report = minimize_context( + bundle=bundle, + tasks=_suite(), + backend=_SelectionBackend(), + selection_trials=1, + validation_trials=2, + min_input_reduction=0.10, + ) + + assert report.removed_ids == ("style-noise",) + assert report.selected_ids == ("core", "release-policy") + assert "IRRELEVANT_STYLE" not in selected_text + assert "RELEASE_POLICY" in selected_text + attempts = {attempt.component_id: attempt for attempt in report.attempts} + assert attempts["style-noise"].removed is True + assert attempts["release-policy"].removed is False + assert attempts["release-policy"].decision == AuditDecision.REGRESSED + assert report.final_audit.decision == AuditDecision.PRESERVED + assert report.target_met is True + assert report.to_dict()["schema_version"] == CONTEXT_SELECTION_SCHEMA_VERSION + + +def test_missing_provider_usage_cannot_meet_savings_target(tmp_path: Path) -> None: + bundle = load_context_bundle(_write_bundle(tmp_path)) + + _selected, report = minimize_context( + bundle=bundle, + tasks=_suite(), + backend=_SelectionBackend(report_usage=False), + validation_trials=1, + ) + + assert report.final_audit.decision == AuditDecision.PRESERVED + assert report.observed_input_reduction_pct is None + assert report.target_met is False + assert "did not report" in report.outcome_reason + + +def test_manifest_rejects_component_path_outside_bundle(tmp_path: Path) -> None: + outside = tmp_path.parent / "outside.md" + outside.write_text("do not load", encoding="utf-8") + manifest = tmp_path / "bundle.json" + manifest.write_text( + json.dumps( + { + "schema_version": "denser.context-bundle/v1", + "name": "unsafe", + "task_type": "system_prompt", + "negative_control_drop": ["core"], + "components": [ + { + "id": "core", + "kind": "system_prompt", + "path": "../outside.md", + "required": True, + }, + { + "id": "optional", + "kind": "memory_entry", + "path": "optional.md", + }, + ], + } + ), + encoding="utf-8", + ) + (tmp_path / "optional.md").write_text("optional", encoding="utf-8") + + with pytest.raises(ValueError, match="stay inside"): + load_context_bundle(manifest) + + +def test_minimize_rejects_holdout_suite(tmp_path: Path) -> None: + bundle = load_context_bundle(_write_bundle(tmp_path)) + holdout = ReplaySuite( + tasks=_suite().tasks, + role=ReplaySuiteRole.HOLDOUT, + freeze=ReplaySuiteFreeze( + original_sha256="a" * 64, + candidate_sha256="b" * 64, + candidate_commit="c" * 40, + frozen_at_utc="2026-08-18T00:00:00Z", + ), + authoring=ReplaySuiteAuthoring( + method="blind", + authored_at_utc="2026-08-18T00:00:00Z", + candidate_visible=False, + backend="test", + model="test", + reasoning_effort="medium", + ), + ) + + with pytest.raises(ValueError, match="development replay suite"): + minimize_context(bundle=bundle, tasks=holdout, backend=_SelectionBackend()) + + +def test_cli_writes_selected_context_and_evidence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + manifest = _write_bundle(tmp_path) + suite = tmp_path / "suite.json" + suite.write_text(json.dumps(_suite().to_dict()), encoding="utf-8") + selected = tmp_path / "selected.md" + evidence = tmp_path / "selection.json" + monkeypatch.setattr( + "denser.cli._build_backend", + lambda *args, **kwargs: _SelectionBackend(), + ) + + result = CliRunner().invoke( + main, + [ + "minimize-context", + str(manifest), + "--suite", + str(suite), + "--validation-trials", + "1", + "--out", + str(selected), + "--json-out", + str(evidence), + "--no-progress", + ], + ) + + assert result.exit_code == 0, result.output + assert "Target met: true" in result.output + assert "IRRELEVANT_STYLE" not in selected.read_text(encoding="utf-8") + data = json.loads(evidence.read_text(encoding="utf-8")) + assert data["target_met"] is True + assert data["components"]["removed"] == ["style-noise"] + + +def test_cli_does_not_write_context_when_final_audit_is_inconclusive( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + class _FinalFailureBackend(_SelectionBackend): + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + def complete(self, *, system: str, user: str, max_tokens: int = 4096) -> str: + self.calls += 1 + if self.calls > 12: + raise RuntimeError("simulated final validation outage") + return super().complete(system=system, user=user, max_tokens=max_tokens) + + manifest = _write_bundle(tmp_path) + suite = tmp_path / "suite.json" + suite.write_text(json.dumps(_suite().to_dict()), encoding="utf-8") + selected = tmp_path / "selected.md" + evidence = tmp_path / "selection.json" + monkeypatch.setattr( + "denser.cli._build_backend", + lambda *args, **kwargs: _FinalFailureBackend(), + ) + + result = CliRunner().invoke( + main, + [ + "minimize-context", + str(manifest), + "--suite", + str(suite), + "--validation-trials", + "1", + "--out", + str(selected), + "--json-out", + str(evidence), + "--no-progress", + ], + ) + + assert result.exit_code == 3, result.output + assert not selected.exists() + assert evidence.exists() + assert "Did not write selected context" in result.output + data = json.loads(evidence.read_text(encoding="utf-8")) + assert data["final_audit"]["decision"] == "inconclusive" + + +def test_committed_tool_workflow_evidence_is_bound_and_clears_gate() -> None: + example = REPO_ROOT / "examples" / "context_bundles" / "tool_workflows" + bundle = load_context_bundle(example / "bundle.json") + selected = (example / "selected.codex-standard.2026-08-18.md").read_text(encoding="utf-8") + report = json.loads( + (example / "selection.codex-standard.3x.2026-08-18.json").read_text(encoding="utf-8") + ) + suite = load_replay_suite(example / "replay.json") + + assert report["schema_version"] == CONTEXT_SELECTION_SCHEMA_VERSION + assert report["target_met"] is True + assert report["parallelism"] == 6 + assert report["components"]["removed"] == ["archived-handbook"] + assert report["components"]["selected"] == [ + "execution-contract", + "release-policy", + "ci-policy", + ] + assert ( + report["baseline_sha256"] + == hashlib.sha256(bundle.baseline_text.encode("utf-8")).hexdigest() + ) + assert report["selected_sha256"] == hashlib.sha256(selected.encode("utf-8")).hexdigest() + prompts = "\n".join(case.prompt for task in suite.tasks for case in task.cases) + for identifier in ("rel-7Q4M", "ci-9K2P"): + assert identifier not in bundle.baseline_text + assert identifier not in prompts + + final = report["final_audit"] + assert final["decision"] == "preserved" + assert final["negative_control_detected"] is True + assert final["measurements"]["observed_input_reduction_pct"] >= 0.10 + runtime = final["comparison"]["runtime_config"] + assert runtime["capability_profile"] == "standard" + assert runtime["reasoning_effort"] == "medium" + assert all( + capability not in runtime["disabled_features"] + for capability in ("shell_tool", "plugins", "skill_search") + ) + for side in ("original", "candidate"): + side_report = final["comparison"][side] + assert side_report["overall_pass_rate"] == 1.0 + assert side_report["n_errors"] == 0 + assert all( + case["n_passed"] == case["n_trials"] == 3 + for task in side_report["task_results"] + for case in task["case_results"] + )