diff --git a/README.md b/README.md index 026f9d0..40d14ab 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Get the plugin · Install · Try it yourself · - 30-second demo · + 29-second demo · How it works · Evidence · Controls · @@ -39,10 +39,15 @@ Memory is built and organized locally, **without LLM calls or additional billed tokens for memory management**. These results cover one task with existing prior work; see [Evidence](#evidence) for the comparison and its limits. -**[Watch the 30-second walkthrough](docs/demo.md)** or -**[try it in two chats](docs/try-it.md)**. Ask ordinary questions: Eggshell -automatically brings relevant findings from the first investigation into the -follow-up. The walkthrough is an edited English summary of the measured study. +**Watch Eggshell carry work into a new chat — 29 seconds.** + +https://github.com/user-attachments/assets/939eca34-4897-4bcf-9dbc-a6326bc9ba3a + +Actual Codex CLI, Luna xhigh, independent ephemeral chats. In this recorded pair, +Eggshell used **81.8% fewer input + output tokens in the second chat**; both +answers passed static source review. One pair, with time edited for clarity. +[Demo and measurement details](docs/cli-demo.md) · +[Try it in two chats](docs/try-it.md). Use the published **Codex plugin**, or the separate, experimental [adapters for Claude Code, Gemini CLI, Cursor, and OpenCode](adapters/README.md). diff --git a/docs/benchmarks/cli-demo.json b/docs/benchmarks/cli-demo.json new file mode 100644 index 0000000..478fb37 --- /dev/null +++ b/docs/benchmarks/cli-demo.json @@ -0,0 +1,73 @@ +{ + "description": "One recorded Clang follow-up comparison in independent ephemeral Codex chats.", + "model": "gpt-5.6-luna", + "reasoning_effort": "xhigh", + "ephemeral": true, + "llvm_commit": "6dfe1677ab8dffbc6ec13d53a1e0215d75147689", + "source_tree": "1dbf5dbe50ba286d5e9e008bbd230e47d4de0dbd", + "initial_prompt_sha256": "ac3c77019e4a2c83c7b0c8253a391fb1275cc7d01b0d063c5d42b3b910c1b31f", + "followup_prompt_sha256": "8a6ee07fec059fbea973da71398ffd6ba60c9f78c0765fce6fe5b9d57eae4acf", + "prior_egg_sha256": "f087d3548e83de69f8dc9029578203886a51857d304239ed215bb5c83c3b0591", + "runtime_sha256": "d9d687ce1cb4cc994ebfa99d9bed7b4a6784841b7a83c53f09ed7592d19813f0", + "shared_initial_investigation": true, + "token_definition": "Input plus output, including cached input. Reasoning output is already included in output.", + "runs": { + "initial": { + "input_tokens": 828632, + "output_tokens": 6673, + "cached_input_tokens": 725504, + "total_tokens": 835305 + }, + "fresh": { + "input_tokens": 1066830, + "output_tokens": 7667, + "cached_input_tokens": 939520, + "total_tokens": 1074497 + }, + "eggshell": { + "input_tokens": 188254, + "output_tokens": 7114, + "cached_input_tokens": 149760, + "total_tokens": 195368 + } + }, + "totals": { + "fresh_plus_fresh": 1909802, + "fresh_plus_eggshell": 1030673, + "followup_reduction": 0.8181772494478812, + "overall_reduction": 0.4603246828728842 + }, + "quality": { + "method": "Single-reviewer, non-blinded comparison with the fixed source snapshot and native execution records. No runtime tests were requested or run; this is not a general equivalence claim.", + "criteria": [ + "Direct target and architecture options can affect the triple before initial host toolchain lookup.", + "Architecture-scoped wrapper expansion happens later and does not retroactively choose the already selected host toolchain.", + "The answer covers architecture/offload matching and the parser or NoXarchOption restrictions.", + "The answer names a relevant existing test and source locations.", + "The answer distinguishes static source evidence from runtime behavior that was not tested." + ], + "fresh": { + "result": "usable", + "criteria_satisfied": 5, + "answer_sha256": "25bdcdfb449d725d7d0aa7ddd48aa015fe92edf096b781d16a55e759f00820e6" + }, + "eggshell": { + "result": "usable", + "criteria_satisfied": 5, + "answer_sha256": "d894cb324e92f1619c97a12c26f2204fcc489c8790a439b6447511e20ca77954" + } + }, + "video": { + "url": "https://github.com/user-attachments/assets/939eca34-4897-4bcf-9dbc-a6326bc9ba3a", + "duration_seconds": 29, + "sha256": "736434e89b1393b2c0949530613bd47295f14481e22e34b77fe01896ee174261", + "editing": "Elapsed time compressed, with pauses and editorial annotations at handoff delivery and explicit reuse. Native output text and measurements unchanged." + }, + "limitations": [ + "One recorded pair; results vary by task.", + "Shared initial investigation is charged to both routes, not two independent initial runs.", + "Static source review by one non-blinded reviewer; no Clang runtime tests.", + "Token reduction is not a dollar-cost estimate.", + "Raw private transcripts are not published; hashes identify local evidence and do not replace access to it." + ] +} diff --git a/docs/cli-demo.md b/docs/cli-demo.md new file mode 100644 index 0000000..31b303b --- /dev/null +++ b/docs/cli-demo.md @@ -0,0 +1,55 @@ +# Two chats. Less repeated work. + +[Project home](../README.md) · [Try it yourself](try-it.md) · [Measurement record](benchmarks/cli-demo.json) + +https://github.com/user-attachments/assets/939eca34-4897-4bcf-9dbc-a6326bc9ba3a + +This 29-second demo uses actual Codex CLI recordings with **Luna xhigh** and +**independent ephemeral chats**. One chat investigates Clang’s toolchain selection. +Two new chats receive the same related question: one starts fresh with Eggshell +hooks disabled; the other receives relevant findings from the saved `.egg`. + +The recording highlights the handoff delivery and the agent’s explicit reuse +statement. Elapsed time is compressed, with pauses for those annotations. The +terminal output and measured token totals are unchanged. + +## The recorded comparison + +| Work | Fresh + fresh | Fresh + Eggshell | +| --- | ---: | ---: | +| Shared first investigation | 835,305 | 835,305 | +| Second-chat follow-up | 1,074,497 | 195,368 | +| Both chats | 1,909,802 | 1,030,673 | + +**81.8% fewer tokens in the second chat; 46.0% fewer across both chats.** +The same first investigation is charged to both routes. Tokens mean input plus +output, including cached input; reasoning is already included in output. These +are token counts, not dollar-cost estimates. Memory organization makes no extra +LLM calls; the selected handoff still consumes normal model input tokens. + +Both final answers passed five static-source review criteria covering selection +order, wrapper restrictions, relevant source/tests, and accurate reporting of +what was not run. This was one recorded pair and a single non-blinded reviewer, +without Clang runtime tests. It does not establish general savings or equivalent +answer quality across tasks. The README’s [ten-trial study](../README.md#evidence) +and its [earlier walkthrough](demo.md) are a separate experiment. + +## The follow-up question + +```text +In this Clang source snapshot, can an architecture-scoped -Xarch_* argument +change the already selected host toolchain in the same way that a direct +-target, -m32, or -m64 option can? Explain the relevant ordering and constraints, +with function names and source-line citations, and identify a relevant existing +driver test. Keep the final answer within 250 words. Do not modify files or +build LLVM. Distinguish static source findings from untested runtime behavior. +``` + +Both follow-ups used the same prompt, model, effort, and fixed +[LLVM source commit](https://github.com/llvm/llvm-project/tree/6dfe1677ab8dffbc6ec13d53a1e0215d75147689). +Source, prompt, prior-work, runtime, answer and video hashes are included in the +[measurement record](benchmarks/cli-demo.json). Raw private execution logs remain +local; hashes identify those records but are not a substitute for access to them. + +The key ordering is visible in [host toolchain selection](https://github.com/llvm/llvm-project/blob/6dfe1677ab8dffbc6ec13d53a1e0215d75147689/clang/lib/Driver/Driver.cpp#L1695-L1722) +and [per-toolchain argument translation](https://github.com/llvm/llvm-project/blob/6dfe1677ab8dffbc6ec13d53a1e0215d75147689/clang/lib/Driver/Compilation.cpp#L63-L111).