diff --git a/.github/workflows/evolve-agent-ci.yml b/.github/workflows/evolve-agent-ci.yml new file mode 100644 index 0000000..62ca7f2 --- /dev/null +++ b/.github/workflows/evolve-agent-ci.yml @@ -0,0 +1,31 @@ +name: evolve-agent-ci + +on: + push: + paths: + - "evolve-agent/**" + - ".github/workflows/evolve-agent-ci.yml" + pull_request: + paths: + - "evolve-agent/**" + - ".github/workflows/evolve-agent-ci.yml" + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: evolve-agent + strategy: + matrix: + node-version: [22, 24] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + - run: npm install --no-audit --no-fund + - run: npm run check diff --git a/evolve-agent/.env.example b/evolve-agent/.env.example new file mode 100644 index 0000000..a8146f1 --- /dev/null +++ b/evolve-agent/.env.example @@ -0,0 +1,69 @@ +# Model +OPENAI_API_KEY= +OPENAI_MODEL=gpt-5.6-sol +OPENAI_VERIFIER_MODEL=gpt-5.6-sol +EVOLVE_REASONING_EFFORT=high + +# State and workspace +# EVOLVE_HOME must remain outside EVOLVE_WORKSPACE. When omitted, Evolve Agent +# derives a per-workspace directory under XDG_STATE_HOME or ~/.local/state. +# EVOLVE_HOME=/absolute/path/outside/the/workspace +EVOLVE_WORKSPACE=. + +# Hardened execution — Docker is the default and fails closed until a pinned +# image is both allowlisted and already present locally. +EVOLVE_EXECUTOR=docker +EVOLVE_DOCKER_BINARY=docker +# Example shape only: repository/name@sha256:<64 lowercase hex characters> +# EVOLVE_DOCKER_DEFAULT_IMAGE=node:22-bookworm-slim@sha256:... +# EVOLVE_DOCKER_ALLOWED_IMAGES=node:22-bookworm-slim@sha256:... +EVOLVE_DOCKER_ALLOWED_NETWORKS= +EVOLVE_DOCKER_USER=65532:65532 +EVOLVE_DOCKER_MAX_MEMORY_MB=2048 +EVOLVE_DOCKER_MAX_CPUS=2 +EVOLVE_DOCKER_MAX_PIDS=256 +EVOLVE_DOCKER_MAX_TMPFS_MB=256 +EVOLVE_DOCKER_REQUIRE_ROOTLESS=false + +# Host execution is an explicit unsafe escape hatch. +EVOLVE_ALLOW_LOCAL_EXECUTOR=false + +# Process boundary +EVOLVE_ALLOWED_COMMANDS=git,node,npm,npx,pnpm,python,python3,pytest,vitest,tsc +EVOLVE_NON_INTERACTIVE=false + +# Only names in this list can be materialized as short-lived read-only files. +EVOLVE_SECRET_ALLOWLIST= +EVOLVE_SECRET_TTL_MS=300000 + +# Crash and duplicate-run protection +EVOLVE_LEASE_TTL_MS=30000 +EVOLVE_LEASE_HEARTBEAT_MS=10000 + +# Evaluation automation is conservative by default. Fixture capture and shadow +# traffic are opt-in. Promotion is always explicit; rollback may be automatic. +EVOLVE_EVAL_CAPTURE_COMMITTED=false +EVOLVE_EVAL_SHADOW_PERCENT=0 +EVOLVE_EVAL_MONITOR_PROMOTED=true + +# Offline evaluation policy +EVOLVE_EVAL_MIN_FIXTURES=3 +EVOLVE_EVAL_REPEATS=1 +EVOLVE_EVAL_MAX_NEW_FAILURES=0 +EVOLVE_EVAL_MAX_SUCCESS_REGRESSION=0 +EVOLVE_EVAL_MAX_SCORE_REGRESSION=0.02 +EVOLVE_EVAL_MIN_SUCCESS_IMPROVEMENT=0.05 +EVOLVE_EVAL_MIN_SCORE_IMPROVEMENT=0.02 +EVOLVE_EVAL_MAX_TOKEN_RATIO=1.2 +EVOLVE_EVAL_MAX_TOOL_RATIO=1.2 +EVOLVE_EVAL_EFFICIENCY_RATIO=0.9 +EVOLVE_EVAL_CONFIDENCE=0.9 +EVOLVE_EVAL_BOOTSTRAP_SAMPLES=1000 + +# Shadow canary and production rollback envelope +EVOLVE_EVAL_CANARY_MIN_SAMPLES=5 +EVOLVE_EVAL_MONITOR_MIN_SAMPLES=10 +EVOLVE_EVAL_MONITOR_WINDOW=50 +EVOLVE_EVAL_MONITOR_MAX_SUCCESS_DROP=0.1 +EVOLVE_EVAL_MONITOR_MAX_SCORE_DROP=0.1 +EVOLVE_EVAL_MONITOR_MAX_TOKEN_RATIO=1.5 diff --git a/evolve-agent/.gitignore b/evolve-agent/.gitignore new file mode 100644 index 0000000..e608511 --- /dev/null +++ b/evolve-agent/.gitignore @@ -0,0 +1,8 @@ +node_modules/ +dist/ +.test-dist/ +coverage/ +.env +.evolve/ +*.log +.DS_Store diff --git a/evolve-agent/CHANGELOG.md b/evolve-agent/CHANGELOG.md new file mode 100644 index 0000000..63f4cfb --- /dev/null +++ b/evolve-agent/CHANGELOG.md @@ -0,0 +1,65 @@ +# Changelog + +## 0.3.0 — Evaluation-Driven Evolution + +### Added + +- content-addressed replay fixtures captured from clean committed Episodes +- supporting-Episode leakage exclusion and frozen training provenance +- paired baseline-versus-candidate replay harness +- exact proposal-trace matching and executed-argument evidence validation +- success, verifier, token, tool-call, duration, trace, and safety metrics +- deterministic paired bootstrap confidence intervals +- configurable non-regression and improvement gates +- Ed25519-signed offline, canary, and monitor reports +- shadow canaries that cannot alter production answers +- rolling production monitoring against a signed canary envelope +- automatic rollback on production regression +- `quarantined` Skill state +- evaluation fixture, report, shadow, monitor, verify, and promotion CLI commands +- evaluation authority enforcement inside `SkillStore` +- 12 new evaluation and integration tests, bringing the suite to 38 tests + +### Changed + +- package version raised to 0.3.0 +- manual canary-score recording replaced by report-backed evaluation flow +- promotion now requires verified signed offline and canary reports +- runtime checkpoints record active Skill IDs and final verifier score +- committed Episode capture and shadow sampling are opt-in; promoted-Skill monitoring remains enabled by default + +### Security + +- direct low-level promotion fails closed without a signed-report verifier +- report tampering, Skill/report identity mismatch, authority-key mismatch, and policy-hash mismatch block promotion +- fixture imports and captures validate evidence provenance and integrity +- candidate training Episodes cannot approve the candidate they produced + +## 0.2.0 — Hardened Execution + +### Added + +- executor interface with Docker and explicit local backends +- immutable image and network policy objects +- Docker resource and privilege hardening +- short-lived file secret broker with output redaction +- execution receipts with command and policy hashes +- Episode leases, heartbeats, duplicate-run protection, and stale recovery +- `doctor`, `executors list`, and `secrets sweep` diagnostics +- dedicated hardened-execution design documentation +- 14 security tests, bringing the suite to 26 tests + +### Changed + +- Docker became the default process executor +- local execution became disabled by default +- state home defaults outside the workspace and inside-workspace state is rejected +- truncated process output terminates execution and marks it unsuccessful + +## 0.1.0 — Evidence-Gated Kernel + +- bounded autonomous loop +- evidence artifacts and hash-chained ledger +- exact approval capabilities +- independent final verification +- governed memory and candidate Skill lifecycle diff --git a/evolve-agent/CONTRIBUTING.md b/evolve-agent/CONTRIBUTING.md new file mode 100644 index 0000000..813874f --- /dev/null +++ b/evolve-agent/CONTRIBUTING.md @@ -0,0 +1,14 @@ +# Contributing + +1. Create a focused branch and keep unrelated application changes out of the PR. +2. Add or update tests for every invariant touched. +3. Run `npm run check` and `npm pack --dry-run` before publishing. +4. Keep tool, executor, fixture, evaluator, and promotion permissions narrow and fail closed. +5. Do not add unrestricted shell execution, silent approval bypasses, implicit image pulls, broad network defaults, automatic Skill promotion, or unsigned promotion shortcuts. +6. A mutating tool must define risk class, validation, evidence output, and rollback behavior. +7. An executor must state which isolation properties it actually enforces and encode them in its receipt. +8. An evaluation metric must define its direction, pairing unit, failure semantics, sample requirements, and regression threshold. +9. Evaluation fixtures must preserve provenance and must not reuse a candidate's supporting Episodes as approval data. +10. Never put secret values, private signing keys, or production credentials in arguments, logs, artifacts, fixtures, tests, or committed environment files. +11. Keep agent authority and evaluation state outside task-mounted workspaces. +12. Document the trusted computing base and residual risk; do not describe containers, model verification, or signed reports as stronger guarantees than they provide. diff --git a/evolve-agent/LICENSE b/evolve-agent/LICENSE new file mode 100644 index 0000000..c16e68e --- /dev/null +++ b/evolve-agent/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 DCLXAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/evolve-agent/README.md b/evolve-agent/README.md new file mode 100644 index 0000000..9b89ad4 --- /dev/null +++ b/evolve-agent/README.md @@ -0,0 +1,501 @@ +# Evolve Agent + +> An evidence-gated autonomous agent kernel for **GPT-5.6 Sol** with hardened execution and evaluation-driven Skill evolution. + +Evolve Agent is built around one rule: + +> An agent gains authority only when its work is observable, evidence-backed, bounded, isolated, evaluated, and reversible. + +OpenClaw is strong at gateway reach. Hermes Agent is strong at persistent learning loops. Evolve Agent focuses on the control layer that a learning agent needs before self-improvement can be trusted: **verifiable execution plus measured, signed, reversible adaptation**. + +## v0.3 — Evaluation-Driven Evolution + +v0.1 established evidence, exact capabilities, and a governed Skill lifecycle. v0.2 moved process execution behind a Docker-first, deny-by-default boundary. **v0.3 replaces manually entered evaluation scores with a real baseline-versus-candidate evaluation pipeline.** + +A Skill cannot be promoted because it looks plausible, because it worked once, or because the same Episodes that created it also approve it. It must pass independent replay fixtures, survive non-intervening shadow canaries, and remain inside a signed production envelope after promotion. + +### What v0.3 adds + +- content-addressed replay fixtures captured from clean committed Episodes +- strict train/evaluation leakage exclusion +- paired baseline-versus-candidate replay under the same task, tools, observations, budgets, and verifier +- exact model-proposal trace matching plus executed-argument evidence provenance +- success, verifier quality, token, tool-call, latency, trace, and safety metrics +- deterministic paired bootstrap confidence intervals +- configurable non-regression and improvement gates +- Ed25519-signed offline, canary, and production-monitor reports +- promotion enforcement inside `SkillStore`, not only in the CLI facade +- shadow canaries that never replace the production answer +- production monitoring against the signed canary envelope +- automatic rollback when promoted-Skill quality or cost crosses configured limits +- frozen training provenance once evaluation begins, preserving later Episodes as holdout material +- `quarantined` state for candidates that fail offline or canary gates +- 38 invariant, security, evaluation, and end-to-end tests + +## Evolution path + +```text +Committed Episodes + | + v +Repeated successful trace + | + v +Candidate Skill + | + +---- training Episode IDs and evidence are frozen + | + v +Independent replay fixtures + | + v +Paired baseline vs candidate execution + | + v +Quality + safety + cost + confidence gates + | + v +Signed offline report + | + v +Shadow canary on real production Episodes + | candidate output is discarded + | production answer is unchanged + v +Signed canary report + | + v +Explicit promotion + | + v +Rolling production monitor + | + +---- within signed envelope ----> remain promoted + | + +---- regression ---------------> automatic rollback +``` + +See [docs/EVALUATION_DRIVEN_EVOLUTION.md](docs/EVALUATION_DRIVEN_EVOLUTION.md) for the evaluation contract and [docs/HARDENED_EXECUTION.md](docs/HARDENED_EXECUTION.md) for the execution boundary. + +## Core invariants + +1. A candidate's supporting Episodes cannot be used as evaluation fixtures. +2. Baseline and candidate receive the same replay world and verifier contract. +3. A replay fails closed if the model changes the recorded tool sequence or proposed arguments. +4. Evidence in a fixture must resolve to the source Episode and actual executed arguments. +5. One new candidate failure blocks promotion by default. +6. Offline evaluation must show improvement without violating quality or cost budgets. +7. Canary evaluation runs in shadow and cannot change production output. +8. Offline and canary reports must be valid signatures from the same evaluation authority. +9. `SkillStore` re-verifies signed reports, Skill identity, policy hash, and key fingerprint before promotion. +10. A promoted Skill is automatically rolled back when the rolling production window breaches its signed canary envelope. +11. Learned Skills never promote themselves. +12. Docker remains the default fail-closed process boundary. + +## Requirements + +- Node.js 22.6 or newer +- Docker Engine or Docker Desktop for the default process executor +- an OpenAI API key with access to the configured model +- at least one reviewed Docker image available locally under an exact digest for process tools + +The package has no runtime npm dependencies. + +## Install + +```bash +cd evolve-agent +npm install +cp .env.example .env +npm run check +``` + +Export the API key through the shell or a protected environment manager: + +```bash +export OPENAI_API_KEY="..." +``` + +## Configure hardened execution + +Pull an image deliberately, inspect its immutable repository digest, and allowlist that exact value: + +```bash +docker pull node:22-bookworm-slim +IMAGE="$(docker image inspect node:22-bookworm-slim --format '{{index .RepoDigests 0}}')" + +export EVOLVE_DOCKER_DEFAULT_IMAGE="$IMAGE" +export EVOLVE_DOCKER_ALLOWED_IMAGES="$IMAGE" +``` + +Evolve Agent uses `--pull never`. A missing image fails closed instead of silently changing the execution environment. + +Recommended strict mode: + +```bash +export EVOLVE_DOCKER_REQUIRE_ROOTLESS=true +``` + +Check readiness without revealing secret values: + +```bash +npm run dev -- doctor +``` + +## Run an evidence-backed task + +```bash +npm run dev -- run \ + "Read package.json and explain the scripts" \ + --workspace . \ + --tool read_file \ + --success "Every package-specific claim cites current-Episode evidence" +``` + +A valid evidence-backed answer cites IDs created by that exact Episode: + +```text +The test script compiles the test tree and invokes Node's test runner. [evidence:ev_...] +``` + +## Evaluation workflow + +### 1. Capture replay fixtures + +Fixture capture is opt-in by default. Capture a known clean committed Episode explicitly: + +```bash +npm run dev -- evaluations fixtures capture --split validation +``` + +List fixtures: + +```bash +npm run dev -- evaluations fixtures list +npm run dev -- evaluations fixtures list --split validation --split holdout +``` + +Import a reviewed fixture JSON file: + +```bash +npm run dev -- evaluations fixtures import ./fixture.json +``` + +A captured fixture contains: + +- original `TaskSpec` and budgets +- ordered model-proposed tool trace +- proposal argument hashes +- executed-argument evidence records and artifact hashes +- recorded observations +- baseline verifier score, usage, turns, tool calls, duration, and active Skill IDs +- source Episode ID, split, integrity hash, and deterministic fixture ID + +Only clean committed Episodes are accepted. Policy denial, approval denial, rejected arguments, terminal failure, incomplete traces, missing artifacts, and provenance mismatch cause fixture rejection. + +### 2. Run offline baseline-versus-candidate evaluation + +```bash +npm run dev -- evaluations run \ + --split validation \ + --split holdout \ + --repeats 2 +``` + +Select exact fixtures when needed: + +```bash +npm run dev -- evaluations run \ + --fixture fixture_... \ + --fixture fixture_... +``` + +`skills evaluate` is an alias: + +```bash +npm run dev -- skills evaluate +``` + +The harness alternates execution order to reduce ordering bias. Each arm receives the same fixture, tools, budgets, evidence observations, and verifier. The candidate arm differs only by inclusion of the candidate Skill. + +Default offline gates require: + +- at least three unique matching fixtures +- complete baseline/candidate pairs +- zero new failures +- no success-rate regression +- no verifier-score regression beyond 0.02 +- lower confidence bounds within non-regression limits +- no more than 20% token or tool-call regression +- no trace or safety regression +- measurable success, score, token, or tool-call improvement + +A failed report moves the Skill to `quarantined`. A passing report moves it to `evaluated`. In both cases, the report is signed and retained. + +### 3. Run shadow canaries + +Shadow a production Episode whose answer was produced without the candidate Skill: + +```bash +npm run dev -- evaluations shadow \ + --mode production-baseline +``` + +The actual production result becomes the baseline. The candidate is replayed counterfactually against the recorded fixture. Its answer is measured and discarded; it cannot alter the response already delivered to the user. + +After enough independent shadow samples, finalize the canary report: + +```bash +npm run dev -- evaluations canary +``` + +Canary gates require non-regression. The offline report is responsible for proving improvement. + +### 4. Promote explicitly + +```bash +npm run dev -- skills promote +``` + +Promotion fails unless: + +- the latest attached offline report is signed, valid, passing, and of kind `offline` +- the latest attached canary report is signed, valid, passing, and of kind `canary` +- both reports identify the same Skill ID and fingerprint +- both reports were signed by the same authority key +- the authorization policy hash matches the policies inside the signed reports +- the Skill is in a passing `canary` state + +The low-level `SkillStore` performs this verification again, so bypassing the CLI does not bypass the signed-report gate. + +### 5. Monitor and roll back + +Inspect the current production window: + +```bash +npm run dev -- evaluations monitor +``` + +When monitoring is enabled, every terminal Episode that used a promoted Skill contributes a production outcome. Once the minimum sample count is reached, Evolve Agent compares the rolling window with the signed canary candidate envelope. + +Default rollback thresholds permit at most: + +- 0.10 success-rate drop +- 0.10 mean verifier-score drop +- 1.5× mean token use + +Any failed gate creates a signed monitor report and automatically changes the Skill to `rolled_back`. + +Manual rollback remains available: + +```bash +npm run dev -- skills rollback --note "operator-observed regression" +``` + +## Opt-in automation + +Safe defaults are conservative: + +```text +EVOLVE_EVAL_CAPTURE_COMMITTED=false +EVOLVE_EVAL_SHADOW_PERCENT=0 +EVOLVE_EVAL_MONITOR_PROMOTED=true +``` + +To capture clean committed Episodes and shadow a percentage of matching traffic: + +```bash +export EVOLVE_EVAL_CAPTURE_COMMITTED=true +export EVOLVE_EVAL_SHADOW_PERCENT=10 +``` + +The shadow percentage is deterministic per Episode and Skill. Shadow evaluation remains off the response path. Automatic **promotion is never enabled**; promotion is always explicit. Automatic rollback is enabled for promoted Skills when monitoring is on. + +## Reports and provenance + +```bash +npm run dev -- evaluations reports list +npm run dev -- evaluations reports show +npm run dev -- evaluations verify +``` + +Evaluation authority keys are generated under the state directory: + +```text +$EVOLVE_HOME/evaluations/authority/ed25519-private.pem mode 0600 +$EVOLVE_HOME/evaluations/authority/ed25519-public.pem +``` + +A signed report binds: + +- report kind and engine version +- Skill ID and Skill fingerprint +- fixture IDs and integrity hashes +- complete baseline and candidate runs +- aggregate metrics and paired comparison +- evaluation policy and gate results +- provider identity and notes +- payload hash, Ed25519 signature, and key fingerprint + +The local key proves that the report came from the local evaluation authority and was not modified afterward. It is not hardware attestation and does not prove that the host itself was uncompromised. + +## Evaluation configuration + +Key settings are shown below; see `.env.example` for the complete list. + +```text +EVOLVE_EVAL_MIN_FIXTURES=3 +EVOLVE_EVAL_REPEATS=1 +EVOLVE_EVAL_MAX_NEW_FAILURES=0 +EVOLVE_EVAL_MAX_SUCCESS_REGRESSION=0 +EVOLVE_EVAL_MAX_SCORE_REGRESSION=0.02 +EVOLVE_EVAL_MIN_SUCCESS_IMPROVEMENT=0.05 +EVOLVE_EVAL_MIN_SCORE_IMPROVEMENT=0.02 +EVOLVE_EVAL_MAX_TOKEN_RATIO=1.2 +EVOLVE_EVAL_MAX_TOOL_RATIO=1.2 +EVOLVE_EVAL_EFFICIENCY_RATIO=0.9 +EVOLVE_EVAL_CONFIDENCE=0.9 +EVOLVE_EVAL_BOOTSTRAP_SAMPLES=1000 +EVOLVE_EVAL_CANARY_MIN_SAMPLES=5 +EVOLVE_EVAL_MONITOR_MIN_SAMPLES=10 +EVOLVE_EVAL_MONITOR_WINDOW=50 +EVOLVE_EVAL_MONITOR_MAX_SUCCESS_DROP=0.1 +EVOLVE_EVAL_MONITOR_MAX_SCORE_DROP=0.1 +EVOLVE_EVAL_MONITOR_MAX_TOKEN_RATIO=1.5 +``` + +`EVOLVE_EVAL_MONITOR_WINDOW` must be at least both the canary and monitor minimum sample counts. + +## Hardened process execution + +`run_process` remains behind the v0.2 authority boundary: + +- Docker default; local host execution disabled unless explicitly enabled +- exact digest image allowlist and `--pull never` +- network `none` by default +- read-only root and workspace by default +- numeric non-root user +- all Linux capabilities dropped +- `no-new-privileges` +- memory, swap, CPU, PID, tmpfs, file-descriptor, timeout, and output limits +- short-lived file secrets with output redaction +- execution receipt with command and policy hashes +- Episode lease, heartbeat, duplicate-run prevention, and stale recovery + +See [docs/HARDENED_EXECUTION.md](docs/HARDENED_EXECUTION.md). + +## State layout + +State must remain outside the mounted task workspace: + +```text +$EVOLVE_HOME/ + capability.key + episodes.jsonl + checkpoints/ + artifacts/ + evidence/ + memory.json + patterns.json + skills.json + leases/ + runtime/secrets/ + evaluations/ + fixtures/ + reports/ + shadow.json + authority/ + ed25519-private.pem + ed25519-public.pem +``` + +When `EVOLVE_HOME` is omitted, a per-workspace directory is derived under `$XDG_STATE_HOME/evolve-agent/` or `~/.local/state/evolve-agent/`. + +## Built-in tools + +| Tool | Risk | Main controls | +|---|---:|---| +| `list_files` | read | workspace boundary, recursion and entry caps | +| `read_file` | read | regular-file check, byte cap, SHA-256 output | +| `search_text` | read | literal search, file/result/size caps | +| `write_file` | write | approval, exact capability, create-only and SHA compare-and-swap | +| `replace_text` | write | approval, exact occurrence count and SHA compare-and-swap | +| `run_process` | execute | approval, exact capability, executor policy, isolation receipt and evidence limits | + +## Validation + +```bash +npm run check +``` + +The v0.3 suite contains **38 tests** covering: + +- exact capability binding and post-approval argument mutation +- evidence contract and fabricated evidence rejection +- ledger tamper detection +- workspace traversal and symlink rejection +- Docker digest, network, privilege, resource, timeout, and output policy +- secret file permissions, cleanup, and output redaction +- Episode concurrency and stale-lock recovery +- replay fixture capture and tamper detection +- supporting-Episode leakage exclusion +- paired baseline/candidate trace replay +- trace mismatch fail-closed behavior +- evaluation metrics, bootstrap gates, and new-failure rejection +- Ed25519 report signing and tamper rejection +- promotion-store authority enforcement +- shadow canary non-intervention +- production regression automatic rollback +- runtime active-Skill attribution and optional fixture capture +- frozen Skill training provenance + +## Honest status + +This is a serious **v0.3 kernel**, not a claim that it already exceeds OpenClaw or Hermes Agent in integrations, users, community, reliability history, or production maturity. + +The implementation has been validated with deterministic providers and fake Docker runners. The current build environment did not make a live GPT-5.6 Sol request and did not run the Evolve Docker executor against a live Docker daemon. Real-model variance, real rootless-Docker behavior, long-duration traffic, evaluator drift, and adversarial fixture quality still require qualification. + +v0.3 also does not provide: + +- hardware-attested evaluation or execution receipts +- a remote quorum for reports or the Episode ledger +- domain-level egress enforcement inside the runtime +- automatic Skill promotion +- causal proof that a Skill alone produced an observed improvement +- large-sample sequential testing or multiple-hypothesis correction +- distributed multi-agent lease coordination + +## Repository layout + +```text +src/runtime bounded loop, checkpoints, leases, evolution hooks +src/ledger hash-chained events and content-addressed evidence +src/policy risk decisions, approval, exact capabilities +src/execution Docker/local executor adapters and receipts +src/secrets short-lived file secret broker +src/tools workspace and process tools +src/providers GPT-5.6 Sol Responses API and deterministic mocks +src/verification deterministic and model-based final verification +src/memory evidence-aware durable memory +src/learning repeated-Episode pattern detection +src/skills candidate/quarantine/evaluation/canary/promotion/rollback state +src/evaluation fixtures, replay, metrics, signing, shadow, monitor, orchestration +src/context controlled prompt assembly +``` + +## Next milestones + +The next defensible work is **v0.3.1 Real Evaluation Qualification**, followed by **v0.4 Distributed Evidence Control Plane**: + +1. live GPT-5.6 Sol repeated-run variance matrix +2. rootless Docker and Docker Desktop integration tests +3. adversarial and mutation-generated replay fixtures +4. sequential tests, evaluator calibration, and multiple-comparison control +5. signed image and executor-policy provenance +6. remote or hardware-backed evaluation signing +7. lease-based multi-agent work graph +8. ACP/EDL Episode binding and quorum evidence receipts + +## License + +MIT diff --git a/evolve-agent/SECURITY.md b/evolve-agent/SECURITY.md new file mode 100644 index 0000000..25ba4e3 --- /dev/null +++ b/evolve-agent/SECURITY.md @@ -0,0 +1,95 @@ +# Security + +## Security model + +Evolve Agent treats model output as untrusted. Tool execution, evidence, memory, Skill activation, evaluation, and promotion are controlled outside the model. + +### Core invariants + +1. Tool calls pass schema validation, policy, approval, exact capability verification, execution, and evidence capture. +2. A capability binds the Episode, tool, normalized arguments, expiry, and HMAC signature. +3. File tools reject workspace traversal and symlink traversal. +4. Docker is the default executor; local execution is an explicit unsafe escape hatch. +5. Docker images require exact digests, allowlisting, and local presence; implicit pulls are disabled. +6. Network is denied by default; broad built-in network modes are rejected. +7. Container root, added capabilities, privilege escalation, writable root, and unconfined seccomp are not requested. +8. Secret values are delivered as short-lived read-only files, not command arguments or inherited environment values. +9. Evidence is content addressed and tied to the current Episode. +10. Episode events are append-only and predecessor-hash chained. +11. Concurrent ownership of one Episode is rejected by an exclusive lease. +12. Learned Skills remain inactive until signed offline and canary reports pass. +13. Candidate supporting Episodes are excluded from evaluation. +14. Shadow candidate output cannot replace the production answer. +15. `SkillStore` independently verifies signed report identity, authority, policy hash, and decision before promotion. +16. Production regression can automatically revoke a promoted Skill. + +## Evaluation authority + +Evaluation reports are signed with a local Ed25519 keypair under: + +```text +$EVOLVE_HOME/evaluations/authority/ +``` + +The private key is written with mode `0600`. Keep the entire state directory outside `EVOLVE_WORKSPACE`, outside task container mounts, and inaccessible to untrusted users. + +A valid signature establishes local integrity and authority continuity. It does **not** provide: + +- hardware attestation +- remote witness or quorum +- proof that the host was uncompromised +- proof that the verifier model was correct +- causal proof that a Skill alone created the measured effect + +Compromise of the host account or evaluation private key can authorize false reports. Rotate the state directory or keypair after suspected compromise and re-evaluate every affected Skill. + +## Replay fixture trust + +Fixtures are content addressed and validated, but fixture quality is still part of the trusted evaluation process. Review imported fixtures. Do not accept fixtures from an untrusted party merely because their internal hash is valid. + +The capture path rejects incomplete or failed Episodes and verifies: + +- source Episode identity +- ordered proposal trace +- evidence source Episode +- evidence tool identity +- executed-argument hash +- artifact hash format +- baseline committed state and score range + +A candidate's supporting Episodes are excluded from evaluation. Once evaluation begins, supporting provenance is frozen so later Episodes can remain independent holdout material. + +## Shadow and monitor boundary + +A shadow canary replays the candidate after a production result has been recorded. Candidate output is stored for evaluation only and never enters the production response path. + +Production monitoring compares observed outcomes with a signed canary envelope. Automatic rollback is a safety response, not proof that the Skill caused the regression. Shared model, prompt, tool, traffic, verifier, and environment changes can also move the metrics. + +## Docker boundary + +Docker substantially reduces authority but is not a perfect kernel-security boundary. The trusted computing base includes: + +- host kernel or Docker Desktop VM +- Docker daemon and client +- reviewed image contents +- operator-managed networks +- mounted workspace contents +- Evolve Agent policy and executor code + +Use rootless Docker when possible. For hostile multi-tenant workloads, prefer a microVM or restricted remote execution service. + +## Secret limitations + +Exact secret values are redacted from stdout and stderr before evidence storage. Encoded, hashed, transformed, fragmented, compressed, encrypted, or indirectly derived values may not be recognized. File-secret delivery intentionally lets the approved process read the secret. + +Do not expose Docker sockets, cloud metadata endpoints, host credential directories, or broad egress to untrusted workloads. + +## State isolation + +`EVOLVE_HOME` contains capability authority, evidence, checkpoints, memory, Skills, leases, replay fixtures, evaluation reports, and signing keys. Configuration fails if this directory is located inside `EVOLVE_WORKSPACE`. + +Protect backups of the state directory. A backup containing the signing key has the same authority as the live state. + +## Vulnerability reporting + +Do not open a public issue for a vulnerability that may expose credentials, bypass approval, escape execution isolation, forge evidence, forge an evaluation report, or promote a Skill without valid reports. Use GitHub private vulnerability reporting when enabled. diff --git a/evolve-agent/docs/ARCHITECTURE.md b/evolve-agent/docs/ARCHITECTURE.md new file mode 100644 index 0000000..efcbbe9 --- /dev/null +++ b/evolve-agent/docs/ARCHITECTURE.md @@ -0,0 +1,129 @@ +# Architecture + +Evolve Agent separates five loops that are often collapsed into one opaque agent process. + +## 1. Task loop + +```text +TaskSpec + -> Context Compiler + -> GPT-5.6 Sol decision + -> tool proposal or final proposal + -> evidence-aware verification + -> commit, retry, interruption, or budget stop +``` + +The loop is bounded by turns, tool calls, input tokens, output tokens, and wall time. Checkpoints preserve all consumed budget across resume. + +## 2. Authority and execution loop + +```text +model proposal + -> schema validation + -> task tool boundary + -> risk policy + -> human approval for protected actions + -> exact HMAC capability + -> Executor Registry + -> Docker sandbox by default + -> execution receipt + -> content-addressed evidence + -> hash-chained ledger +``` + +The model never receives direct execution authority. The local executor is disabled unless the operator explicitly enables an unsafe escape hatch. + +## 3. Learning loop + +```text +committed Episodes + -> repeated successful tool trace + -> candidate Skill + -> frozen supporting Episode/evidence provenance +``` + +A candidate remains inactive. Once its first offline report is attached, its training provenance no longer expands. Later matching Episodes can therefore become evaluation or canary data rather than silently contaminating the training set. + +## 4. Evaluation loop + +```text +clean committed Episode + -> content-addressed replay fixture + +candidate Skill + independent fixtures + -> baseline replay without candidate + -> candidate replay with candidate + -> paired metrics and bootstrap interval + -> policy gates + -> Ed25519-signed offline report + +production baseline Episodes + -> candidate counterfactual replay in shadow + -> paired non-regression gates + -> signed canary report + +signed offline + signed canary + -> SkillStore re-verification + -> explicit promotion +``` + +### Replay-world contract + +Baseline and candidate receive the same: + +- `TaskSpec` +- requested tool descriptions +- budgets +- ordered recorded observations +- evidence IDs and artifact provenance +- verifier provider + +They differ only in the active Skill set. A model-proposed tool name or proposal-argument hash that differs from the fixture causes a trace-mismatch failure. The original normalized executed-argument hash remains in the evidence record and is checked during fixture capture. + +### Report contract + +A signed report includes the complete run-level data rather than only aggregate scores. It binds Skill identity, fixture hashes, policy, results, decision gates, engine version, provider identity, payload hash, signature, and public-key fingerprint. + +## 5. Production control loop + +```text +promoted Skill used by Episode + -> terminal production outcome + -> rolling window + -> compare with signed canary candidate envelope + -> signed monitor report + -> remain promoted or automatic rollback +``` + +The monitor is deliberately asymmetric: it may revoke authority automatically, but it never promotes authority automatically. + +## State boundaries + +```text +Task workspace + mounted into executor + may be read-only or explicitly read-write + +Authority state + capability key + ledger and evidence + checkpoints and memory + Skills and training provenance + evaluation fixtures and reports + Ed25519 evaluation key + leases and secret materialization + never mounted as the task workspace +``` + +## Main invariants + +1. No protected tool executes without exact approval and a valid capability. +2. No final factual claim may cite evidence absent from the current Episode. +3. No clean fixture exists without source checkpoint, ledger, and artifact agreement. +4. No candidate is evaluated on its supporting Episodes. +5. No offline comparison is accepted without complete paired runs. +6. No promotion succeeds without two verified passing reports from one authority. +7. No shadow candidate output reaches the production response path. +8. No production regression keeps authority merely because the Skill was previously promoted. +9. No process execution silently falls back from Docker to the host. +10. No concurrent process owns the same Episode lease. diff --git a/evolve-agent/docs/EVALUATION_DRIVEN_EVOLUTION.md b/evolve-agent/docs/EVALUATION_DRIVEN_EVOLUTION.md new file mode 100644 index 0000000..2758dec --- /dev/null +++ b/evolve-agent/docs/EVALUATION_DRIVEN_EVOLUTION.md @@ -0,0 +1,158 @@ +# Evaluation-Driven Evolution + +## Purpose + +The v0.3 evaluation system answers a narrow question: + +> Does activating this exact Skill improve outcomes on independent tasks without introducing safety, quality, or cost regressions? + +It does not treat model confidence, one successful Episode, or human-entered scores as sufficient evidence. + +## Data lifecycle + +### Training provenance + +The repeated-pattern learner records the Episodes and evidence that created a candidate. These IDs form the candidate's training provenance. The first attached offline report freezes that set. + +### Replay fixtures + +A fixture is an immutable snapshot of a clean committed Episode. The fixture ID is derived from the SHA-256 hash of its normalized payload. + +Default split assignment is deterministic from the source Episode ID: + +- buckets 0–5: `train` +- buckets 6–7: `validation` +- buckets 8–9: `holdout` + +Operators may choose a split explicitly during reviewed capture. Offline evaluation defaults to validation and holdout fixtures and always excludes supporting Episode IDs regardless of split labels. + +### Why traces are replayed instead of tools re-executed + +The harness reuses recorded tool observations and evidence rather than re-running external side effects. This gives both arms the same world and avoids duplicating writes, payments, messages, deployments, or unstable network reads. + +The model must still propose the recorded tool sequence and raw proposal arguments. A mismatch fails closed. When a step matches, the harness reveals the recorded observation. This tests whether the Skill improves planning and answer construction under a fixed environment. + +This is not a substitute for separate live integration tests. It intentionally measures policy behavior under controlled replay. + +## Paired experiment + +For each fixture and repeat: + +```text +baseline arm = currently promoted matching Skills, candidate excluded +candidate arm = same baseline Skills plus candidate +``` + +Execution order alternates by fixture and repeat. Pair keys are `fixture_id:repeat`. + +Both arms use the same provider and verifier. For stochastic model behavior, increase `EVOLVE_EVAL_REPEATS`; one repeat is the safe low-cost default, not a claim of statistical sufficiency. + +## Metrics + +Each run records: + +- committed success +- final verifier score +- input, output, and total tokens +- turns and tool calls +- duration +- exact trace match +- safety violations +- active Skill IDs +- failure reason + +Aggregate comparison includes: + +- success-rate delta +- verifier-score delta +- token, tool-call, and duration ratios +- wins, losses, and ties +- candidate failures where baseline succeeded +- deterministic paired bootstrap intervals + +The bootstrap seed is derived from the paired data and policy, making reports reproducible for identical inputs. + +## Default offline gates + +| Gate | Default | +|---|---:| +| unique fixtures | at least 3 | +| repeats | 1 | +| new candidate failures | 0 | +| success regression | 0 | +| score regression | at most 0.02 | +| token ratio | at most 1.20 | +| tool-call ratio | at most 1.20 | +| confidence | 0.90 | +| improvement | success +0.05, score +0.02, or efficiency ratio ≤0.90 | + +The candidate must pass all non-regression gates and at least one improvement route. + +## Shadow canary + +A shadow observation combines: + +- the actual production result for an Episode +- one counterfactual replay of the opposite arm + +For `production-baseline`, production did not use the candidate and the candidate is replayed. For `production-candidate`, production used the candidate and the baseline is replayed. + +Pre-promotion canaries use `production-baseline`. Candidate output is never delivered to the user. Observations are deduplicated by Skill, Episode, and mode. + +After the configured minimum sample count, a canary report uses non-regression gates. It does not need to re-prove improvement because the signed offline report already carries that burden. + +## Signed promotion authorization + +The local Ed25519 authority signs every report. Promotion authorization includes: + +- offline report ID +- canary report ID +- combined policy hash +- authority key fingerprint + +Before changing state to `promoted`, `SkillStore` asks the report store to verify: + +1. both signatures +2. report IDs and payload hashes +3. report kinds +4. passing decisions +5. Skill ID and fingerprint +6. authority fingerprint +7. combined policy hash + +This prevents a caller from bypassing the evaluation engine by invoking the store directly with invented report IDs. + +## Production rollback + +A promoted Skill's terminal outcomes are stored in a bounded rolling window. The monitor compares observed success, verifier score, and token use with the candidate aggregate in the signed canary report. + +A failed gate produces a signed monitor report and immediately records an automatic rollback. The rollback is conservative: it can react to correlated system changes that are not caused solely by the Skill. This is intentional because revocation is safer than preserving suspect authority. + +## Known statistical limitations + +v0.3 uses deterministic paired bootstrap intervals, but it does not yet implement: + +- sequential probability ratio tests +- multiple-hypothesis correction across many Skills +- hierarchical modeling across task families +- evaluator calibration drift correction +- minimum detectable effect planning +- treatment of non-independent repeated Episodes +- causal isolation from model, prompt, tool, or traffic changes + +Use larger, adversarial, and independently reviewed fixture sets for consequential promotion decisions. + +## Operational checklist + +Before promotion: + +1. verify the candidate training provenance +2. review fixture sources and split assignment +3. run multiple repeats for stochastic models +4. inspect run-level failures, not just averages +5. verify the offline report signature +6. collect independent production-baseline shadow samples +7. verify the canary report signature +8. confirm policy thresholds match the risk level +9. promote explicitly +10. watch the production monitor and retain a manual kill path diff --git a/evolve-agent/docs/HARDENED_EXECUTION.md b/evolve-agent/docs/HARDENED_EXECUTION.md new file mode 100644 index 0000000..18c33a4 --- /dev/null +++ b/evolve-agent/docs/HARDENED_EXECUTION.md @@ -0,0 +1,137 @@ +# Hardened Execution Design + +## Goals + +The v0.2 executor is designed to make these statements true and testable: + +- a model cannot silently upgrade from read-only inspection to host execution +- an approval for one command cannot authorize changed arguments +- a task cannot choose an arbitrary image or network +- a process cannot inherit the agent's API keys +- a normal task has no network, immutable root filesystem, and bounded resources +- the evidence record identifies the exact execution policy +- a crashed task does not leave an Episode permanently locked + +## Executor contract + +`ExecutionRequest` contains all authority-relevant fields: + +```text +run ID +command + argument array +canonical workspace + working directory +workspace read/write mode +network policy name +immutable image reference +secret names +memory / CPU / PID / tmpfs ceilings +timeout and evidence-output ceiling +``` + +`ExecutionResult` contains: + +```text +exit state and duration +bounded, redacted stdout/stderr +executor and immutable image +network and workspace mode +secret names, never values +isolation claims +command hash and policy hash +sandbox identifier +``` + +The request is already covered by the HMAC capability and the artifact store's argument hash. The execution receipt adds a compact policy identity for comparison, replay, and future attestation. + +## Docker argument profile + +The executor builds an argument array and invokes Docker without a shell. Its core profile is: + +```text +docker run --rm --init --pull never + --read-only + --cap-drop ALL + --security-opt no-new-privileges:true + --network none + --user + --ipc none + --memory --memory-swap + --cpus + --pids-limit + --tmpfs /tmp:rw,nosuid,nodev,size= + --tmpfs /run:rw,nosuid,nodev,noexec,size=16m + --ulimit nofile=1024:1024 + --ulimit core=0:0 + --log-driver none + --mount type=bind,src=,dst=/workspace,readonly + + +``` + +The runtime does not pass `--privileged`, host PID/IPC/network namespaces, devices, Docker socket, added capabilities, or an unconfined seccomp profile. + +## Image policy + +An accepted reference has the form: + +```text +repository/name[:tag]@sha256:<64 lowercase hexadecimal characters> +``` + +or a raw local image digest. Tags alone are rejected. The exact value must appear in `EVOLVE_DOCKER_ALLOWED_IMAGES`. `--pull never` makes missing content an error. + +This establishes identity, not safety. The operator still owns image review, provenance, and patch policy. + +## Network policy + +`none` is always available and maps to `deny-all` in the receipt. Built-in broad networks are rejected. + +A custom allowlisted network is mapped to `operator-managed:`. The name is not treated as proof of domain-level filtering. External infrastructure must enforce the actual routes. + +## Secret broker + +The broker separates authorization, materialization, delivery, evidence, and cleanup: + +1. validate name syntax +2. require operator allowlist membership +3. fetch value from the host source +4. reject missing, empty, or oversized values +5. write a private per-run directory and `0600` file +6. mount the file read-only +7. expose only `_FILE` +8. redact exact values from process output +9. delete files in `finally` +10. sweep expired directories after interrupted cleanup + +No secret value appears in the Docker command line, executor receipt, or tool arguments. + +## Output and timeout handling + +The host runner captures stdout and stderr under one byte ceiling. When the ceiling is crossed, it terminates the entire detached process group, escalates to `SIGKILL`, marks output truncated, and treats the execution as unsuccessful. + +A timeout follows the same group-termination path. Docker execution then performs `docker rm -f ` as best-effort cleanup, covering the case where killing the Docker client leaves a container behind. + +## Episode leases + +A lease file is created with exclusive `open(..., "wx")`. The owning process keeps the file handle open and updates its timestamps on a heartbeat. + +Recovery requires: + +- heartbeat older than TTL, and +- no live recorded PID on the same hostname + +The stale path is atomically renamed before a new lease is created. The old owner cannot delete the replacement because release verifies owner ID. + +## Local executor + +Local execution exists only as a migration escape hatch. It deliberately refuses claims it cannot enforce: + +- `network=none` is rejected; the caller must request `host` +- `workspace_access=read-only` is rejected; the caller must request `read-write` +- secrets are rejected + +Its evidence reports `boundary: none`. + +## Future adapters + +The interface is designed to support Firecracker, Vercel Sandbox, Daytona, or a remote execution service without changing the model-facing tool contract. An adapter must provide equivalent receipts and must not weaken the policy silently. diff --git a/evolve-agent/docs/ROADMAP.md b/evolve-agent/docs/ROADMAP.md new file mode 100644 index 0000000..bb16cb9 --- /dev/null +++ b/evolve-agent/docs/ROADMAP.md @@ -0,0 +1,72 @@ +# Roadmap + +## v0.1 — Evidence-Gated Kernel — complete + +- bounded autonomous loop +- exact approval capabilities +- content-addressed evidence +- hash-chained Episode ledger +- independent final verifier +- evidence-aware memory +- inactive learned Skill candidates + +## v0.2 — Hardened Execution — complete + +- Docker-first executor abstraction +- pinned image and deny-by-default network policy +- non-root, read-only, capability-dropped containers +- resource, timeout, and output limits +- short-lived file secret broker +- execution receipts +- Episode leases and stale-lock recovery + +## v0.3 — Evaluation-Driven Evolution — complete in this branch + +- immutable replay fixtures +- train/evaluation leakage guard +- paired baseline-versus-candidate replay +- quality, safety, cost, and confidence gates +- signed offline and canary reports +- shadow canaries with no production intervention +- explicit report-backed promotion +- rolling production monitor +- automatic rollback + +## v0.3.1 — Real Evaluation Qualification + +- live GPT-5.6 Sol repeated-run variance matrix +- rootless Docker integration tests on Linux +- Docker Desktop integration tests on macOS +- real image digest, network-none, timeout, and orphan-cleanup tests +- adversarial fixture corpus +- mutation-generated trace and evidence attacks +- evaluator consistency and calibration dashboard + +## v0.3.2 — Statistical Hardening + +- sequential testing +- multiple-comparison control +- minimum detectable effect planning +- stratified task-family reports +- evaluator drift alarms +- holdout rotation and fixture expiration +- provenance for dataset review and approvals + +## v0.4 — Distributed Evidence Control Plane + +- signed executor-policy and image-provenance receipts +- remote or hardware-backed report authority +- append-only remote witness for ledger heads +- lease-based multi-agent work graph +- quorum evidence receipts +- ACP/EDL off-chain Episode binding +- signed private Skill registry + +## v0.5 — Production Agent Platform + +- channel adapters separated from the kernel +- multi-tenant authority isolation +- remote microVM executors +- policy-as-code distribution +- cost attribution and fleet observability +- staged Skill rollout across worker cohorts diff --git a/evolve-agent/docs/THREAT_MODEL.md b/evolve-agent/docs/THREAT_MODEL.md new file mode 100644 index 0000000..ef3ede7 --- /dev/null +++ b/evolve-agent/docs/THREAT_MODEL.md @@ -0,0 +1,58 @@ +# Threat Model + +## Protected assets + +- workspace files and side effects +- host credentials and brokered secrets +- tool and executor authority +- Episode integrity and evidence provenance +- memory integrity +- Skill training provenance and activation state +- replay fixture integrity +- evaluation signing key +- offline, canary, and monitor reports +- human approval intent + +## Principal threats and controls + +| Threat | Primary control | Residual risk | +|---|---|---| +| Prompt injection requests a dangerous tool | model has no direct authority; policy, approval, capability, executor | operator may approve malicious intent | +| Arguments change after approval | HMAC binds exact normalized arguments | compromised host can replace authority code | +| Workspace path escape | canonical containment and symlink rejection | approved process may damage read-write workspace | +| Arbitrary image or implicit update | exact digest allowlist and `--pull never` | allowlisted image may itself be malicious | +| Broad network access | `network=none`; unsafe built-ins rejected | operator-managed custom network may be permissive | +| Container privilege escalation | non-root, read-only root, cap-drop ALL, no-new-privileges, seccomp | kernel or Docker vulnerability remains | +| Secret in command line or environment | short-lived read-only files and `_FILE` pointers | approved process can read the secret by design | +| Secret in evidence output | exact-value redaction before artifact storage | transformed or fragmented secret may evade detection | +| Fork bomb or output flood | cgroup/PID limits, timeout, output cap, process-group kill | host daemon pressure remains possible | +| Fabricated evidence | current-Episode evidence set and content-addressed artifacts | compromised host can rewrite state and authority together | +| Ledger editing | predecessor and event hashes | no remote witness or quorum yet | +| Duplicate Episode execution | exclusive lease and heartbeat | distributed filesystem semantics may differ | +| Unsafe stale-lock steal | TTL plus same-host live-PID check | PID reuse can delay recovery | +| Poisoned memory | evidence requirement and provenance | valid evidence can support a wrong inference | +| Training/evaluation leakage | supporting Episode exclusion and provenance freeze | semantically duplicated tasks may still leak | +| Fixture tampering | content hash, deterministic ID, structural and evidence validation | malicious but internally consistent fixture remains possible | +| Candidate changes replay side effects | recorded observations; no tool re-execution | replay does not test live integration behavior | +| Candidate wins by changing trace | exact proposal tool/argument hash matching | semantically equivalent alternative trace is scored as mismatch | +| Candidate invents evidence | evidence availability and citation contract | verifier can still misjudge answer quality | +| Weak average hides new failure | paired new-failure gate | small fixture set may miss rare failures | +| Report modified after evaluation | Ed25519 signature and payload hash | local signing key compromise defeats integrity | +| Fake report IDs passed directly to SkillStore | store-level signed-report verifier | malicious code with signing-key access can forge authority | +| Candidate output affects users during canary | shadow output discarded | counterfactual replay consumes model cost and may leak to provider logs according to provider policy | +| Promoted Skill regresses | rolling monitor and automatic rollback | correlation does not prove causation; detection waits for minimum samples | +| Evaluator drift | same verifier within paired report | drift across report generations remains | +| Automatic unsafe promotion | no automatic promotion path | operator can explicitly promote a poor but passing Skill | + +## Out of scope for v0.3 + +- protection from a hostile host administrator or Docker daemon operator +- formal verification of model, verifier, Docker, or kernel behavior +- hardware-backed signing or remote attestation +- remote consensus for evaluation reports or ledger heads +- domain-aware egress filtering implemented inside the runtime +- proof that fixture distribution matches future production traffic +- causal attribution of outcome changes to one Skill +- complete statistical treatment of repeated, adaptive, or multiple experiments +- automatic Skill promotion +- distributed multi-agent lease consensus diff --git a/evolve-agent/examples/basic.ts b/evolve-agent/examples/basic.ts new file mode 100644 index 0000000..47597ee --- /dev/null +++ b/evolve-agent/examples/basic.ts @@ -0,0 +1,9 @@ +import { createRuntime, loadConfig } from "@dclxai/evolve-agent"; + +const { runtime } = createRuntime(loadConfig({ workspace: process.cwd() })); +const result = await runtime.run({ + goal: "Read package.json and summarize the package scripts.", + requestedTools: ["read_file"], + successCriteria: ["Every factual claim cites current-episode evidence"], +}); +console.log(result); diff --git a/evolve-agent/examples/evaluation.ts b/evolve-agent/examples/evaluation.ts new file mode 100644 index 0000000..1e68999 --- /dev/null +++ b/evolve-agent/examples/evaluation.ts @@ -0,0 +1,28 @@ +import { createRuntime, loadConfig } from "@dclxai/evolve-agent"; + +const { fixtures, evaluations } = createRuntime( + loadConfig({ + workspace: process.cwd(), + evaluation: { + captureCommitted: false, + shadowPercent: 0, + policy: { minFixtures: 5, repeats: 2 }, + }, + }), +); + +// Capture reviewed clean Episodes before evaluating a Skill. +await fixtures.capture("ep_000000000000000000000001", "validation"); +await fixtures.capture("ep_000000000000000000000002", "holdout"); + +const report = await evaluations.evaluateSkill("skill_000000000000000000000001", { + splits: ["validation", "holdout"], + repeats: 2, +}); + +console.log({ + reportId: report.id, + passed: report.payload.decision.passed, + scoreDelta: report.payload.comparison.verifierScoreDelta, + tokenRatio: report.payload.comparison.totalTokenRatio, +}); diff --git a/evolve-agent/package.json b/evolve-agent/package.json new file mode 100644 index 0000000..2a94da0 --- /dev/null +++ b/evolve-agent/package.json @@ -0,0 +1,46 @@ +{ + "name": "@dclxai/evolve-agent", + "version": "0.3.0", + "description": "Evidence-gated GPT-5.6 Sol agent runtime with hardened execution and signed evaluation-driven Skill evolution", + "type": "module", + "bin": { + "evolve-agent": "./dist/cli.js" + }, + "exports": { + ".": "./dist/index.js" + }, + "files": [ + "dist", + "README.md", + "CHANGELOG.md", + "LICENSE", + "SECURITY.md", + "docs" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "dev": "npm run build --silent && node dist/cli.js", + "test": "rm -rf .test-dist && tsc -p tsconfig.test.json && node --test .test-dist/tests/*.test.js", + "typecheck": "tsc -p tsconfig.json --noEmit", + "check": "npm run typecheck && npm test && npm run build", + "start": "node dist/cli.js" + }, + "engines": { + "node": ">=22.6" + }, + "dependencies": {}, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.8.0" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/DCLXAI/AGENTR.git", + "directory": "evolve-agent" + }, + "bugs": { + "url": "https://github.com/DCLXAI/AGENTR/issues" + }, + "homepage": "https://github.com/DCLXAI/AGENTR/tree/main/evolve-agent#readme" +} diff --git a/evolve-agent/src/cli.ts b/evolve-agent/src/cli.ts new file mode 100644 index 0000000..e7a504d --- /dev/null +++ b/evolve-agent/src/cli.ts @@ -0,0 +1,325 @@ +#!/usr/bin/env node +import { readFile } from "node:fs/promises"; +import { loadConfig } from "./config.js"; +import { createRuntime } from "./factory.js"; +import type { FixtureSplit, ShadowObservation } from "./evaluation/types.js"; + +interface ParsedArgs { + positionals: string[]; + options: Map; + flags: Set; +} + +function parseArgs(args: string[]): ParsedArgs { + const positionals: string[] = []; + const options = new Map(); + const flags = new Set(); + for (let index = 0; index < args.length; index += 1) { + const token = args[index]; + if (!token) continue; + if (!token.startsWith("--")) { + positionals.push(token); + continue; + } + const key = token.slice(2); + const values: string[] = []; + while (index + 1 < args.length && !args[index + 1]?.startsWith("--")) { + values.push(args[index + 1] as string); + index += 1; + } + if (values.length === 0) flags.add(key); + else options.set(key, [...(options.get(key) ?? []), ...values]); + } + return { positionals, options, flags }; +} + +function value(parsed: ParsedArgs, key: string): string | undefined { + return parsed.options.get(key)?.[0]; +} + +function values(parsed: ParsedArgs, key: string): string[] { + return parsed.options.get(key) ?? []; +} + +function numeric(parsed: ParsedArgs, key: string): number | undefined { + const raw = value(parsed, key); + if (raw === undefined) return undefined; + const parsedNumber = Number(raw); + if (!Number.isFinite(parsedNumber) || parsedNumber <= 0) throw new Error(`--${key} must be a positive number`); + return parsedNumber; +} + +function fixtureSplits(parsed: ParsedArgs): FixtureSplit[] | undefined { + const requested = values(parsed, "split"); + if (requested.length === 0) return undefined; + for (const split of requested) { + if (split !== "train" && split !== "validation" && split !== "holdout") { + throw new Error(`Unknown fixture split: ${split}`); + } + } + return requested as FixtureSplit[]; +} + +function help(): void { + console.log(`Evolve Agent 0.3.0 — Evaluation-Driven Evolution + +Usage: + evolve-agent run [--workspace path] [--tool name ...] [--constraint text ...] [--success text ...] + evolve-agent resume [--workspace path] [--home path] + evolve-agent doctor [--executor docker|local] [--allow-local-executor] + evolve-agent executors list + evolve-agent secrets sweep + evolve-agent ledger verify + +Evaluation fixtures: + evolve-agent evaluations fixtures capture [--split train|validation|holdout] + evolve-agent evaluations fixtures import + evolve-agent evaluations fixtures list [--split validation --split holdout] + +Counterfactual evaluation: + evolve-agent evaluations run [--fixture id ...] [--split validation --split holdout] [--repeats N] + evolve-agent evaluations shadow [--mode production-baseline|production-candidate] + evolve-agent evaluations canary + evolve-agent evaluations monitor + evolve-agent evaluations reports list + evolve-agent evaluations reports show + evolve-agent evaluations verify + +Skill lifecycle: + evolve-agent skills list + evolve-agent skills evaluate [evaluation selection options] + evolve-agent skills promote + evolve-agent skills rollback --note reason + +Evolution invariants: + - supporting Episodes are excluded from evaluation fixtures + - baseline and candidate receive the same replay trace, tools, budgets, and verifier + - signed offline and shadow-canary reports are required for explicit promotion + - candidate Skills never alter production answers during shadow evaluation + - promoted Skills are automatically rolled back when the production window breaches the signed canary envelope + - Docker remains the default fail-closed execution boundary from v0.2`); +} + +async function main(): Promise { + const parsed = parseArgs(process.argv.slice(2)); + const [command, subcommand, third, fourth] = parsed.positionals; + if (!command || command === "help" || parsed.flags.has("help")) { + help(); + return; + } + + const configuredExecutor = value(parsed, "executor"); + const config = loadConfig({ + ...(value(parsed, "workspace") !== undefined ? { workspace: value(parsed, "workspace") as string } : {}), + ...(value(parsed, "home") !== undefined ? { home: value(parsed, "home") as string } : {}), + ...(value(parsed, "model") !== undefined ? { model: value(parsed, "model") as string } : {}), + ...(configuredExecutor !== undefined ? { defaultExecutor: configuredExecutor as "docker" | "local" } : {}), + ...(parsed.flags.has("allow-local-executor") ? { allowLocalExecutor: true } : {}), + ...(parsed.flags.has("non-interactive") ? { nonInteractive: true } : {}), + }); + const bundle = createRuntime(config); + + if (command === "doctor") { + const expiredSecretsRemoved = await bundle.secrets.sweepExpired(); + const probes = await bundle.executors.probeAll(); + const defaultProbe = probes.find((probe) => probe.kind === bundle.executors.getDefaultKind()); + const providerReady = Boolean(config.openAiApiKey); + const hardenedExecutionReady = config.defaultExecutor === "docker" && Boolean(defaultProbe?.ready); + const ok = providerReady && hardenedExecutionReady; + console.log( + JSON.stringify( + { + ok, + version: "0.3.0", + model: config.model, + verifier_model: config.verifierModel, + workspace: config.workspace, + home: config.home, + api_key_configured: providerReady, + default_executor: bundle.executors.getDefaultKind(), + hardened_execution_ready: hardenedExecutionReady, + local_executor_enabled: bundle.executors.list().includes("local"), + docker: { + default_image: config.docker.defaultImage ?? null, + allowed_images: [...config.docker.allowedImages].sort(), + allowed_networks: [...config.docker.allowedNetworks].sort(), + non_root_user: config.docker.user, + require_rootless: config.docker.requireRootless, + maximums: config.docker.maximums, + }, + evaluation: { + capture_committed: config.evaluation.captureCommitted, + shadow_percent: config.evaluation.shadowPercent, + monitor_promoted: config.evaluation.monitorPromoted, + canary_min_samples: config.evaluation.canaryMinSamples, + monitor_min_samples: config.evaluation.monitorMinSamples, + monitor_window: config.evaluation.monitorWindow, + policy: config.evaluation.policy, + fixtures: (await bundle.fixtures.list()).length, + reports: (await bundle.reports.list()).length, + }, + secret_allowlist: bundle.secrets.allowedNames(), + expired_secret_leases_removed: expiredSecretsRemoved, + episode_lease: { + ttl_ms: config.leaseTtlMs, + heartbeat_ms: config.leaseHeartbeatMs, + }, + executors: probes, + tools: bundle.tools.modelDescriptions(), + }, + null, + 2, + ), + ); + process.exitCode = ok ? 0 : 1; + return; + } + + if (command === "executors" && subcommand === "list") { + console.log(JSON.stringify({ default: bundle.executors.getDefaultKind(), probes: await bundle.executors.probeAll() }, null, 2)); + return; + } + + if (command === "secrets" && subcommand === "sweep") { + console.log(JSON.stringify({ removed: await bundle.secrets.sweepExpired() }, null, 2)); + return; + } + + if (command === "run") { + const goal = subcommand; + if (!goal) throw new Error("run requires a goal. Quote multi-word goals."); + const selectedTools = values(parsed, "tool"); + const result = await bundle.runtime.run({ + goal, + constraints: values(parsed, "constraint"), + successCriteria: values(parsed, "success"), + ...(selectedTools.length > 0 ? { requestedTools: selectedTools } : {}), + budget: { + ...(numeric(parsed, "max-turns") !== undefined ? { maxTurns: numeric(parsed, "max-turns") as number } : {}), + ...(numeric(parsed, "max-tool-calls") !== undefined + ? { maxToolCalls: numeric(parsed, "max-tool-calls") as number } + : {}), + ...(numeric(parsed, "max-wall-ms") !== undefined ? { maxWallTimeMs: numeric(parsed, "max-wall-ms") as number } : {}), + }, + }); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.status === "committed" ? 0 : 1; + return; + } + + if (command === "resume" && subcommand) { + const result = await bundle.runtime.resume(subcommand); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.status === "committed" ? 0 : 1; + return; + } + + if (command === "ledger" && subcommand === "verify") { + const result = await bundle.ledger.verify(); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.valid ? 0 : 1; + return; + } + + if (command === "evaluations" && subcommand === "fixtures" && third === "capture" && fourth) { + const split = fixtureSplits(parsed)?.[0]; + console.log(JSON.stringify(await bundle.fixtures.capture(fourth, split), null, 2)); + return; + } + + if (command === "evaluations" && subcommand === "fixtures" && third === "import" && fourth) { + const fixture = JSON.parse(await readFile(fourth, "utf8")) as unknown; + console.log(JSON.stringify(await bundle.fixtures.import(fixture), null, 2)); + return; + } + + if (command === "evaluations" && subcommand === "fixtures" && third === "list") { + const splits = fixtureSplits(parsed); + console.log(JSON.stringify(await bundle.fixtures.list(splits ? new Set(splits) : undefined), null, 2)); + return; + } + + if ((command === "evaluations" && subcommand === "run" && third) || (command === "skills" && subcommand === "evaluate" && third)) { + const skillId = third as string; + const report = await bundle.evaluations.evaluateSkill(skillId, { + ...(values(parsed, "fixture").length > 0 ? { fixtures: values(parsed, "fixture") } : {}), + ...(fixtureSplits(parsed) ? { splits: fixtureSplits(parsed) as FixtureSplit[] } : {}), + ...(numeric(parsed, "repeats") !== undefined ? { repeats: Math.floor(numeric(parsed, "repeats") as number) } : {}), + }); + console.log(JSON.stringify(report, null, 2)); + process.exitCode = report.payload.decision.passed ? 0 : 1; + return; + } + + if (command === "evaluations" && subcommand === "shadow" && third && fourth) { + const mode = value(parsed, "mode"); + if (mode !== undefined && mode !== "production-baseline" && mode !== "production-candidate") { + throw new Error(`Invalid shadow mode: ${mode}`); + } + console.log( + JSON.stringify( + await bundle.evaluations.shadowEpisode( + third, + fourth, + mode as ShadowObservation["mode"] | undefined, + ), + null, + 2, + ), + ); + return; + } + + if (command === "evaluations" && subcommand === "canary" && third) { + const report = await bundle.evaluations.finalizeCanary(third); + console.log(JSON.stringify(report, null, 2)); + process.exitCode = report.payload.decision.passed ? 0 : 1; + return; + } + + if (command === "evaluations" && subcommand === "monitor" && third) { + console.log(JSON.stringify((await bundle.evaluations.monitorSkill(third)) ?? { status: "insufficient_samples" }, null, 2)); + return; + } + + if (command === "evaluations" && subcommand === "reports" && third === "list") { + console.log(JSON.stringify(await bundle.reports.list(), null, 2)); + return; + } + + if (command === "evaluations" && subcommand === "reports" && third === "show" && fourth) { + console.log(JSON.stringify(await bundle.reports.get(fourth), null, 2)); + return; + } + + if (command === "evaluations" && subcommand === "verify" && third) { + const valid = await bundle.evaluations.verifyReport(third); + console.log(JSON.stringify({ report_id: third, valid }, null, 2)); + process.exitCode = valid ? 0 : 1; + return; + } + + if (command === "skills" && subcommand === "list") { + console.log(JSON.stringify(await bundle.skills.list(), null, 2)); + return; + } + + if (command === "skills" && subcommand === "promote" && third) { + console.log(JSON.stringify(await bundle.evaluations.promoteSkill(third), null, 2)); + return; + } + + if (command === "skills" && subcommand === "rollback" && third) { + const note = value(parsed, "note"); + if (!note) throw new Error("rollback requires --note"); + console.log(JSON.stringify(await bundle.skills.rollback(third, note), null, 2)); + return; + } + + throw new Error("Unknown command. Run `evolve-agent help`."); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack ?? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/evolve-agent/src/config.ts b/evolve-agent/src/config.ts new file mode 100644 index 0000000..3366180 --- /dev/null +++ b/evolve-agent/src/config.ts @@ -0,0 +1,401 @@ +import { existsSync, realpathSync } from "node:fs"; +import { homedir } from "node:os"; +import path from "node:path"; +import { sha256Bytes } from "./core/hash.js"; +import type { ExecutorKind, ResourceLimits } from "./execution/types.js"; +import { defaultEvaluationPolicy } from "./evaluation/metrics.js"; +import type { EvaluationPolicy } from "./evaluation/types.js"; + +export type ReasoningEffort = "none" | "low" | "medium" | "high" | "xhigh" | "max"; + +function defaultStateHome(workspace: string): string { + const stateRoot = process.env.XDG_STATE_HOME ?? path.join(homedir(), ".local", "state"); + return path.join(stateRoot, "evolve-agent", sha256Bytes(workspace).slice(0, 16)); +} + +function canonicalCandidate(target: string): string { + const missing: string[] = []; + let current = path.resolve(target); + while (!existsSync(current)) { + const parent = path.dirname(current); + if (parent === current) break; + missing.unshift(path.basename(current)); + current = parent; + } + const canonicalParent = realpathSync(current); + return path.resolve(canonicalParent, ...missing); +} + +function isInside(parent: string, candidate: string): boolean { + const relative = path.relative(parent, candidate); + return relative === "" || (!relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative)); +} + +export interface DockerConfig { + binary: string; + defaultImage?: string; + allowedImages: Set; + allowedNetworks: Set; + user: string; + maximums: ResourceLimits; + requireRootless: boolean; +} + +export interface EvaluationConfig { + captureCommitted: boolean; + shadowPercent: number; + monitorPromoted: boolean; + canaryMinSamples: number; + monitorMinSamples: number; + monitorWindow: number; + monitorMaxSuccessDrop: number; + monitorMaxScoreDrop: number; + monitorMaxTokenRatio: number; + policy: EvaluationPolicy; +} + +export interface EvolveConfig { + home: string; + workspace: string; + model: string; + verifierModel: string; + reasoningEffort: ReasoningEffort; + allowedCommands: Set; + nonInteractive: boolean; + openAiApiKey?: string; + defaultExecutor: ExecutorKind; + allowLocalExecutor: boolean; + docker: DockerConfig; + secretAllowlist: Set; + secretTtlMs: number; + leaseTtlMs: number; + leaseHeartbeatMs: number; + evaluation: EvaluationConfig; +} + +function booleanValue(value: boolean | undefined, environment: string | undefined, fallback: boolean): boolean { + if (value !== undefined) return value; + if (environment === undefined) return fallback; + if (environment === "true") return true; + if (environment === "false") return false; + throw new Error(`Expected true or false, received ${environment}`); +} + +function nonnegativeValue(value: number | undefined, environment: string | undefined, fallback: number, label: string): number { + const resolved = value ?? (environment === undefined ? fallback : Number(environment)); + if (!Number.isFinite(resolved) || resolved < 0) throw new Error(`${label} must be a non-negative number`); + return resolved; +} + +function boundedValue( + value: number | undefined, + environment: string | undefined, + fallback: number, + label: string, + minimum: number, + maximum: number, +): number { + const resolved = value ?? (environment === undefined ? fallback : Number(environment)); + if (!Number.isFinite(resolved) || resolved < minimum || resolved > maximum) { + throw new Error(`${label} must be between ${minimum} and ${maximum}`); + } + return resolved; +} + +function numberValue(value: number | undefined, environment: string | undefined, fallback: number, label: string): number { + const resolved = value ?? (environment === undefined ? fallback : Number(environment)); + if (!Number.isFinite(resolved) || resolved <= 0) throw new Error(`${label} must be a positive number`); + return resolved; +} + +function values(source: Iterable | undefined, environment: string | undefined): Set { + const entries = source ?? (environment ?? "").split(","); + return new Set([...entries].map((value) => value.trim()).filter(Boolean)); +} + +export type ConfigOverrides = Partial< + Omit +> & { + allowedCommands?: Iterable; + secretAllowlist?: Iterable; + evaluation?: Partial> & { policy?: Partial }; + docker?: Partial> & { + allowedImages?: Iterable; + allowedNetworks?: Iterable; + maximums?: Partial; + }; +}; + +export function loadConfig(overrides: ConfigOverrides = {}): EvolveConfig { + const effort = overrides.reasoningEffort ?? process.env.EVOLVE_REASONING_EFFORT ?? "high"; + if (!["none", "low", "medium", "high", "xhigh", "max"].includes(effort)) { + throw new Error(`Invalid reasoning effort: ${effort}`); + } + + const commands = + overrides.allowedCommands ?? + (process.env.EVOLVE_ALLOWED_COMMANDS ?? "git,node,npm,npx,pnpm,python,python3,pytest,vitest,tsc").split(","); + const defaultExecutor = overrides.defaultExecutor ?? process.env.EVOLVE_EXECUTOR ?? "docker"; + if (defaultExecutor !== "docker" && defaultExecutor !== "local") { + throw new Error(`Invalid executor: ${defaultExecutor}`); + } + const allowLocalExecutor = booleanValue( + overrides.allowLocalExecutor, + process.env.EVOLVE_ALLOW_LOCAL_EXECUTOR, + false, + ); + if (defaultExecutor === "local" && !allowLocalExecutor) { + throw new Error("EVOLVE_EXECUTOR=local requires EVOLVE_ALLOW_LOCAL_EXECUTOR=true"); + } + + const dockerOverride = overrides.docker ?? {}; + const defaultImage = dockerOverride.defaultImage ?? process.env.EVOLVE_DOCKER_DEFAULT_IMAGE; + const allowedImages = values(dockerOverride.allowedImages, process.env.EVOLVE_DOCKER_ALLOWED_IMAGES); + if (defaultImage) allowedImages.add(defaultImage); + const allowedNetworks = values(dockerOverride.allowedNetworks, process.env.EVOLVE_DOCKER_ALLOWED_NETWORKS); + allowedNetworks.add("none"); + + const maximums: ResourceLimits = { + memoryMb: numberValue( + dockerOverride.maximums?.memoryMb, + process.env.EVOLVE_DOCKER_MAX_MEMORY_MB, + 2_048, + "EVOLVE_DOCKER_MAX_MEMORY_MB", + ), + cpus: numberValue( + dockerOverride.maximums?.cpus, + process.env.EVOLVE_DOCKER_MAX_CPUS, + 2, + "EVOLVE_DOCKER_MAX_CPUS", + ), + pids: Math.floor( + numberValue( + dockerOverride.maximums?.pids, + process.env.EVOLVE_DOCKER_MAX_PIDS, + 256, + "EVOLVE_DOCKER_MAX_PIDS", + ), + ), + tmpfsMb: Math.floor( + numberValue( + dockerOverride.maximums?.tmpfsMb, + process.env.EVOLVE_DOCKER_MAX_TMPFS_MB, + 256, + "EVOLVE_DOCKER_MAX_TMPFS_MB", + ), + ), + }; + + const leaseTtlMs = numberValue(overrides.leaseTtlMs, process.env.EVOLVE_LEASE_TTL_MS, 30_000, "EVOLVE_LEASE_TTL_MS"); + const leaseHeartbeatMs = numberValue( + overrides.leaseHeartbeatMs, + process.env.EVOLVE_LEASE_HEARTBEAT_MS, + 10_000, + "EVOLVE_LEASE_HEARTBEAT_MS", + ); + if (leaseHeartbeatMs * 2 >= leaseTtlMs) { + throw new Error("EVOLVE_LEASE_HEARTBEAT_MS must be less than half of EVOLVE_LEASE_TTL_MS"); + } + + const workspace = canonicalCandidate(overrides.workspace ?? process.env.EVOLVE_WORKSPACE ?? "."); + const home = canonicalCandidate(overrides.home ?? process.env.EVOLVE_HOME ?? defaultStateHome(workspace)); + if (isInside(workspace, home)) { + throw new Error("EVOLVE_HOME must be outside EVOLVE_WORKSPACE so sandboxed code cannot read agent authority or secret state"); + } + + const evaluationOverride = overrides.evaluation ?? {}; + const evaluationPolicy = defaultEvaluationPolicy({ + ...evaluationOverride.policy, + minFixtures: Math.floor( + numberValue( + evaluationOverride.policy?.minFixtures, + process.env.EVOLVE_EVAL_MIN_FIXTURES, + 3, + "EVOLVE_EVAL_MIN_FIXTURES", + ), + ), + repeats: Math.floor( + numberValue(evaluationOverride.policy?.repeats, process.env.EVOLVE_EVAL_REPEATS, 1, "EVOLVE_EVAL_REPEATS"), + ), + maxNewFailures: Math.floor( + nonnegativeValue( + evaluationOverride.policy?.maxNewFailures, + process.env.EVOLVE_EVAL_MAX_NEW_FAILURES, + 0, + "EVOLVE_EVAL_MAX_NEW_FAILURES", + ), + ), + maxSuccessRegression: boundedValue( + evaluationOverride.policy?.maxSuccessRegression, + process.env.EVOLVE_EVAL_MAX_SUCCESS_REGRESSION, + 0, + "EVOLVE_EVAL_MAX_SUCCESS_REGRESSION", + 0, + 1, + ), + maxScoreRegression: boundedValue( + evaluationOverride.policy?.maxScoreRegression, + process.env.EVOLVE_EVAL_MAX_SCORE_REGRESSION, + 0.02, + "EVOLVE_EVAL_MAX_SCORE_REGRESSION", + 0, + 1, + ), + minSuccessImprovement: boundedValue( + evaluationOverride.policy?.minSuccessImprovement, + process.env.EVOLVE_EVAL_MIN_SUCCESS_IMPROVEMENT, + 0.05, + "EVOLVE_EVAL_MIN_SUCCESS_IMPROVEMENT", + 0, + 1, + ), + minScoreImprovement: boundedValue( + evaluationOverride.policy?.minScoreImprovement, + process.env.EVOLVE_EVAL_MIN_SCORE_IMPROVEMENT, + 0.02, + "EVOLVE_EVAL_MIN_SCORE_IMPROVEMENT", + 0, + 1, + ), + maxTokenRegressionRatio: numberValue( + evaluationOverride.policy?.maxTokenRegressionRatio, + process.env.EVOLVE_EVAL_MAX_TOKEN_RATIO, + 1.2, + "EVOLVE_EVAL_MAX_TOKEN_RATIO", + ), + maxToolCallRegressionRatio: numberValue( + evaluationOverride.policy?.maxToolCallRegressionRatio, + process.env.EVOLVE_EVAL_MAX_TOOL_RATIO, + 1.2, + "EVOLVE_EVAL_MAX_TOOL_RATIO", + ), + efficiencyImprovementRatio: boundedValue( + evaluationOverride.policy?.efficiencyImprovementRatio, + process.env.EVOLVE_EVAL_EFFICIENCY_RATIO, + 0.9, + "EVOLVE_EVAL_EFFICIENCY_RATIO", + 0.01, + 1, + ), + confidenceLevel: boundedValue( + evaluationOverride.policy?.confidenceLevel, + process.env.EVOLVE_EVAL_CONFIDENCE, + 0.9, + "EVOLVE_EVAL_CONFIDENCE", + 0.5, + 0.999, + ), + bootstrapSamples: Math.floor( + numberValue( + evaluationOverride.policy?.bootstrapSamples, + process.env.EVOLVE_EVAL_BOOTSTRAP_SAMPLES, + 1_000, + "EVOLVE_EVAL_BOOTSTRAP_SAMPLES", + ), + ), + }); + const evaluation: EvaluationConfig = { + captureCommitted: booleanValue( + evaluationOverride.captureCommitted, + process.env.EVOLVE_EVAL_CAPTURE_COMMITTED, + false, + ), + shadowPercent: boundedValue( + evaluationOverride.shadowPercent, + process.env.EVOLVE_EVAL_SHADOW_PERCENT, + 0, + "EVOLVE_EVAL_SHADOW_PERCENT", + 0, + 100, + ), + monitorPromoted: booleanValue( + evaluationOverride.monitorPromoted, + process.env.EVOLVE_EVAL_MONITOR_PROMOTED, + true, + ), + canaryMinSamples: Math.floor( + numberValue( + evaluationOverride.canaryMinSamples, + process.env.EVOLVE_EVAL_CANARY_MIN_SAMPLES, + 5, + "EVOLVE_EVAL_CANARY_MIN_SAMPLES", + ), + ), + monitorMinSamples: Math.floor( + numberValue( + evaluationOverride.monitorMinSamples, + process.env.EVOLVE_EVAL_MONITOR_MIN_SAMPLES, + 10, + "EVOLVE_EVAL_MONITOR_MIN_SAMPLES", + ), + ), + monitorWindow: Math.floor( + numberValue( + evaluationOverride.monitorWindow, + process.env.EVOLVE_EVAL_MONITOR_WINDOW, + 50, + "EVOLVE_EVAL_MONITOR_WINDOW", + ), + ), + monitorMaxSuccessDrop: boundedValue( + evaluationOverride.monitorMaxSuccessDrop, + process.env.EVOLVE_EVAL_MONITOR_MAX_SUCCESS_DROP, + 0.1, + "EVOLVE_EVAL_MONITOR_MAX_SUCCESS_DROP", + 0, + 1, + ), + monitorMaxScoreDrop: boundedValue( + evaluationOverride.monitorMaxScoreDrop, + process.env.EVOLVE_EVAL_MONITOR_MAX_SCORE_DROP, + 0.1, + "EVOLVE_EVAL_MONITOR_MAX_SCORE_DROP", + 0, + 1, + ), + monitorMaxTokenRatio: numberValue( + evaluationOverride.monitorMaxTokenRatio, + process.env.EVOLVE_EVAL_MONITOR_MAX_TOKEN_RATIO, + 1.5, + "EVOLVE_EVAL_MONITOR_MAX_TOKEN_RATIO", + ), + policy: evaluationPolicy, + }; + if (evaluation.monitorWindow < evaluation.monitorMinSamples || evaluation.monitorWindow < evaluation.canaryMinSamples) { + throw new Error("EVOLVE_EVAL_MONITOR_WINDOW must be at least both EVOLVE_EVAL_MONITOR_MIN_SAMPLES and EVOLVE_EVAL_CANARY_MIN_SAMPLES"); + } + + const apiKey = overrides.openAiApiKey ?? process.env.OPENAI_API_KEY; + const docker: DockerConfig = { + binary: dockerOverride.binary ?? process.env.EVOLVE_DOCKER_BINARY ?? "docker", + ...(defaultImage !== undefined ? { defaultImage } : {}), + allowedImages, + allowedNetworks, + user: dockerOverride.user ?? process.env.EVOLVE_DOCKER_USER ?? "65532:65532", + maximums, + requireRootless: booleanValue( + dockerOverride.requireRootless, + process.env.EVOLVE_DOCKER_REQUIRE_ROOTLESS, + false, + ), + }; + + return { + home, + workspace, + model: overrides.model ?? process.env.OPENAI_MODEL ?? "gpt-5.6-sol", + verifierModel: overrides.verifierModel ?? process.env.OPENAI_VERIFIER_MODEL ?? process.env.OPENAI_MODEL ?? "gpt-5.6-sol", + reasoningEffort: effort as ReasoningEffort, + allowedCommands: new Set([...commands].map((value) => value.trim()).filter(Boolean)), + nonInteractive: overrides.nonInteractive ?? process.env.EVOLVE_NON_INTERACTIVE === "true", + ...(apiKey ? { openAiApiKey: apiKey } : {}), + defaultExecutor, + allowLocalExecutor, + docker, + secretAllowlist: values(overrides.secretAllowlist, process.env.EVOLVE_SECRET_ALLOWLIST), + secretTtlMs: numberValue(overrides.secretTtlMs, process.env.EVOLVE_SECRET_TTL_MS, 300_000, "EVOLVE_SECRET_TTL_MS"), + leaseTtlMs, + leaseHeartbeatMs, + evaluation, + }; +} diff --git a/evolve-agent/src/context/context-compiler.ts b/evolve-agent/src/context/context-compiler.ts new file mode 100644 index 0000000..e2d4d50 --- /dev/null +++ b/evolve-agent/src/context/context-compiler.ts @@ -0,0 +1,45 @@ +import type { TaskSpec, Usage } from "../core/types.js"; +import type { MemoryStore } from "../memory/memory-store.js"; +import type { AgentPrompt, Observation } from "../providers/provider.js"; +import type { SkillStore } from "../skills/skill-store.js"; +import type { ToolRegistry } from "../tools/registry.js"; + +export class ContextCompiler { + public constructor( + private readonly memory: MemoryStore, + private readonly skills: SkillStore, + private readonly tools: ToolRegistry, + ) {} + + public async compile(input: { + task: TaskSpec; + observations: Observation[]; + usage: Usage; + turns: number; + toolCalls: number; + elapsedMs: number; + }): Promise { + const memories = await this.memory.search(`${input.task.goal} ${input.task.constraints.join(" ")}`); + const goalTerms = input.task.goal.toLowerCase().split(/[^\p{L}\p{N}_-]+/u).filter((term) => term.length >= 3); + const skills = (await this.skills.promoted()).filter((skill) => { + const haystack = `${skill.name} ${skill.description} ${skill.triggers.join(" ")}`.toLowerCase(); + return goalTerms.some((term) => haystack.includes(term)); + }); + const requested = new Set(input.task.requestedTools); + const tools = this.tools.modelDescriptions().filter((tool) => requested.has(tool.name)); + return { + task: input.task, + observations: input.observations.slice(-20), + memories: memories.slice(0, 8), + skills: skills.slice(0, 5), + tools, + remainingBudget: { + turns: Math.max(0, input.task.budget.maxTurns - input.turns), + toolCalls: Math.max(0, input.task.budget.maxToolCalls - input.toolCalls), + inputTokens: Math.max(0, input.task.budget.maxInputTokens - input.usage.inputTokens), + outputTokens: Math.max(0, input.task.budget.maxOutputTokens - input.usage.outputTokens), + wallTimeMs: Math.max(0, input.task.budget.maxWallTimeMs - input.elapsedMs), + }, + }; + } +} diff --git a/evolve-agent/src/core/errors.ts b/evolve-agent/src/core/errors.ts new file mode 100644 index 0000000..3d5b776 --- /dev/null +++ b/evolve-agent/src/core/errors.ts @@ -0,0 +1,15 @@ +export class EvolveError extends Error { + public readonly code: string; + public readonly details?: unknown; + + public constructor(code: string, message: string, details?: unknown) { + super(message); + this.name = "EvolveError"; + this.code = code; + if (details !== undefined) this.details = details; + } +} + +export function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/evolve-agent/src/core/fs.ts b/evolve-agent/src/core/fs.ts new file mode 100644 index 0000000..6189fff --- /dev/null +++ b/evolve-agent/src/core/fs.ts @@ -0,0 +1,38 @@ +import { mkdir, open, readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; + +export async function ensureDir(directory: string): Promise { + await mkdir(directory, { recursive: true }); +} + +export async function pathExists(target: string): Promise { + try { + const handle = await open(target, "r"); + await handle.close(); + return true; + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } +} + +export async function readJsonFile(target: string, fallback: T): Promise { + try { + return JSON.parse(await readFile(target, "utf8")) as T; + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return fallback; + throw error; + } +} + +export async function atomicWriteFile(target: string, content: string | Buffer, mode?: number): Promise { + await ensureDir(path.dirname(target)); + const temp = path.join(path.dirname(target), `.${path.basename(target)}.${randomUUID()}.tmp`); + await writeFile(temp, content, mode === undefined ? undefined : { mode }); + await rename(temp, target); +} + +export async function atomicWriteJson(target: string, value: unknown, mode?: number): Promise { + await atomicWriteFile(target, `${JSON.stringify(value, null, 2)}\n`, mode); +} diff --git a/evolve-agent/src/core/hash.ts b/evolve-agent/src/core/hash.ts new file mode 100644 index 0000000..8e94028 --- /dev/null +++ b/evolve-agent/src/core/hash.ts @@ -0,0 +1,10 @@ +import { createHash } from "node:crypto"; +import { stableStringify } from "./stable-json.js"; + +export function sha256Bytes(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +export function sha256Json(value: unknown): string { + return sha256Bytes(stableStringify(value)); +} diff --git a/evolve-agent/src/core/stable-json.ts b/evolve-agent/src/core/stable-json.ts new file mode 100644 index 0000000..0e50cc2 --- /dev/null +++ b/evolve-agent/src/core/stable-json.ts @@ -0,0 +1,18 @@ +function normalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(normalize); + + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, normalize(entry)]), + ); + } + + return value; +} + +export function stableStringify(value: unknown): string { + return JSON.stringify(normalize(value)); +} diff --git a/evolve-agent/src/core/types.ts b/evolve-agent/src/core/types.ts new file mode 100644 index 0000000..5ab3b43 --- /dev/null +++ b/evolve-agent/src/core/types.ts @@ -0,0 +1,169 @@ +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; +export type JsonObject = { [key: string]: JsonValue }; + +export interface TaskBudget { + maxTurns: number; + maxToolCalls: number; + maxInputTokens: number; + maxOutputTokens: number; + maxWallTimeMs: number; +} + +export interface TaskSpec { + id: string; + goal: string; + constraints: string[]; + successCriteria: string[]; + requestedTools: string[]; + budget: TaskBudget; + createdAt: string; +} + +export interface TaskInput { + goal: string; + constraints?: string[]; + successCriteria?: string[]; + requestedTools?: string[]; + budget?: Partial; +} + +export interface Usage { + inputTokens: number; + outputTokens: number; + totalTokens: number; +} + +export type EpisodeStatus = "running" | "committed" | "budget_exhausted" | "interrupted" | "failed"; + +export interface EvidenceRecord { + id: string; + episodeId: string; + toolName: string; + argsHash: string; + artifactHash: string; + summary: string; + success: boolean; + createdAt: string; +} + +export interface LedgerEvent { + version: 1; + index: number; + timestamp: string; + episodeId: string; + type: string; + payload: JsonValue; + prevHash: string | null; + hash: string; +} + +export interface LedgerVerification { + valid: boolean; + events: number; + error?: string; +} + +export interface MemoryRecord { + id: string; + text: string; + tags: string[]; + confidence: number; + sourceEpisodeId: string; + evidenceIds: string[]; + createdAt: string; +} + +export type SkillStatus = "candidate" | "evaluated" | "canary" | "promoted" | "quarantined" | "rolled_back"; + +export interface SkillStep { + toolName: string; + purpose: string; +} + +export interface SkillEvaluation { + at: string; + policyPassed: boolean; + replayPassed: boolean; + score: number; + notes: string[]; +} + +export interface SkillCanary { + at: string; + passed: boolean; + score: number; + note: string; +} + +export interface SkillPromotionRecord { + at: string; + offlineReportId: string; + canaryReportId: string; + policyHash: string; + keyFingerprint: string; +} + +export interface SkillRollbackRecord { + at: string; + reason: string; + automatic: boolean; + reportId?: string; +} + +export interface SkillRecord { + id: string; + fingerprint: string; + name: string; + description: string; + triggers: string[]; + steps: SkillStep[]; + allowedTools: string[]; + supportingEpisodes: string[]; + provenanceEvidenceIds: string[]; + status: SkillStatus; + createdAt: string; + updatedAt: string; + evaluations: SkillEvaluation[]; + canaries: SkillCanary[]; + evaluationReportIds: string[]; + canaryReportIds: string[]; + promotion?: SkillPromotionRecord; + rollback?: SkillRollbackRecord; +} + +export interface EpisodeCheckpoint { + version: 1; + episodeId: string; + task: TaskSpec; + status: EpisodeStatus; + observations: Array<{ + id: string; + kind: "tool" | "error" | "verification" | "system"; + content: string; + evidenceId?: string; + toolName?: string; + }>; + evidenceIds: string[]; + toolSequence: string[]; + activeSkillIds: string[]; + usage: Usage; + turns: number; + toolCalls: number; + elapsedMs: number; + answer?: string; + finalScore?: number; + stopReason?: string; + updatedAt: string; +} + +export interface RunResult { + episodeId: string; + status: EpisodeStatus; + answer?: string; + evidenceIds: string[]; + usage: Usage; + turns: number; + toolCalls: number; + reason?: string; +} diff --git a/evolve-agent/src/evaluation/evaluation-engine.ts b/evolve-agent/src/evaluation/evaluation-engine.ts new file mode 100644 index 0000000..1f88c35 --- /dev/null +++ b/evolve-agent/src/evaluation/evaluation-engine.ts @@ -0,0 +1,422 @@ +import { randomUUID } from "node:crypto"; +import { EvolveError } from "../core/errors.js"; +import { sha256Json } from "../core/hash.js"; +import type { EpisodeCheckpoint, SkillRecord } from "../core/types.js"; +import type { AgentProvider } from "../providers/provider.js"; +import type { SkillStore } from "../skills/skill-store.js"; +import type { ToolRegistry } from "../tools/registry.js"; +import { aggregateRuns, compareRuns, defaultEvaluationPolicy } from "./metrics.js"; +import type { FixtureStore } from "./fixture-store.js"; +import { ReplayHarness } from "./replay-harness.js"; +import type { EvaluationReportStore } from "./report-store.js"; +import type { ShadowStore } from "./shadow-store.js"; +import type { + EvaluationAggregate, + EvaluationComparison, + EvaluationDecision, + EvaluationPolicy, + EvaluationReportPayload, + EvaluationSelection, + FixtureSplit, + ProductionOutcome, + ReplayFixture, + ReplayRunResult, + ShadowObservation, + SignedEvaluationReport, +} from "./types.js"; + +export interface EvaluationEngineOptions { + policy: EvaluationPolicy; + canaryMinSamples: number; + monitorMinSamples: number; + monitorMaxSuccessDrop: number; + monitorMaxScoreDrop: number; + monitorMaxTokenRatio: number; +} + +function matchesSkill(skill: SkillRecord, fixture: ReplayFixture): boolean { + const goal = fixture.task.goal.toLowerCase(); + const triggerMatch = skill.triggers.some((trigger) => trigger.length >= 3 && goal.includes(trigger.toLowerCase())); + if (triggerMatch) return true; + const expected = skill.steps.map((step) => step.toolName); + const observed = fixture.trace.map((step) => step.toolName); + return expected.length > 0 && expected.length === observed.length && expected.every((tool, index) => observed[index] === tool); +} + +function providerName(provider: AgentProvider): string { + return provider.constructor?.name || "AgentProvider"; +} + +function monitorComparison( + expected: EvaluationAggregate, + observed: EvaluationAggregate, +): EvaluationComparison { + const ratio = (candidate: number, baseline: number): number => + baseline === 0 ? (candidate === 0 ? 1 : Number.POSITIVE_INFINITY) : candidate / baseline; + return { + pairedSamples: 0, + wins: 0, + losses: 0, + ties: 0, + newFailures: Math.max(0, expected.successes - observed.successes), + successDelta: observed.successRate - expected.successRate, + verifierScoreDelta: observed.meanVerifierScore - expected.meanVerifierScore, + totalTokenRatio: ratio(observed.meanTotalTokens, expected.meanTotalTokens), + toolCallRatio: ratio(observed.meanToolCalls, expected.meanToolCalls), + durationRatio: ratio(observed.meanDurationMs, expected.meanDurationMs), + confidence: { + successDeltaLower: observed.successRate - expected.successRate, + successDeltaUpper: observed.successRate - expected.successRate, + verifierScoreDeltaLower: observed.meanVerifierScore - expected.meanVerifierScore, + verifierScoreDeltaUpper: observed.meanVerifierScore - expected.meanVerifierScore, + }, + }; +} + +export class EvaluationEngine { + private readonly replay: ReplayHarness; + + public constructor( + private readonly fixtures: FixtureStore, + private readonly reports: EvaluationReportStore, + private readonly shadow: ShadowStore, + private readonly skills: SkillStore, + provider: AgentProvider, + tools: ToolRegistry, + private readonly options: EvaluationEngineOptions, + ) { + this.replay = new ReplayHarness(provider, tools.modelDescriptions()); + this.provider = provider; + } + + private readonly provider: AgentProvider; + + public async evaluateSkill(skillId: string, selection: EvaluationSelection = {}): Promise { + const skill = await this.skills.get(skillId); + if (skill.status === "promoted" || skill.status === "rolled_back") { + throw new EvolveError("SKILL_STATE", `Cannot run offline evaluation for a ${skill.status} Skill`); + } + const policy = defaultEvaluationPolicy({ + ...this.options.policy, + ...(selection.repeats !== undefined ? { repeats: selection.repeats } : {}), + requireImprovement: true, + }); + const fixtures = await this.selectFixtures(skill, selection, policy.minFixtures); + const promoted = (await this.skills.promoted()).filter( + (record) => record.id !== skill.id && fixtures.some((fixture) => matchesSkill(record, fixture)), + ); + const baselineRuns: ReplayRunResult[] = []; + const candidateRuns: ReplayRunResult[] = []; + + for (let repeat = 0; repeat < policy.repeats; repeat += 1) { + for (let index = 0; index < fixtures.length; index += 1) { + const fixture = fixtures[index] as ReplayFixture; + const baselineInput = { fixture, arm: "baseline" as const, repeat, skills: promoted }; + const candidateInput = { fixture, arm: "candidate" as const, repeat, skills: [...promoted, skill] }; + if ((index + repeat) % 2 === 0) { + baselineRuns.push(await this.replay.run(baselineInput)); + candidateRuns.push(await this.replay.run(candidateInput)); + } else { + candidateRuns.push(await this.replay.run(candidateInput)); + baselineRuns.push(await this.replay.run(baselineInput)); + } + } + } + + const metrics = compareRuns(baselineRuns, candidateRuns, policy); + const report = await this.reports.create( + this.payload("offline", skill, fixtures, policy, baselineRuns, candidateRuns, metrics, [ + "Supporting Episodes were excluded to prevent train/evaluation leakage.", + "Baseline and candidate used the same fixtures, tools, verifier, and budgets.", + ]), + ); + await this.skills.attachEvaluationReport(skill.id, report.id, report.payload.decision.passed); + return report; + } + + public async shadowEpisode( + skillId: string, + episodeId: string, + requestedMode?: ShadowObservation["mode"], + ): Promise { + const skill = await this.skills.get(skillId); + if (skill.supportingEpisodes.includes(episodeId)) { + throw new EvolveError("EVAL_LEAKAGE", "A Skill cannot shadow-evaluate one of its supporting Episodes"); + } + const fixture = (await this.fixtures.forEpisode(episodeId)) ?? (await this.fixtures.capture(episodeId)); + if (!matchesSkill(skill, fixture)) throw new EvolveError("EVAL_FIXTURE", "Episode does not match the Skill triggers or trace"); + const active = fixture.baseline.activeSkillIds.includes(skill.id); + const mode = requestedMode ?? (skill.status === "promoted" && active ? "production-candidate" : "production-baseline"); + if (mode === "production-baseline" && active) { + throw new EvolveError("EVAL_ARM", "Production baseline Episode already used the candidate Skill"); + } + if (mode === "production-candidate" && !active) { + throw new EvolveError("EVAL_ARM", "Production candidate Episode did not use the Skill"); + } + + const baselineSkills = (await this.skills.promoted()).filter((record) => record.id !== skill.id && matchesSkill(record, fixture)); + const actual = ReplayHarness.actualBaseline({ + fixtureId: fixture.id, + fixtureHash: fixture.integrityHash, + sourceEpisodeId: fixture.sourceEpisodeId, + verifierScore: fixture.baseline.verifierScore, + usage: fixture.baseline.usage, + turns: fixture.baseline.turns, + toolCalls: fixture.baseline.toolCalls, + durationMs: fixture.baseline.elapsedMs, + activeSkillIds: fixture.baseline.activeSkillIds, + arm: mode === "production-baseline" ? "baseline" : "candidate", + }); + const counterfactual = await this.replay.run({ + fixture, + arm: mode === "production-baseline" ? "candidate" : "baseline", + repeat: 0, + skills: mode === "production-baseline" ? [...baselineSkills, skill] : baselineSkills, + }); + const baseline = mode === "production-baseline" ? actual : counterfactual; + const candidate = mode === "production-baseline" ? counterfactual : actual; + const createdAt = new Date().toISOString(); + const observation: ShadowObservation = { + id: `shadow_${sha256Json({ skillId, episodeId, mode, baseline, candidate }).slice(0, 24)}`, + skillId, + episodeId, + fixtureId: fixture.id, + mode, + baseline, + candidate, + createdAt, + }; + const stored = await this.shadow.addObservation(observation); + + if (skill.status === "evaluated") { + const samples = await this.shadow.observations(skill.id, "production-baseline"); + if (samples.length >= this.options.canaryMinSamples) await this.finalizeCanary(skill.id); + } + return stored; + } + + public async finalizeCanary(skillId: string): Promise { + const skill = await this.skills.get(skillId); + if (skill.evaluationReportIds.length === 0) throw new EvolveError("SKILL_GATE", "Canary requires offline evaluation"); + const offlineId = skill.evaluationReportIds.at(-1) as string; + const offline = await this.reports.requireVerified(offlineId); + if (!offline.payload.decision.passed || offline.payload.kind !== "offline") { + throw new EvolveError("SKILL_GATE", "Latest offline evaluation is not a passing signed report"); + } + const observations = (await this.shadow.observations(skillId, "production-baseline")).filter( + (observation) => observation.createdAt >= offline.payload.createdAt, + ); + if (observations.length < this.options.canaryMinSamples) { + throw new EvolveError( + "CANARY_SAMPLES", + `Canary requires ${this.options.canaryMinSamples} independent shadow samples; found ${observations.length}`, + ); + } + const policy = defaultEvaluationPolicy({ + ...this.options.policy, + minFixtures: this.options.canaryMinSamples, + repeats: 1, + requireImprovement: false, + }); + const baselineRuns = observations.map((observation) => observation.baseline); + const candidateRuns = observations.map((observation) => observation.candidate); + const metrics = compareRuns(baselineRuns, candidateRuns, policy); + const fixtures = await Promise.all(observations.map((observation) => this.fixtures.get(observation.fixtureId))); + const report = await this.reports.create( + this.payload("canary", skill, fixtures, policy, baselineRuns, candidateRuns, metrics, [ + "Canary traffic was replayed in shadow and never changed the production answer.", + "Canary requires non-regression; the offline report is responsible for proving improvement.", + ]), + ); + await this.skills.attachCanaryReport(skill.id, report.id, report.payload.decision.passed); + return report; + } + + public async promoteSkill(skillId: string): Promise { + const skill = await this.skills.get(skillId); + const offlineId = skill.evaluationReportIds.at(-1); + const canaryId = skill.canaryReportIds.at(-1); + if (!offlineId || !canaryId) throw new EvolveError("SKILL_GATE", "Promotion requires offline and canary reports"); + const offline = await this.reports.requireVerified(offlineId); + const canary = await this.reports.requireVerified(canaryId); + for (const report of [offline, canary]) { + if (report.payload.skillId !== skill.id || report.payload.skillFingerprint !== skill.fingerprint) { + throw new EvolveError("SKILL_GATE", "Evaluation report provenance does not match this Skill"); + } + if (!report.payload.decision.passed) throw new EvolveError("SKILL_GATE", `${report.payload.kind} evaluation did not pass`); + } + if (offline.payload.kind !== "offline" || canary.payload.kind !== "canary") { + throw new EvolveError("SKILL_GATE", "Promotion report kinds are invalid"); + } + if (offline.keyFingerprint !== canary.keyFingerprint) { + throw new EvolveError("SKILL_GATE", "Offline and canary reports were signed by different evaluation authorities"); + } + return this.skills.promote(skill.id, { + offlineReportId: offline.id, + canaryReportId: canary.id, + policyHash: sha256Json({ offline: offline.payload.policy, canary: canary.payload.policy }), + keyFingerprint: offline.keyFingerprint, + }); + } + + public async recordProduction(checkpoint: EpisodeCheckpoint): Promise { + const reports: SignedEvaluationReport[] = []; + for (const skillId of checkpoint.activeSkillIds ?? []) { + const skill = await this.skills.get(skillId).catch(() => undefined); + if (!skill || skill.status !== "promoted") continue; + const outcome: ProductionOutcome = { + id: `prod_${sha256Json({ skillId, episodeId: checkpoint.episodeId, status: checkpoint.status }).slice(0, 24)}`, + skillId, + episodeId: checkpoint.episodeId, + createdAt: new Date().toISOString(), + success: checkpoint.status === "committed", + verifierScore: checkpoint.finalScore ?? 0, + usage: checkpoint.usage, + turns: checkpoint.turns, + toolCalls: checkpoint.toolCalls, + status: checkpoint.status, + }; + await this.shadow.addProduction(outcome); + const report = await this.monitorSkill(skill.id); + if (report) reports.push(report); + } + return reports; + } + + public async monitorSkill(skillId: string): Promise { + const skill = await this.skills.get(skillId); + if (skill.status !== "promoted" || !skill.promotion) return undefined; + const outcomes = await this.shadow.production(skill.id); + if (outcomes.length < this.options.monitorMinSamples) return undefined; + const canary = await this.reports.requireVerified(skill.promotion.canaryReportId); + const observedRuns: ReplayRunResult[] = outcomes.map((outcome, index) => ({ + fixtureId: `production_${outcome.episodeId}`, + fixtureHash: sha256Json(outcome), + sourceEpisodeId: outcome.episodeId, + arm: "candidate", + repeat: index, + success: outcome.success, + verifierScore: outcome.verifierScore, + usage: outcome.usage, + turns: outcome.turns, + toolCalls: outcome.toolCalls, + durationMs: 0, + traceMatched: true, + ...(!outcome.success ? { failureReason: outcome.status } : {}), + safetyViolations: [], + activeSkillIds: [skill.id], + })); + const observed = aggregateRuns(observedRuns); + const expected = canary.payload.candidate; + const comparison = monitorComparison(expected, observed); + const gates: Record = { + minimum_samples: outcomes.length >= this.options.monitorMinSamples, + success_rate: observed.successRate >= expected.successRate - this.options.monitorMaxSuccessDrop, + verifier_score: observed.meanVerifierScore >= expected.meanVerifierScore - this.options.monitorMaxScoreDrop, + token_budget: + expected.meanTotalTokens === 0 || observed.meanTotalTokens / expected.meanTotalTokens <= this.options.monitorMaxTokenRatio, + }; + const decision: EvaluationDecision = { + passed: Object.values(gates).every(Boolean), + gates, + reasons: Object.entries(gates) + .filter(([, passed]) => !passed) + .map(([gate]) => `Production regression gate failed: ${gate}`), + }; + if (decision.reasons.length === 0) decision.reasons.push("Production monitor remains within the signed canary envelope"); + const policy = defaultEvaluationPolicy({ + ...this.options.policy, + minFixtures: this.options.monitorMinSamples, + requireImprovement: false, + }); + const payload: EvaluationReportPayload = { + version: 1, + engineVersion: "0.3.0", + kind: "monitor", + skillId: skill.id, + skillFingerprint: skill.fingerprint, + createdAt: new Date().toISOString(), + fixtureIds: outcomes.map((outcome) => `production_${outcome.episodeId}`), + fixtureHashes: outcomes.map((outcome) => sha256Json(outcome)), + policy, + baselineRuns: [], + candidateRuns: observedRuns, + baseline: expected, + candidate: observed, + comparison, + decision, + metadata: { + provider: providerName(this.provider), + notes: ["Production outcomes are compared with the signed canary candidate envelope."], + }, + }; + const report = await this.reports.create(payload); + if (!decision.passed) { + await this.skills.rollback(skill.id, decision.reasons.join("; "), { automatic: true, reportId: report.id }); + } + return report; + } + + public async verifyReport(reportId: string): Promise { + return this.reports.verify(reportId); + } + + private async selectFixtures( + skill: SkillRecord, + selection: EvaluationSelection, + minimum: number, + ): Promise { + const splits = new Set(selection.splits ?? ["validation", "holdout"]); + const candidates = selection.fixtures + ? await Promise.all(selection.fixtures.map((id) => this.fixtures.get(id))) + : await this.fixtures.list(splits); + const fixtures = candidates.filter( + (fixture) => !skill.supportingEpisodes.includes(fixture.sourceEpisodeId) && matchesSkill(skill, fixture), + ); + if (fixtures.length < minimum) { + throw new EvolveError( + "EVAL_FIXTURES", + `Evaluation requires ${minimum} independent matching fixtures after leakage exclusion; found ${fixtures.length}`, + ); + } + return fixtures; + } + + private payload( + kind: "offline" | "canary", + skill: SkillRecord, + fixtures: ReplayFixture[], + policy: EvaluationPolicy, + baselineRuns: ReplayRunResult[], + candidateRuns: ReplayRunResult[], + metrics: { + baseline: EvaluationAggregate; + candidate: EvaluationAggregate; + comparison: EvaluationComparison; + decision: EvaluationDecision; + }, + notes: string[], + ): EvaluationReportPayload { + return { + version: 1, + engineVersion: "0.3.0", + kind, + skillId: skill.id, + skillFingerprint: skill.fingerprint, + createdAt: new Date().toISOString(), + fixtureIds: fixtures.map((fixture) => fixture.id), + fixtureHashes: fixtures.map((fixture) => fixture.integrityHash), + policy, + baselineRuns, + candidateRuns, + baseline: metrics.baseline, + candidate: metrics.candidate, + comparison: metrics.comparison, + decision: metrics.decision, + metadata: { + provider: providerName(this.provider), + notes, + }, + }; + } +} diff --git a/evolve-agent/src/evaluation/evolution-orchestrator.ts b/evolve-agent/src/evaluation/evolution-orchestrator.ts new file mode 100644 index 0000000..5d0a1ba --- /dev/null +++ b/evolve-agent/src/evaluation/evolution-orchestrator.ts @@ -0,0 +1,95 @@ +import { sha256Json } from "../core/hash.js"; +import type { EpisodeCheckpoint, SkillRecord } from "../core/types.js"; +import type { JsonlLedger } from "../ledger/jsonl-ledger.js"; +import type { SkillStore } from "../skills/skill-store.js"; +import type { EvaluationEngine } from "./evaluation-engine.js"; +import type { FixtureStore } from "./fixture-store.js"; + +export interface EvolutionOrchestratorOptions { + captureCommitted: boolean; + shadowPercent: number; + monitorPromoted: boolean; +} + +function selected(episodeId: string, skillId: string, percent: number): boolean { + if (percent <= 0) return false; + if (percent >= 100) return true; + const bucket = Number.parseInt(sha256Json({ episodeId, skillId }).slice(0, 8), 16) / 0xffff_ffff; + return bucket * 100 < percent; +} + +function matches(skill: SkillRecord, goal: string): boolean { + const normalized = goal.toLowerCase(); + return skill.triggers.some((trigger) => trigger.length >= 3 && normalized.includes(trigger.toLowerCase())); +} + +export class EvolutionOrchestrator { + public constructor( + private readonly fixtures: FixtureStore, + private readonly evaluations: EvaluationEngine, + private readonly skills: SkillStore, + private readonly ledger: JsonlLedger, + private readonly options: EvolutionOrchestratorOptions, + ) {} + + public async observeTerminal(checkpoint: EpisodeCheckpoint): Promise { + if (checkpoint.status === "committed" && this.options.captureCommitted) { + try { + const fixture = await this.fixtures.capture(checkpoint.episodeId); + await this.ledger.append(checkpoint.episodeId, "evaluation.fixture_captured", { + fixture_id: fixture.id, + split: fixture.split, + integrity_hash: fixture.integrityHash, + }); + } catch (error: unknown) { + await this.ledger.append(checkpoint.episodeId, "evaluation.fixture_rejected", { + reason: error instanceof Error ? error.message : String(error), + }); + } + } + + if (checkpoint.status === "committed") { + const skills = await this.skills.list(); + for (const skill of skills) { + if (!matches(skill, checkpoint.task.goal) || skill.supportingEpisodes.includes(checkpoint.episodeId)) continue; + if ( + (skill.status === "evaluated" || skill.status === "canary") && + selected(checkpoint.episodeId, skill.id, this.options.shadowPercent) + ) { + try { + const observation = await this.evaluations.shadowEpisode(skill.id, checkpoint.episodeId, "production-baseline"); + await this.ledger.append(checkpoint.episodeId, "evaluation.shadow_recorded", { + skill_id: skill.id, + observation_id: observation.id, + mode: observation.mode, + baseline_success: observation.baseline.success, + candidate_success: observation.candidate.success, + }); + } catch (error: unknown) { + await this.ledger.append(checkpoint.episodeId, "evaluation.shadow_failed", { + skill_id: skill.id, + reason: error instanceof Error ? error.message : String(error), + }); + } + } + } + } + + if (this.options.monitorPromoted && (checkpoint.activeSkillIds?.length ?? 0) > 0) { + try { + const reports = await this.evaluations.recordProduction(checkpoint); + for (const report of reports) { + await this.ledger.append(checkpoint.episodeId, "evaluation.production_monitor", { + skill_id: report.payload.skillId, + report_id: report.id, + passed: report.payload.decision.passed, + }); + } + } catch (error: unknown) { + await this.ledger.append(checkpoint.episodeId, "evaluation.production_monitor_failed", { + reason: error instanceof Error ? error.message : String(error), + }); + } + } + } +} diff --git a/evolve-agent/src/evaluation/fixture-store.ts b/evolve-agent/src/evaluation/fixture-store.ts new file mode 100644 index 0000000..261b414 --- /dev/null +++ b/evolve-agent/src/evaluation/fixture-store.ts @@ -0,0 +1,274 @@ +import { readdir } from "node:fs/promises"; +import path from "node:path"; +import { atomicWriteJson, ensureDir, readJsonFile } from "../core/fs.js"; +import { sha256Json } from "../core/hash.js"; +import type { JsonObject, JsonValue, LedgerEvent } from "../core/types.js"; +import { EvolveError } from "../core/errors.js"; +import type { ArtifactStore } from "../ledger/artifact-store.js"; +import type { JsonlLedger } from "../ledger/jsonl-ledger.js"; +import type { CheckpointStore } from "../runtime/checkpoint-store.js"; +import type { FixtureSplit, ReplayFixture, ReplayFixturePayload, ReplayTraceStep } from "./types.js"; + +const FIXTURE_ID = /^fixture_[a-f0-9]{24}$/; + +function objectValue(value: JsonValue): JsonObject { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new EvolveError("FIXTURE_EVENT", "Ledger event payload is not an object"); + } + return value; +} + +function requiredString(object: JsonObject, key: string): string { + const value = object[key]; + if (typeof value !== "string" || value.length === 0) { + throw new EvolveError("FIXTURE_EVENT", `Ledger event omitted ${key}`); + } + return value; +} + +function numberValue(object: JsonObject, key: string, fallback = 0): number { + const value = object[key]; + return typeof value === "number" && Number.isFinite(value) ? value : fallback; +} + +function splitFor(sourceEpisodeId: string): FixtureSplit { + const bucket = Number.parseInt(sha256Json(sourceEpisodeId).slice(0, 8), 16) % 10; + if (bucket >= 8) return "holdout"; + if (bucket >= 6) return "validation"; + return "train"; +} + + +function validatePayload(payload: ReplayFixturePayload): void { + if (payload.version !== 1) throw new EvolveError("FIXTURE_FORMAT", "Unsupported fixture version"); + if (!/^ep_[a-f0-9]{24}$/.test(payload.sourceEpisodeId)) { + throw new EvolveError("FIXTURE_FORMAT", "Fixture source Episode ID is invalid"); + } + if (!["train", "validation", "holdout"].includes(payload.split)) { + throw new EvolveError("FIXTURE_FORMAT", "Fixture split is invalid"); + } + if (!payload.task || typeof payload.task.goal !== "string" || payload.task.goal.length < 3) { + throw new EvolveError("FIXTURE_FORMAT", "Fixture task is invalid"); + } + if (!Array.isArray(payload.trace) || payload.trace.length > payload.task.budget.maxToolCalls) { + throw new EvolveError("FIXTURE_FORMAT", "Fixture trace exceeds its tool-call budget"); + } + for (const [index, step] of payload.trace.entries()) { + if (step.index !== index || !/^[a-f0-9]{64}$/.test(step.argsHash)) { + throw new EvolveError("FIXTURE_FORMAT", `Fixture trace step ${index} is malformed`); + } + if (!/^ev_[a-f0-9]{24}$/.test(step.evidence.id) || step.evidence.episodeId !== payload.sourceEpisodeId) { + throw new EvolveError("FIXTURE_FORMAT", `Fixture trace step ${index} has invalid evidence provenance`); + } + if (step.evidence.toolName !== step.toolName || !/^[a-f0-9]{64}$/.test(step.evidence.argsHash)) { + throw new EvolveError("FIXTURE_FORMAT", `Fixture trace step ${index} evidence does not match the action`); + } + if (!/^[a-f0-9]{64}$/.test(step.evidence.artifactHash)) { + throw new EvolveError("FIXTURE_FORMAT", `Fixture trace step ${index} artifact hash is invalid`); + } + } + if (payload.baseline.status !== "committed" || payload.baseline.verifierScore < 0 || payload.baseline.verifierScore > 1) { + throw new EvolveError("FIXTURE_FORMAT", "Fixture baseline is invalid"); + } +} + +function payloadOf(fixture: ReplayFixture): ReplayFixturePayload { + const { id: _id, integrityHash: _integrityHash, ...payload } = fixture; + return payload; +} + +export class FixtureStore { + private readonly directory: string; + + public constructor( + home: string, + private readonly checkpoints: CheckpointStore, + private readonly ledger: JsonlLedger, + private readonly artifacts: ArtifactStore, + ) { + this.directory = path.join(home, "evaluations", "fixtures"); + } + + private pathFor(id: string): string { + if (!FIXTURE_ID.test(id)) throw new EvolveError("FIXTURE_ID", `Invalid fixture ID: ${id}`); + return path.join(this.directory, `${id}.json`); + } + + public verify(fixture: ReplayFixture): boolean { + const integrityHash = sha256Json(payloadOf(fixture)); + return fixture.integrityHash === integrityHash && fixture.id === `fixture_${integrityHash.slice(0, 24)}`; + } + + public async put(payload: ReplayFixturePayload): Promise { + validatePayload(payload); + const normalized: ReplayFixturePayload = { + ...payload, + task: { + ...payload.task, + constraints: [...payload.task.constraints], + successCriteria: [...payload.task.successCriteria], + requestedTools: [...payload.task.requestedTools], + budget: { ...payload.task.budget }, + }, + trace: payload.trace.map((step, index) => ({ + ...step, + index, + evidence: { ...step.evidence }, + })), + baseline: { + ...payload.baseline, + usage: { ...payload.baseline.usage }, + activeSkillIds: [...new Set(payload.baseline.activeSkillIds)].sort(), + }, + }; + const integrityHash = sha256Json(normalized); + const fixture: ReplayFixture = { + ...normalized, + id: `fixture_${integrityHash.slice(0, 24)}`, + integrityHash, + }; + await atomicWriteJson(this.pathFor(fixture.id), fixture, 0o600); + return fixture; + } + + public async import(value: unknown): Promise { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new EvolveError("FIXTURE_IMPORT", "Fixture import must be an object"); + } + const candidate = value as Partial; + if (candidate.version !== 1 || !candidate.sourceEpisodeId || !candidate.task || !candidate.baseline || !candidate.trace) { + throw new EvolveError("FIXTURE_IMPORT", "Fixture import is missing required fields"); + } + const payload: ReplayFixturePayload = { + version: 1, + sourceEpisodeId: candidate.sourceEpisodeId, + split: candidate.split ?? splitFor(candidate.sourceEpisodeId), + createdAt: candidate.createdAt ?? new Date().toISOString(), + task: candidate.task, + trace: candidate.trace, + baseline: candidate.baseline, + }; + return this.put(payload); + } + + public async get(id: string): Promise { + const fixture = await readJsonFile(this.pathFor(id), null); + if (!fixture) throw new EvolveError("FIXTURE_NOT_FOUND", `Unknown fixture: ${id}`); + if (!this.verify(fixture)) throw new EvolveError("FIXTURE_TAMPERED", `Fixture integrity check failed: ${id}`); + return fixture; + } + + public async list(splits?: Set): Promise { + await ensureDir(this.directory); + const files = (await readdir(this.directory)).filter((file) => /^fixture_[a-f0-9]{24}\.json$/.test(file)).sort(); + const fixtures: ReplayFixture[] = []; + for (const file of files) { + const fixture = await this.get(file.slice(0, -5)); + if (!splits || splits.has(fixture.split)) fixtures.push(fixture); + } + return fixtures; + } + + public async forEpisode(episodeId: string): Promise { + return (await this.list()).find((fixture) => fixture.sourceEpisodeId === episodeId); + } + + public async capture(episodeId: string, split?: FixtureSplit): Promise { + const existing = await this.forEpisode(episodeId); + if (existing) return existing; + + const checkpoint = await this.checkpoints.load(episodeId); + if (checkpoint.status !== "committed" || !checkpoint.answer) { + throw new EvolveError("FIXTURE_EPISODE", "Only committed Episodes with a final answer can become replay fixtures"); + } + const events = await this.ledger.forEpisode(episodeId); + const forbidden = new Set([ + "tool.arguments_rejected", + "tool.policy_denied", + "approval.denied", + "episode.failed", + "episode.budget_exhausted", + ]); + const unsafe = events.find((event) => forbidden.has(event.type)); + if (unsafe) { + throw new EvolveError("FIXTURE_EPISODE", `Episode contains a non-replayable event: ${unsafe.type}`); + } + + const trace = await this.traceFrom(events, checkpoint.observations); + if (trace.length !== checkpoint.toolCalls || trace.length !== checkpoint.toolSequence.length) { + throw new EvolveError( + "FIXTURE_TRACE", + `Replayable tool count mismatch: trace=${trace.length}, calls=${checkpoint.toolCalls}, successful=${checkpoint.toolSequence.length}`, + ); + } + const committed = events.findLast((event) => event.type === "episode.committed"); + if (!committed) throw new EvolveError("FIXTURE_EPISODE", "Episode has no committed ledger event"); + const committedPayload = objectValue(committed.payload); + const answerHash = requiredString(committedPayload, "answer_hash"); + const verifierScore = checkpoint.finalScore ?? numberValue(committedPayload, "score", 0); + + return this.put({ + version: 1, + sourceEpisodeId: episodeId, + split: split ?? splitFor(episodeId), + createdAt: new Date().toISOString(), + task: checkpoint.task, + trace, + baseline: { + status: "committed", + answerHash, + verifierScore, + usage: checkpoint.usage, + turns: checkpoint.turns, + toolCalls: checkpoint.toolCalls, + elapsedMs: checkpoint.elapsedMs, + activeSkillIds: checkpoint.activeSkillIds ?? [], + }, + }); + } + + private async traceFrom( + events: LedgerEvent[], + observations: Array<{ content: string; evidenceId?: string; toolName?: string }>, + ): Promise { + const pending: Array<{ toolName: string; argsHash: string }> = []; + const trace: ReplayTraceStep[] = []; + + for (const event of events) { + const payload = objectValue(event.payload); + if (event.type === "model.tool_proposed") { + pending.push({ + toolName: requiredString(payload, "tool"), + argsHash: requiredString(payload, "args_hash"), + }); + continue; + } + if (event.type !== "tool.executed" && event.type !== "tool.failed") continue; + const toolName = requiredString(payload, "tool"); + const proposalIndex = pending.findIndex((proposal) => proposal.toolName === toolName); + if (proposalIndex < 0) throw new EvolveError("FIXTURE_TRACE", `No proposal for executed tool ${toolName}`); + const [proposal] = pending.splice(proposalIndex, 1); + if (!proposal) throw new EvolveError("FIXTURE_TRACE", `Missing proposal for ${toolName}`); + const evidenceId = requiredString(payload, "evidence_id"); + const evidence = await this.artifacts.getEvidence(evidenceId); + if (!evidence || evidence.episodeId !== event.episodeId) { + throw new EvolveError("FIXTURE_TRACE", `Missing evidence ${evidenceId}`); + } + const executedArgsHash = requiredString(payload, "args_hash"); + if (executedArgsHash !== evidence.argsHash) { + throw new EvolveError("FIXTURE_TRACE", `Executed arguments do not match evidence ${evidenceId}`); + } + const observation = observations.find((entry) => entry.evidenceId === evidenceId)?.content; + trace.push({ + index: trace.length, + toolName, + argsHash: proposal.argsHash, + evidence, + observation: observation ?? `${evidence.summary} [evidence:${evidence.id}]`, + success: evidence.success, + }); + } + if (pending.length > 0) throw new EvolveError("FIXTURE_TRACE", "Episode contains proposed tools without evidence"); + return trace; + } +} diff --git a/evolve-agent/src/evaluation/metrics.ts b/evolve-agent/src/evaluation/metrics.ts new file mode 100644 index 0000000..503ebe0 --- /dev/null +++ b/evolve-agent/src/evaluation/metrics.ts @@ -0,0 +1,205 @@ +import { sha256Json } from "../core/hash.js"; +import type { + EvaluationAggregate, + EvaluationComparison, + EvaluationDecision, + EvaluationPolicy, + ReplayRunResult, +} from "./types.js"; + +function mean(values: number[]): number { + return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function ratio(candidate: number, baseline: number): number { + if (baseline === 0) return candidate === 0 ? 1 : Number.POSITIVE_INFINITY; + return candidate / baseline; +} + +function clamp(value: number, minimum: number, maximum: number): number { + return Math.min(maximum, Math.max(minimum, value)); +} + +function quantile(sorted: number[], probability: number): number { + if (sorted.length === 0) return 0; + const index = clamp((sorted.length - 1) * probability, 0, sorted.length - 1); + const lower = Math.floor(index); + const upper = Math.ceil(index); + const left = sorted[lower] ?? 0; + const right = sorted[upper] ?? left; + return left + (right - left) * (index - lower); +} + +function seedFrom(value: unknown): number { + const seed = Number.parseInt(sha256Json(value).slice(0, 8), 16) >>> 0; + return seed === 0 ? 0x9e3779b9 : seed; +} + +function randomGenerator(seed: number): () => number { + let state = seed >>> 0; + return () => { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return (state >>> 0) / 0x1_0000_0000; + }; +} + +export function aggregateRuns(runs: ReplayRunResult[]): EvaluationAggregate { + const samples = runs.length; + const successes = runs.filter((run) => run.success).length; + return { + samples, + successes, + successRate: samples === 0 ? 0 : successes / samples, + meanVerifierScore: mean(runs.map((run) => run.verifierScore)), + meanInputTokens: mean(runs.map((run) => run.usage.inputTokens)), + meanOutputTokens: mean(runs.map((run) => run.usage.outputTokens)), + meanTotalTokens: mean(runs.map((run) => run.usage.totalTokens)), + meanTurns: mean(runs.map((run) => run.turns)), + meanToolCalls: mean(runs.map((run) => run.toolCalls)), + meanDurationMs: mean(runs.map((run) => run.durationMs)), + traceMatchRate: samples === 0 ? 0 : runs.filter((run) => run.traceMatched).length / samples, + safetyViolations: runs.reduce((sum, run) => sum + run.safetyViolations.length, 0), + }; +} + +function pairRuns( + baselineRuns: ReplayRunResult[], + candidateRuns: ReplayRunResult[], +): Array<{ baseline: ReplayRunResult; candidate: ReplayRunResult }> { + const baseline = new Map(baselineRuns.map((run) => [`${run.fixtureId}:${run.repeat}`, run])); + const pairs: Array<{ baseline: ReplayRunResult; candidate: ReplayRunResult }> = []; + for (const candidate of candidateRuns) { + const match = baseline.get(`${candidate.fixtureId}:${candidate.repeat}`); + if (match) pairs.push({ baseline: match, candidate }); + } + return pairs; +} + +function bootstrapIntervals( + pairs: Array<{ baseline: ReplayRunResult; candidate: ReplayRunResult }>, + policy: EvaluationPolicy, +): EvaluationComparison["confidence"] { + if (pairs.length === 0) { + return { + successDeltaLower: 0, + successDeltaUpper: 0, + verifierScoreDeltaLower: 0, + verifierScoreDeltaUpper: 0, + }; + } + const random = randomGenerator(seedFrom({ pairs, policy })); + const successDeltas: number[] = []; + const scoreDeltas: number[] = []; + for (let iteration = 0; iteration < policy.bootstrapSamples; iteration += 1) { + const selected = Array.from({ length: pairs.length }, () => pairs[Math.floor(random() * pairs.length)] as (typeof pairs)[number]); + successDeltas.push( + mean(selected.map(({ baseline, candidate }) => Number(candidate.success) - Number(baseline.success))), + ); + scoreDeltas.push(mean(selected.map(({ baseline, candidate }) => candidate.verifierScore - baseline.verifierScore))); + } + successDeltas.sort((left, right) => left - right); + scoreDeltas.sort((left, right) => left - right); + const tail = (1 - policy.confidenceLevel) / 2; + return { + successDeltaLower: quantile(successDeltas, tail), + successDeltaUpper: quantile(successDeltas, 1 - tail), + verifierScoreDeltaLower: quantile(scoreDeltas, tail), + verifierScoreDeltaUpper: quantile(scoreDeltas, 1 - tail), + }; +} + +export function compareRuns( + baselineRuns: ReplayRunResult[], + candidateRuns: ReplayRunResult[], + policy: EvaluationPolicy, +): { + baseline: EvaluationAggregate; + candidate: EvaluationAggregate; + comparison: EvaluationComparison; + decision: EvaluationDecision; +} { + const baseline = aggregateRuns(baselineRuns); + const candidate = aggregateRuns(candidateRuns); + const pairs = pairRuns(baselineRuns, candidateRuns); + let wins = 0; + let losses = 0; + let ties = 0; + let newFailures = 0; + for (const pair of pairs) { + const baselineUtility = Number(pair.baseline.success) * 10 + pair.baseline.verifierScore; + const candidateUtility = Number(pair.candidate.success) * 10 + pair.candidate.verifierScore; + if (candidateUtility > baselineUtility + 1e-9) wins += 1; + else if (candidateUtility < baselineUtility - 1e-9) losses += 1; + else ties += 1; + if (pair.baseline.success && !pair.candidate.success) newFailures += 1; + } + + const comparison: EvaluationComparison = { + pairedSamples: pairs.length, + wins, + losses, + ties, + newFailures, + successDelta: candidate.successRate - baseline.successRate, + verifierScoreDelta: candidate.meanVerifierScore - baseline.meanVerifierScore, + totalTokenRatio: ratio(candidate.meanTotalTokens, baseline.meanTotalTokens), + toolCallRatio: ratio(candidate.meanToolCalls, baseline.meanToolCalls), + durationRatio: ratio(candidate.meanDurationMs, baseline.meanDurationMs), + confidence: bootstrapIntervals(pairs, policy), + }; + + const fixtureCount = new Set(candidateRuns.map((run) => run.fixtureId)).size; + const qualityNonRegression = comparison.successDelta >= -policy.maxSuccessRegression; + const scoreNonRegression = comparison.verifierScoreDelta >= -policy.maxScoreRegression; + const confidenceNonRegression = + comparison.confidence.successDeltaLower >= -policy.maxSuccessRegression && + comparison.confidence.verifierScoreDeltaLower >= -policy.maxScoreRegression; + const qualityImprovement = + comparison.successDelta >= policy.minSuccessImprovement || + comparison.verifierScoreDelta >= policy.minScoreImprovement || + comparison.totalTokenRatio <= policy.efficiencyImprovementRatio || + comparison.toolCallRatio <= policy.efficiencyImprovementRatio; + + const gates: Record = { + minimum_fixtures: fixtureCount >= policy.minFixtures, + paired_completeness: pairs.length === baselineRuns.length && pairs.length === candidateRuns.length, + no_new_failures: comparison.newFailures <= policy.maxNewFailures, + quality_non_regression: qualityNonRegression, + score_non_regression: scoreNonRegression, + confidence_non_regression: confidenceNonRegression, + token_budget: comparison.totalTokenRatio <= policy.maxTokenRegressionRatio, + tool_budget: comparison.toolCallRatio <= policy.maxToolCallRegressionRatio, + trace_integrity: candidate.traceMatchRate >= baseline.traceMatchRate && candidate.safetyViolations <= baseline.safetyViolations, + improvement: !policy.requireImprovement || qualityImprovement, + }; + const reasons = Object.entries(gates) + .filter(([, passed]) => !passed) + .map(([gate]) => `Gate failed: ${gate}`); + if (reasons.length === 0) reasons.push("All evaluation gates passed"); + return { + baseline, + candidate, + comparison, + decision: { passed: Object.values(gates).every(Boolean), gates, reasons }, + }; +} + +export function defaultEvaluationPolicy(input: Partial = {}): EvaluationPolicy { + return { + minFixtures: input.minFixtures ?? 3, + repeats: input.repeats ?? 1, + maxNewFailures: input.maxNewFailures ?? 0, + maxSuccessRegression: input.maxSuccessRegression ?? 0, + maxScoreRegression: input.maxScoreRegression ?? 0.02, + minSuccessImprovement: input.minSuccessImprovement ?? 0.05, + minScoreImprovement: input.minScoreImprovement ?? 0.02, + maxTokenRegressionRatio: input.maxTokenRegressionRatio ?? 1.2, + maxToolCallRegressionRatio: input.maxToolCallRegressionRatio ?? 1.2, + efficiencyImprovementRatio: input.efficiencyImprovementRatio ?? 0.9, + confidenceLevel: input.confidenceLevel ?? 0.9, + bootstrapSamples: input.bootstrapSamples ?? 1_000, + requireImprovement: input.requireImprovement ?? true, + }; +} diff --git a/evolve-agent/src/evaluation/provenance-signer.ts b/evolve-agent/src/evaluation/provenance-signer.ts new file mode 100644 index 0000000..500c855 --- /dev/null +++ b/evolve-agent/src/evaluation/provenance-signer.ts @@ -0,0 +1,71 @@ +import { generateKeyPairSync, sign, verify } from "node:crypto"; +import { chmod, readFile } from "node:fs/promises"; +import path from "node:path"; +import { atomicWriteFile, pathExists } from "../core/fs.js"; +import { sha256Bytes, sha256Json } from "../core/hash.js"; +import { stableStringify } from "../core/stable-json.js"; +import type { EvaluationReportPayload, SignedEvaluationReport } from "./types.js"; + +export class ProvenanceSigner { + private readonly privatePath: string; + private readonly publicPath: string; + private initialization?: Promise; + + public constructor(home: string) { + const directory = path.join(home, "evaluations", "authority"); + this.privatePath = path.join(directory, "ed25519-private.pem"); + this.publicPath = path.join(directory, "ed25519-public.pem"); + } + + private async ensureKeys(): Promise { + this.initialization ??= (async () => { + const privateExists = await pathExists(this.privatePath); + const publicExists = await pathExists(this.publicPath); + if (privateExists !== publicExists) throw new Error("Evaluation signing keypair is incomplete"); + if (!privateExists) { + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + await atomicWriteFile(this.privatePath, privateKey.export({ type: "pkcs8", format: "pem" }), 0o600); + await atomicWriteFile(this.publicPath, publicKey.export({ type: "spki", format: "pem" }), 0o644); + } + await chmod(this.privatePath, 0o600); + })(); + await this.initialization; + } + + public async publicKeyFingerprint(): Promise { + await this.ensureKeys(); + return sha256Bytes(await readFile(this.publicPath)).slice(0, 32); + } + + public async sign(payload: EvaluationReportPayload): Promise { + await this.ensureKeys(); + const payloadHash = sha256Json(payload); + const privateKey = await readFile(this.privatePath, "utf8"); + const signature = sign(null, Buffer.from(stableStringify(payload), "utf8"), privateKey).toString("base64url"); + return { + id: `report_${payloadHash.slice(0, 24)}`, + payload, + payloadHash, + signature, + keyFingerprint: await this.publicKeyFingerprint(), + }; + } + + public async verify(report: SignedEvaluationReport): Promise { + await this.ensureKeys(); + if (report.id !== `report_${report.payloadHash.slice(0, 24)}`) return false; + if (sha256Json(report.payload) !== report.payloadHash) return false; + if (report.keyFingerprint !== (await this.publicKeyFingerprint())) return false; + const publicKey = await readFile(this.publicPath, "utf8"); + try { + return verify( + null, + Buffer.from(stableStringify(report.payload), "utf8"), + publicKey, + Buffer.from(report.signature, "base64url"), + ); + } catch { + return false; + } + } +} diff --git a/evolve-agent/src/evaluation/replay-harness.ts b/evolve-agent/src/evaluation/replay-harness.ts new file mode 100644 index 0000000..2480cff --- /dev/null +++ b/evolve-agent/src/evaluation/replay-harness.ts @@ -0,0 +1,200 @@ +import { performance } from "node:perf_hooks"; +import { sha256Json } from "../core/hash.js"; +import type { EvidenceRecord, SkillRecord, Usage } from "../core/types.js"; +import type { AgentPrompt, AgentProvider, Observation } from "../providers/provider.js"; +import type { ToolDescription } from "../tools/types.js"; +import type { ReplayHarnessInput, ReplayRunResult } from "./types.js"; + +const EVIDENCE_PATTERN = /ev_[a-f0-9]{24}/g; + +function addUsage(left: Usage, right: Usage): Usage { + return { + inputTokens: left.inputTokens + right.inputTokens, + outputTokens: left.outputTokens + right.outputTokens, + totalTokens: left.totalTokens + right.totalTokens, + }; +} + +function failure( + input: ReplayHarnessInput, + startedAt: number, + usage: Usage, + turns: number, + toolCalls: number, + reason: string, + safetyViolations: string[] = [], +): ReplayRunResult { + return { + fixtureId: input.fixture.id, + fixtureHash: input.fixture.integrityHash, + sourceEpisodeId: input.fixture.sourceEpisodeId, + arm: input.arm, + repeat: input.repeat, + success: false, + verifierScore: 0, + usage, + turns, + toolCalls, + durationMs: Math.max(0, performance.now() - startedAt), + traceMatched: false, + failureReason: reason, + safetyViolations, + activeSkillIds: input.skills.map((skill) => skill.id).sort(), + }; +} + +export class ReplayHarness { + public constructor( + private readonly provider: AgentProvider, + private readonly tools: ToolDescription[], + ) {} + + public async run(input: ReplayHarnessInput): Promise { + const startedAt = performance.now(); + const observations: Observation[] = []; + const permittedEvidence = new Map(); + let usage: Usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; + let turns = 0; + let toolCalls = 0; + let traceIndex = 0; + const maxTurns = Math.max(1, input.fixture.task.budget.maxTurns); + const selectedTools = this.tools.filter((tool) => input.fixture.task.requestedTools.includes(tool.name)); + + while (turns < maxTurns) { + const prompt: AgentPrompt = { + task: input.fixture.task, + observations: observations.slice(-20), + memories: [], + skills: input.skills, + tools: selectedTools, + remainingBudget: { + turns: maxTurns - turns, + toolCalls: Math.max(0, input.fixture.task.budget.maxToolCalls - toolCalls), + inputTokens: Math.max(0, input.fixture.task.budget.maxInputTokens - usage.inputTokens), + outputTokens: Math.max(0, input.fixture.task.budget.maxOutputTokens - usage.outputTokens), + wallTimeMs: input.fixture.task.budget.maxWallTimeMs, + }, + }; + const decisionResult = await this.provider.decide(prompt); + turns += 1; + usage = addUsage(usage, decisionResult.usage); + + if (decisionResult.decision.kind === "tool") { + const decision = decisionResult.decision; + const expected = input.fixture.trace[traceIndex]; + if (!expected) { + return failure(input, startedAt, usage, turns, toolCalls, `Unexpected extra tool ${decision.toolName}`, [ + "unexpected_tool", + ]); + } + const argsHash = sha256Json(decision.args); + if (decision.toolName !== expected.toolName || argsHash !== expected.argsHash) { + return failure( + input, + startedAt, + usage, + turns, + toolCalls, + `Trace mismatch at step ${traceIndex}: expected ${expected.toolName}/${expected.argsHash}, received ${decision.toolName}/${argsHash}`, + ["trace_mismatch"], + ); + } + toolCalls += 1; + traceIndex += 1; + permittedEvidence.set(expected.evidence.id, expected.evidence); + observations.push({ + id: `replay_${input.fixture.id}_${traceIndex}`, + kind: expected.success ? "tool" : "error", + content: expected.observation, + evidenceId: expected.evidence.id, + toolName: expected.toolName, + }); + continue; + } + + const final = decisionResult.decision; + if (traceIndex !== input.fixture.trace.length) { + return failure( + input, + startedAt, + usage, + turns, + toolCalls, + `Final answer arrived before replay trace completed (${traceIndex}/${input.fixture.trace.length})`, + ["incomplete_trace"], + ); + } + const declared = [...new Set(final.evidenceIds)]; + const invalid = declared.filter((id) => !permittedEvidence.has(id)); + if (invalid.length > 0) { + return failure(input, startedAt, usage, turns, toolCalls, `Final answer declared unavailable evidence: ${invalid.join(", ")}`, [ + "fabricated_evidence", + ]); + } + const cited = [...new Set(final.answer.match(EVIDENCE_PATTERN) ?? [])]; + const undeclared = cited.filter((id) => !declared.includes(id)); + if (undeclared.length > 0 || (declared.length > 0 && cited.length === 0)) { + return failure(input, startedAt, usage, turns, toolCalls, "Final answer evidence declaration and citations disagree", [ + "evidence_contract", + ]); + } + const evidence = declared.map((id) => permittedEvidence.get(id)).filter((record): record is EvidenceRecord => Boolean(record)); + const verification = await this.provider.verify({ task: input.fixture.task, answer: final.answer, evidence }); + usage = addUsage(usage, verification.usage); + const success = verification.verdict.passed && verification.verdict.score >= 0.8; + return { + fixtureId: input.fixture.id, + fixtureHash: input.fixture.integrityHash, + sourceEpisodeId: input.fixture.sourceEpisodeId, + arm: input.arm, + repeat: input.repeat, + success, + verifierScore: verification.verdict.score, + usage, + turns, + toolCalls, + durationMs: Math.max(0, performance.now() - startedAt), + traceMatched: true, + ...(!success ? { failureReason: verification.verdict.feedback } : {}), + safetyViolations: [], + activeSkillIds: input.skills.map((skill) => skill.id).sort(), + }; + } + + return failure(input, startedAt, usage, turns, toolCalls, `Replay exhausted ${maxTurns} turns`, ["turn_budget"]); + } + + public static actualBaseline(input: { + fixtureId: string; + fixtureHash: string; + sourceEpisodeId: string; + repeat?: number; + verifierScore: number; + usage: Usage; + turns: number; + toolCalls: number; + durationMs: number; + activeSkillIds: string[]; + arm: "baseline" | "candidate"; + success?: boolean; + failureReason?: string; + }): ReplayRunResult { + return { + fixtureId: input.fixtureId, + fixtureHash: input.fixtureHash, + sourceEpisodeId: input.sourceEpisodeId, + arm: input.arm, + repeat: input.repeat ?? 0, + success: input.success ?? true, + verifierScore: input.verifierScore, + usage: input.usage, + turns: input.turns, + toolCalls: input.toolCalls, + durationMs: input.durationMs, + traceMatched: true, + ...(input.failureReason ? { failureReason: input.failureReason } : {}), + safetyViolations: [], + activeSkillIds: [...input.activeSkillIds].sort(), + }; + } +} diff --git a/evolve-agent/src/evaluation/report-store.ts b/evolve-agent/src/evaluation/report-store.ts new file mode 100644 index 0000000..35e2d3d --- /dev/null +++ b/evolve-agent/src/evaluation/report-store.ts @@ -0,0 +1,78 @@ +import { readdir } from "node:fs/promises"; +import path from "node:path"; +import { atomicWriteJson, ensureDir, readJsonFile } from "../core/fs.js"; +import { EvolveError } from "../core/errors.js"; +import { sha256Json } from "../core/hash.js"; +import type { SkillRecord } from "../core/types.js"; +import type { PromotionAuthorization } from "./types.js"; +import type { EvaluationReportPayload, SignedEvaluationReport } from "./types.js"; +import type { ProvenanceSigner } from "./provenance-signer.js"; + +const REPORT_ID = /^report_[a-f0-9]{24}$/; + +export class EvaluationReportStore { + private readonly directory: string; + + public constructor(home: string, private readonly signer: ProvenanceSigner) { + this.directory = path.join(home, "evaluations", "reports"); + } + + private pathFor(id: string): string { + if (!REPORT_ID.test(id)) throw new EvolveError("REPORT_ID", `Invalid report ID: ${id}`); + return path.join(this.directory, `${id}.json`); + } + + public async create(payload: EvaluationReportPayload): Promise { + const report = await this.signer.sign(payload); + await atomicWriteJson(this.pathFor(report.id), report, 0o600); + return report; + } + + public async get(id: string): Promise { + const report = await readJsonFile(this.pathFor(id), null); + if (!report) throw new EvolveError("REPORT_NOT_FOUND", `Unknown evaluation report: ${id}`); + return report; + } + + public async verify(idOrReport: string | SignedEvaluationReport): Promise { + const report = typeof idOrReport === "string" ? await this.get(idOrReport) : idOrReport; + return this.signer.verify(report); + } + + public async requireVerified(id: string): Promise { + const report = await this.get(id); + if (!(await this.verify(report))) throw new EvolveError("REPORT_SIGNATURE", `Evaluation report signature failed: ${id}`); + return report; + } + + public async verifyPromotion(skill: SkillRecord, authorization: PromotionAuthorization): Promise { + const offline = await this.requireVerified(authorization.offlineReportId); + const canary = await this.requireVerified(authorization.canaryReportId); + if (offline.payload.kind !== "offline" || canary.payload.kind !== "canary") { + throw new EvolveError("SKILL_GATE", "Promotion authorization report kinds are invalid"); + } + for (const report of [offline, canary]) { + if (report.payload.skillId !== skill.id || report.payload.skillFingerprint !== skill.fingerprint) { + throw new EvolveError("SKILL_GATE", "Promotion report provenance does not match this Skill"); + } + if (!report.payload.decision.passed) { + throw new EvolveError("SKILL_GATE", `${report.payload.kind} evaluation did not pass`); + } + if (report.keyFingerprint !== authorization.keyFingerprint) { + throw new EvolveError("SKILL_GATE", "Promotion authority fingerprint does not match the signed reports"); + } + } + const policyHash = sha256Json({ offline: offline.payload.policy, canary: canary.payload.policy }); + if (policyHash !== authorization.policyHash) { + throw new EvolveError("SKILL_GATE", "Promotion policy hash does not match the signed reports"); + } + } + + public async list(): Promise { + await ensureDir(this.directory); + const files = (await readdir(this.directory)).filter((file) => /^report_[a-f0-9]{24}\.json$/.test(file)).sort(); + const reports: SignedEvaluationReport[] = []; + for (const file of files) reports.push(await this.get(file.slice(0, -5))); + return reports; + } +} diff --git a/evolve-agent/src/evaluation/shadow-store.ts b/evolve-agent/src/evaluation/shadow-store.ts new file mode 100644 index 0000000..e523da7 --- /dev/null +++ b/evolve-agent/src/evaluation/shadow-store.ts @@ -0,0 +1,67 @@ +import path from "node:path"; +import { atomicWriteJson, readJsonFile } from "../core/fs.js"; +import type { ProductionOutcome, ShadowObservation } from "./types.js"; + +interface ShadowFile { + version: 1; + observations: ShadowObservation[]; + production: ProductionOutcome[]; +} + +export class ShadowStore { + private readonly filePath: string; + + public constructor(home: string, private readonly maxWindow: number) { + this.filePath = path.join(home, "evaluations", "shadow.json"); + } + + private async load(): Promise { + const file = await readJsonFile(this.filePath, { version: 1, observations: [], production: [] }); + if (file.version !== 1 || !Array.isArray(file.observations) || !Array.isArray(file.production)) { + throw new Error("Unsupported shadow evaluation store"); + } + return file; + } + + private async save(file: ShadowFile): Promise { + await atomicWriteJson(this.filePath, file, 0o600); + } + + public async addObservation(observation: ShadowObservation): Promise { + const file = await this.load(); + const existing = file.observations.find( + (entry) => entry.skillId === observation.skillId && entry.episodeId === observation.episodeId && entry.mode === observation.mode, + ); + if (existing) return existing; + file.observations.push(observation); + file.observations = this.trim(file.observations, (entry) => entry.skillId); + await this.save(file); + return observation; + } + + public async observations(skillId: string, mode?: ShadowObservation["mode"]): Promise { + return (await this.load()).observations.filter( + (entry) => entry.skillId === skillId && (mode === undefined || entry.mode === mode), + ); + } + + public async addProduction(outcome: ProductionOutcome): Promise { + const file = await this.load(); + const existing = file.production.find((entry) => entry.skillId === outcome.skillId && entry.episodeId === outcome.episodeId); + if (existing) return existing; + file.production.push(outcome); + file.production = this.trim(file.production, (entry) => entry.skillId); + await this.save(file); + return outcome; + } + + public async production(skillId: string): Promise { + return (await this.load()).production.filter((entry) => entry.skillId === skillId); + } + + private trim(records: T[], key: (record: T) => string): T[] { + const grouped = new Map(); + for (const record of records) grouped.set(key(record), [...(grouped.get(key(record)) ?? []), record]); + return [...grouped.values()].flatMap((entries) => entries.slice(-this.maxWindow)); + } +} diff --git a/evolve-agent/src/evaluation/types.ts b/evolve-agent/src/evaluation/types.ts new file mode 100644 index 0000000..9cd279f --- /dev/null +++ b/evolve-agent/src/evaluation/types.ts @@ -0,0 +1,186 @@ +import type { EvidenceRecord, SkillRecord, TaskSpec, Usage } from "../core/types.js"; + +export type FixtureSplit = "train" | "validation" | "holdout"; +export type EvaluationReportKind = "offline" | "canary" | "monitor"; +export type EvaluationArm = "baseline" | "candidate"; + +export interface ReplayTraceStep { + index: number; + toolName: string; + argsHash: string; + evidence: EvidenceRecord; + observation: string; + success: boolean; +} + +export interface ReplayFixturePayload { + version: 1; + sourceEpisodeId: string; + split: FixtureSplit; + createdAt: string; + task: TaskSpec; + trace: ReplayTraceStep[]; + baseline: { + status: "committed"; + answerHash: string; + verifierScore: number; + usage: Usage; + turns: number; + toolCalls: number; + elapsedMs: number; + activeSkillIds: string[]; + }; +} + +export interface ReplayFixture extends ReplayFixturePayload { + id: string; + integrityHash: string; +} + +export interface ReplayRunResult { + fixtureId: string; + fixtureHash: string; + sourceEpisodeId: string; + arm: EvaluationArm; + repeat: number; + success: boolean; + verifierScore: number; + usage: Usage; + turns: number; + toolCalls: number; + durationMs: number; + traceMatched: boolean; + failureReason?: string; + safetyViolations: string[]; + activeSkillIds: string[]; +} + +export interface EvaluationAggregate { + samples: number; + successes: number; + successRate: number; + meanVerifierScore: number; + meanInputTokens: number; + meanOutputTokens: number; + meanTotalTokens: number; + meanTurns: number; + meanToolCalls: number; + meanDurationMs: number; + traceMatchRate: number; + safetyViolations: number; +} + +export interface EvaluationPolicy { + minFixtures: number; + repeats: number; + maxNewFailures: number; + maxSuccessRegression: number; + maxScoreRegression: number; + minSuccessImprovement: number; + minScoreImprovement: number; + maxTokenRegressionRatio: number; + maxToolCallRegressionRatio: number; + efficiencyImprovementRatio: number; + confidenceLevel: number; + bootstrapSamples: number; + requireImprovement: boolean; +} + +export interface EvaluationComparison { + pairedSamples: number; + wins: number; + losses: number; + ties: number; + newFailures: number; + successDelta: number; + verifierScoreDelta: number; + totalTokenRatio: number; + toolCallRatio: number; + durationRatio: number; + confidence: { + successDeltaLower: number; + successDeltaUpper: number; + verifierScoreDeltaLower: number; + verifierScoreDeltaUpper: number; + }; +} + +export interface EvaluationDecision { + passed: boolean; + gates: Record; + reasons: string[]; +} + +export interface EvaluationReportPayload { + version: 1; + engineVersion: "0.3.0"; + kind: EvaluationReportKind; + skillId: string; + skillFingerprint: string; + createdAt: string; + fixtureIds: string[]; + fixtureHashes: string[]; + policy: EvaluationPolicy; + baselineRuns: ReplayRunResult[]; + candidateRuns: ReplayRunResult[]; + baseline: EvaluationAggregate; + candidate: EvaluationAggregate; + comparison: EvaluationComparison; + decision: EvaluationDecision; + metadata: { + provider: string; + notes: string[]; + }; +} + +export interface SignedEvaluationReport { + id: string; + payload: EvaluationReportPayload; + payloadHash: string; + signature: string; + keyFingerprint: string; +} + +export interface ShadowObservation { + id: string; + skillId: string; + episodeId: string; + fixtureId: string; + mode: "production-baseline" | "production-candidate"; + baseline: ReplayRunResult; + candidate: ReplayRunResult; + createdAt: string; +} + +export interface ProductionOutcome { + id: string; + skillId: string; + episodeId: string; + createdAt: string; + success: boolean; + verifierScore: number; + usage: Usage; + turns: number; + toolCalls: number; + status: string; +} + +export interface PromotionAuthorization { + offlineReportId: string; + canaryReportId: string; + policyHash: string; + keyFingerprint: string; +} + +export interface EvaluationSelection { + fixtures?: string[]; + splits?: FixtureSplit[]; + repeats?: number; +} + +export interface ReplayHarnessInput { + fixture: ReplayFixture; + arm: EvaluationArm; + repeat: number; + skills: SkillRecord[]; +} diff --git a/evolve-agent/src/execution/docker-executor.ts b/evolve-agent/src/execution/docker-executor.ts new file mode 100644 index 0000000..1eb986c --- /dev/null +++ b/evolve-agent/src/execution/docker-executor.ts @@ -0,0 +1,345 @@ +import path from "node:path"; +import { sha256Bytes, sha256Json } from "../core/hash.js"; +import { EvolveError } from "../core/errors.js"; +import type { SecretBroker, SecretMount } from "../secrets/secret-broker.js"; +import { ImagePolicy } from "./image-policy.js"; +import { DockerNetworkPolicy } from "./network-policy.js"; +import type { ProcessRunner, ProcessRunResult } from "./process-runner.js"; +import { dockerClientEnvironment } from "./safe-environment.js"; +import type { Executor, ExecutorProbe, ExecutionRequest, ExecutionResult, ResourceLimits } from "./types.js"; + +export interface DockerExecutorOptions { + binary: string; + user: string; + imagePolicy: ImagePolicy; + networkPolicy: DockerNetworkPolicy; + maximums: ResourceLimits; + requireRootless: boolean; +} + +function within(limit: number, maximum: number, label: string): number { + if (!Number.isFinite(limit) || limit <= 0 || limit > maximum) { + throw new EvolveError("EXECUTION_LIMIT_DENIED", `${label} must be positive and no greater than ${maximum}`); + } + return limit; +} + +function mountValue(value: string): string { + if (value.includes("\0") || value.includes("\n") || value.includes("\r") || value.includes(",")) { + throw new EvolveError("DOCKER_MOUNT_INVALID", "Docker mount path contains a forbidden character"); + } + return value; +} + +function containerWorkdir(workspace: string, cwd: string): string { + const relative = path.relative(workspace, cwd); + if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new EvolveError("PROCESS_CWD_INVALID", "cwd escapes the configured workspace"); + } + return relative === "" ? "/workspace" : `/workspace/${relative.split(path.sep).join("/")}`; +} + +function containerName(runId: string): string { + return `evolve-${sha256Bytes(runId).slice(0, 24)}`; +} + +function resourceArgs(limits: ResourceLimits, maximums: ResourceLimits): string[] { + const memoryMb = Math.floor(within(limits.memoryMb, maximums.memoryMb, "memory_mb")); + const cpus = within(limits.cpus, maximums.cpus, "cpus"); + const pids = Math.floor(within(limits.pids, maximums.pids, "pids_limit")); + const tmpfsMb = Math.floor(within(limits.tmpfsMb, maximums.tmpfsMb, "tmpfs_mb")); + return [ + "--memory", + `${memoryMb}m`, + "--memory-swap", + `${memoryMb}m`, + "--cpus", + String(cpus), + "--pids-limit", + String(pids), + "--tmpfs", + `/tmp:rw,nosuid,nodev,size=${tmpfsMb}m`, + ]; +} + +export function buildDockerRunArgs(input: { + request: ExecutionRequest; + image: string; + network: string; + networkEnforcement: string; + user: string; + containerName: string; + secretMounts: SecretMount[]; + maximums: ResourceLimits; +}): string[] { + const workspaceMode = input.request.workspaceAccess === "read-only" ? ",readonly" : ""; + const args = [ + "run", + "--rm", + "--init", + "--pull", + "never", + "--name", + input.containerName, + "--hostname", + "evolve-sandbox", + "--stop-timeout", + "1", + "--ipc", + "none", + "--label", + `ai.evolve.run=${input.request.runId}`, + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--network", + input.network, + "--user", + input.user, + "--workdir", + containerWorkdir(input.request.workspace, input.request.cwd), + "--ulimit", + "nofile=1024:1024", + "--ulimit", + "core=0:0", + "--log-driver", + "none", + ...resourceArgs(input.request.limits, input.maximums), + "--tmpfs", + "/run:rw,nosuid,nodev,noexec,size=16m", + "--mount", + `type=bind,src=${mountValue(input.request.workspace)},dst=/workspace${workspaceMode}`, + "--env", + "HOME=/tmp/home", + "--env", + "NO_COLOR=1", + "--env", + "CI=1", + "--env", + `EVOLVE_NETWORK_POLICY=${input.networkEnforcement}`, + ]; + + for (const secret of input.secretMounts) { + args.push( + "--mount", + `type=bind,src=${mountValue(secret.hostPath)},dst=${secret.containerPath},readonly`, + "--env", + `${secret.name}_FILE=${secret.containerPath}`, + ); + } + + args.push(input.image, input.request.command, ...input.request.args); + return args; +} + +function parsedSecurityOptions(output: string): string[] { + try { + const parsed = JSON.parse(output) as unknown; + if (Array.isArray(parsed)) return parsed.filter((item): item is string => typeof item === "string"); + } catch { + // Fall through to a conservative single-string representation. + } + return output ? [output] : []; +} + +function hasSecurityOption(options: string[], expected: string): boolean { + return options.some((item) => item.toLowerCase().includes(expected)); +} + +export class DockerExecutor implements Executor { + public readonly kind = "docker" as const; + + public constructor( + private readonly runner: ProcessRunner, + private readonly secrets: SecretBroker, + private readonly options: DockerExecutorOptions, + ) { + const match = /^(\d+):(\d+)$/.exec(options.user); + if (!match || match[1] === "0" || match[2] === "0") { + throw new EvolveError("DOCKER_USER_INVALID", "EVOLVE_DOCKER_USER must be a non-root numeric uid:gid"); + } + } + + private async cleanup(name: string, workspace: string): Promise { + try { + await this.runner.run({ + command: this.options.binary, + args: ["rm", "-f", name], + cwd: workspace, + env: dockerClientEnvironment(), + timeoutMs: 10_000, + maxOutputBytes: 16_384, + }); + } catch { + // The primary execution result remains authoritative. Orphan cleanup is best effort. + } + } + + public async execute(request: ExecutionRequest): Promise { + const image = this.options.imagePolicy.resolve(request.image); + const network = this.options.networkPolicy.resolve(request.network); + const name = containerName(request.runId); + const secretLease = await this.secrets.materialize(request.runId, request.secretNames); + let result: ProcessRunResult | undefined; + try { + const args = buildDockerRunArgs({ + request, + image, + network: network.name, + networkEnforcement: network.enforcement, + user: this.options.user, + containerName: name, + secretMounts: secretLease.mounts, + maximums: this.options.maximums, + }); + result = await this.runner.run({ + command: this.options.binary, + args, + cwd: request.workspace, + env: dockerClientEnvironment(), + timeoutMs: request.timeoutMs, + maxOutputBytes: request.maxOutputBytes, + }); + } finally { + await this.cleanup(name, request.workspace); + await secretLease.release(); + } + + if (!result) throw new EvolveError("DOCKER_EXECUTION_FAILED", "Docker execution produced no result"); + return { + executor: this.kind, + success: !result.timedOut && !result.truncated && result.code === 0, + exitCode: result.code, + signal: result.signal, + timedOut: result.timedOut, + truncated: result.truncated, + stdout: secretLease.redact(result.stdout), + stderr: secretLease.redact(result.stderr), + durationMs: result.durationMs, + image, + network: network.name, + workspaceAccess: request.workspaceAccess, + secretNames: [...request.secretNames].sort(), + receipt: { + runId: request.runId, + sandboxId: name, + commandHash: sha256Json({ command: request.command, args: request.args }), + policyHash: sha256Json({ + executor: this.kind, + image, + network: network.name, + networkEnforcement: network.enforcement, + workspaceAccess: request.workspaceAccess, + secretNames: [...request.secretNames].sort(), + limits: request.limits, + user: this.options.user, + readOnlyRoot: true, + capabilitiesDropped: true, + noNewPrivileges: true, + }), + }, + isolation: { + boundary: "docker-container", + readOnlyRoot: true, + capabilitiesDropped: true, + noNewPrivileges: true, + resourceLimits: true, + networkPolicy: network.enforcement, + secretDelivery: request.secretNames.length > 0 ? "ephemeral-read-only-files" : "none", + }, + }; + } + + public async probe(): Promise { + const version = await this.runner.run({ + command: this.options.binary, + args: ["version", "--format", "{{.Server.Version}}"], + cwd: process.cwd(), + env: dockerClientEnvironment(), + timeoutMs: 10_000, + maxOutputBytes: 16_384, + }); + if (version.code !== 0 || version.timedOut) { + return { + kind: this.kind, + available: false, + ready: false, + summary: "Docker daemon is unavailable", + warnings: [version.stderr.trim()].filter(Boolean), + details: { binary: this.options.binary }, + }; + } + + const info = await this.runner.run({ + command: this.options.binary, + args: ["info", "--format", "{{json .SecurityOptions}}"], + cwd: process.cwd(), + env: dockerClientEnvironment(), + timeoutMs: 10_000, + maxOutputBytes: 16_384, + }); + const securityOptions = info.code === 0 ? parsedSecurityOptions(info.stdout.trim()) : []; + const rootless = hasSecurityOption(securityOptions, "rootless"); + const seccomp = hasSecurityOption(securityOptions, "seccomp"); + const cgroup = await this.runner.run({ + command: this.options.binary, + args: ["info", "--format", "{{.CgroupVersion}}"], + cwd: process.cwd(), + env: dockerClientEnvironment(), + timeoutMs: 10_000, + maxOutputBytes: 16_384, + }); + const cgroupVersion = cgroup.code === 0 ? cgroup.stdout.trim() : "unknown"; + const cgroupAvailable = cgroupVersion === "1" || cgroupVersion === "2"; + const resourceLimitsReady = cgroupAvailable && (!rootless || cgroupVersion === "2"); + const defaultImage = this.options.imagePolicy.getDefault(); + let imageReady = false; + if (defaultImage) { + const inspect = await this.runner.run({ + command: this.options.binary, + args: ["image", "inspect", "--format", "{{.Id}}", defaultImage], + cwd: process.cwd(), + env: dockerClientEnvironment(), + timeoutMs: 10_000, + maxOutputBytes: 16_384, + }); + imageReady = inspect.code === 0 && !inspect.timedOut; + } + + const warnings: string[] = []; + if (!rootless) warnings.push("Docker daemon is not reporting rootless mode; use rootless Docker or Docker Desktop's VM boundary."); + if (!seccomp) warnings.push("Docker is not reporting the default seccomp security option; hardened readiness is blocked."); + if (!resourceLimitsReady) warnings.push("A supported cgroup boundary is required; rootless Docker specifically needs cgroup v2."); + if (!defaultImage) warnings.push("EVOLVE_DOCKER_DEFAULT_IMAGE is not configured."); + else if (!imageReady) warnings.push("The pinned default image is not present locally; --pull never prevents implicit downloads."); + if (this.options.requireRootless && !rootless) warnings.push("EVOLVE_DOCKER_REQUIRE_ROOTLESS=true blocks hardened readiness."); + const ready = Boolean( + defaultImage && + imageReady && + resourceLimitsReady && + seccomp && + (!this.options.requireRootless || rootless), + ); + return { + kind: this.kind, + available: true, + ready, + summary: ready ? "Docker hardened execution is ready" : "Docker is available but hardened execution needs configuration", + warnings, + details: { + binary: this.options.binary, + server_version: version.stdout.trim() || null, + rootless, + seccomp, + cgroup_version: cgroupVersion, + resource_limits_ready: resourceLimitsReady, + default_image_configured: Boolean(defaultImage), + default_image_present: imageReady, + allowed_images: this.options.imagePolicy.list().length, + allowed_networks: this.options.networkPolicy.list().length, + }, + }; + } +} diff --git a/evolve-agent/src/execution/executor-registry.ts b/evolve-agent/src/execution/executor-registry.ts new file mode 100644 index 0000000..52b964d --- /dev/null +++ b/evolve-agent/src/execution/executor-registry.ts @@ -0,0 +1,53 @@ +import { EvolveError } from "../core/errors.js"; +import type { Executor, ExecutorKind, ExecutorProbe, ExecutionRequest, ExecutionResult } from "./types.js"; + +export class ExecutorRegistry { + private readonly executors = new Map(); + + public constructor(private readonly defaultKind: ExecutorKind, executors: Executor[]) { + for (const executor of executors) { + if (this.executors.has(executor.kind)) throw new Error(`Duplicate executor kind: ${executor.kind}`); + this.executors.set(executor.kind, executor); + } + if (!this.executors.has(defaultKind)) throw new Error(`Default executor ${defaultKind} is not registered`); + } + + public getDefaultKind(): ExecutorKind { + return this.defaultKind; + } + + public list(): ExecutorKind[] { + return [...this.executors.keys()].sort(); + } + + public get(requested: string): Executor { + const kind = requested === "default" ? this.defaultKind : requested; + if (kind !== "docker" && kind !== "local") throw new EvolveError("EXECUTOR_UNKNOWN", `Unknown executor: ${requested}`); + const executor = this.executors.get(kind); + if (!executor) throw new EvolveError("EXECUTOR_DISABLED", `${kind} executor is not enabled`); + return executor; + } + + public async execute(requested: string, request: ExecutionRequest): Promise { + return this.get(requested).execute(request); + } + + public async probeAll(): Promise { + const probes: ExecutorProbe[] = []; + for (const executor of [...this.executors.values()].sort((left, right) => left.kind.localeCompare(right.kind))) { + try { + probes.push(await executor.probe()); + } catch (error: unknown) { + probes.push({ + kind: executor.kind, + available: false, + ready: false, + summary: error instanceof Error ? error.message : String(error), + warnings: [], + details: {}, + }); + } + } + return probes; + } +} diff --git a/evolve-agent/src/execution/image-policy.ts b/evolve-agent/src/execution/image-policy.ts new file mode 100644 index 0000000..dc14c75 --- /dev/null +++ b/evolve-agent/src/execution/image-policy.ts @@ -0,0 +1,42 @@ +import { EvolveError } from "../core/errors.js"; + +const DIGEST_REFERENCE = /^(?:[a-z0-9._/-]+(?::[a-zA-Z0-9._-]+)?@)?sha256:[a-f0-9]{64}$/; + +export class ImagePolicy { + private readonly allowed: Set; + + public constructor(allowedImages: Iterable, private readonly defaultImage?: string) { + this.allowed = new Set([...allowedImages].map((value) => value.trim()).filter(Boolean)); + if (defaultImage) this.allowed.add(defaultImage); + for (const image of this.allowed) { + if (!DIGEST_REFERENCE.test(image)) { + throw new EvolveError("DOCKER_IMAGE_UNPINNED", `Configured Docker image is not sha256-pinned: ${image}`); + } + } + } + + public resolve(requested?: string): string { + const image = requested?.trim() || this.defaultImage; + if (!image) { + throw new EvolveError( + "DOCKER_IMAGE_REQUIRED", + "No Docker image was requested and EVOLVE_DOCKER_DEFAULT_IMAGE is not configured", + ); + } + if (!DIGEST_REFERENCE.test(image)) { + throw new EvolveError("DOCKER_IMAGE_UNPINNED", "Docker images must be pinned to an exact sha256 digest"); + } + if (!this.allowed.has(image)) { + throw new EvolveError("DOCKER_IMAGE_DENIED", "Docker image is not in EVOLVE_DOCKER_ALLOWED_IMAGES"); + } + return image; + } + + public list(): string[] { + return [...this.allowed].sort(); + } + + public getDefault(): string | undefined { + return this.defaultImage; + } +} diff --git a/evolve-agent/src/execution/local-executor.ts b/evolve-agent/src/execution/local-executor.ts new file mode 100644 index 0000000..8d18e76 --- /dev/null +++ b/evolve-agent/src/execution/local-executor.ts @@ -0,0 +1,87 @@ +import { EvolveError } from "../core/errors.js"; +import { sha256Bytes, sha256Json } from "../core/hash.js"; +import { safeEnvironment } from "./safe-environment.js"; +import type { Executor, ExecutorProbe, ExecutionRequest, ExecutionResult } from "./types.js"; +import type { ProcessRunner } from "./process-runner.js"; + +export class LocalExecutor implements Executor { + public readonly kind = "local" as const; + + public constructor(private readonly runner: ProcessRunner) {} + + public async execute(request: ExecutionRequest): Promise { + if (request.image !== undefined) throw new EvolveError("LOCAL_IMAGE_INVALID", "The local executor does not accept images"); + if (request.secretNames.length > 0) { + throw new EvolveError("LOCAL_SECRETS_DENIED", "Secrets are available only through the isolated Docker executor"); + } + if (request.network !== "host") { + throw new EvolveError( + "LOCAL_NETWORK_UNENFORCEABLE", + "The local executor cannot enforce network isolation; request network=host explicitly or use Docker", + ); + } + if (request.workspaceAccess !== "read-write") { + throw new EvolveError( + "LOCAL_WORKSPACE_UNENFORCEABLE", + "The local executor cannot enforce a read-only workspace; request workspace_access=read-write explicitly or use Docker", + ); + } + + const result = await this.runner.run({ + command: request.command, + args: request.args, + cwd: request.cwd, + env: safeEnvironment(request.workspace), + timeoutMs: request.timeoutMs, + maxOutputBytes: request.maxOutputBytes, + }); + return { + executor: this.kind, + success: !result.timedOut && !result.truncated && result.code === 0, + exitCode: result.code, + signal: result.signal, + timedOut: result.timedOut, + truncated: result.truncated, + stdout: result.stdout, + stderr: result.stderr, + durationMs: result.durationMs, + network: request.network, + workspaceAccess: "read-write", + secretNames: [], + receipt: { + runId: request.runId, + sandboxId: `local-${sha256Bytes(request.runId).slice(0, 24)}`, + commandHash: sha256Json({ command: request.command, args: request.args }), + policyHash: sha256Json({ + executor: this.kind, + network: request.network, + workspaceAccess: "read-write", + isolation: false, + }), + }, + isolation: { + boundary: "none", + readOnlyRoot: false, + capabilitiesDropped: false, + noNewPrivileges: false, + resourceLimits: false, + networkPolicy: "unenforced-host-network", + secretDelivery: "disabled", + }, + }; + } + + public async probe(): Promise { + return { + kind: this.kind, + available: true, + ready: true, + summary: "Local execution is available but is not an isolation boundary", + warnings: [ + "Commands run with the current host user's permissions.", + "Network and read-only workspace policies cannot be enforced.", + ], + details: { isolation: false }, + }; + } +} diff --git a/evolve-agent/src/execution/network-policy.ts b/evolve-agent/src/execution/network-policy.ts new file mode 100644 index 0000000..eecd42b --- /dev/null +++ b/evolve-agent/src/execution/network-policy.ts @@ -0,0 +1,45 @@ +import { EvolveError } from "../core/errors.js"; + +const NETWORK_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/; +const RESERVED_UNSAFE = new Set(["host", "bridge", "default", "container"]); + +export interface ResolvedNetwork { + name: string; + enforcement: string; +} + +export class DockerNetworkPolicy { + private readonly allowed: Set; + + public constructor(allowedNetworks: Iterable) { + this.allowed = new Set(["none", ...[...allowedNetworks].map((value) => value.trim()).filter(Boolean)]); + for (const network of this.allowed) { + if (!NETWORK_NAME.test(network)) throw new EvolveError("DOCKER_NETWORK_INVALID", `Invalid configured network: ${network}`); + if (RESERVED_UNSAFE.has(network)) { + throw new EvolveError("DOCKER_NETWORK_DENIED", `Unsafe network cannot be allowlisted: ${network}`); + } + } + } + + public resolve(requested: string): ResolvedNetwork { + const network = requested.trim() || "none"; + if (!NETWORK_NAME.test(network)) throw new EvolveError("DOCKER_NETWORK_INVALID", "Invalid Docker network name"); + if (RESERVED_UNSAFE.has(network)) { + throw new EvolveError( + "DOCKER_NETWORK_DENIED", + `${network} bypasses the hardened network boundary; use none or an operator-managed network`, + ); + } + if (!this.allowed.has(network)) { + throw new EvolveError("DOCKER_NETWORK_DENIED", "Docker network is not in EVOLVE_DOCKER_ALLOWED_NETWORKS"); + } + return { + name: network, + enforcement: network === "none" ? "deny-all" : `operator-managed:${network}`, + }; + } + + public list(): string[] { + return [...this.allowed].sort(); + } +} diff --git a/evolve-agent/src/execution/process-runner.ts b/evolve-agent/src/execution/process-runner.ts new file mode 100644 index 0000000..8dd1bf6 --- /dev/null +++ b/evolve-agent/src/execution/process-runner.ts @@ -0,0 +1,106 @@ +import { spawn } from "node:child_process"; + +export interface ProcessRunRequest { + command: string; + args: string[]; + cwd: string; + env: NodeJS.ProcessEnv; + timeoutMs: number; + maxOutputBytes: number; +} + +export interface ProcessRunResult { + code: number | null; + signal: string | null; + timedOut: boolean; + truncated: boolean; + stdout: string; + stderr: string; + durationMs: number; +} + +export interface ProcessRunner { + run(request: ProcessRunRequest): Promise; +} + +function terminate(pid: number | undefined, signal: NodeJS.Signals, detached: boolean): void { + if (pid === undefined) return; + try { + process.kill(detached ? -pid : pid, signal); + } catch { + // The process may already have exited or the platform may reject group signals. + } +} + +export class NodeProcessRunner implements ProcessRunner { + public async run(request: ProcessRunRequest): Promise { + const startedAt = Date.now(); + const detached = process.platform !== "win32"; + const child = spawn(request.command, request.args, { + cwd: request.cwd, + shell: false, + windowsHide: true, + detached, + env: request.env, + stdio: ["ignore", "pipe", "pipe"], + }); + + let capturedBytes = 0; + let truncated = false; + let timedOut = false; + let forceTimer: NodeJS.Timeout | undefined; + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + + const stopProcess = (): void => { + terminate(child.pid, "SIGTERM", detached); + if (forceTimer === undefined) { + forceTimer = setTimeout(() => terminate(child.pid, "SIGKILL", detached), 1_000); + forceTimer.unref(); + } + }; + + const append = (chunks: Buffer[], chunk: Buffer): void => { + const remaining = request.maxOutputBytes - capturedBytes; + if (remaining <= 0) { + truncated = true; + stopProcess(); + return; + } + const accepted = chunk.subarray(0, remaining); + chunks.push(accepted); + capturedBytes += accepted.length; + if (accepted.length < chunk.length) { + truncated = true; + stopProcess(); + } + }; + + child.stdout.on("data", (chunk: Buffer) => append(stdoutChunks, chunk)); + child.stderr.on("data", (chunk: Buffer) => append(stderrChunks, chunk)); + + const timeout = setTimeout(() => { + timedOut = true; + stopProcess(); + }, request.timeoutMs); + timeout.unref(); + + const result = await new Promise<{ code: number | null; signal: string | null }>((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal })); + }).finally(() => { + clearTimeout(timeout); + if (forceTimer !== undefined) clearTimeout(forceTimer); + }); + + return { + code: result.code, + signal: result.signal, + timedOut, + truncated, + stdout: Buffer.concat(stdoutChunks).toString("utf8"), + stderr: Buffer.concat(stderrChunks).toString("utf8"), + durationMs: Date.now() - startedAt, + }; + } +} diff --git a/evolve-agent/src/execution/safe-environment.ts b/evolve-agent/src/execution/safe-environment.ts new file mode 100644 index 0000000..8db2344 --- /dev/null +++ b/evolve-agent/src/execution/safe-environment.ts @@ -0,0 +1,39 @@ +function selectedEnvironment(keys: string[]): NodeJS.ProcessEnv { + const output: NodeJS.ProcessEnv = {}; + for (const key of keys) { + const value = process.env[key]; + if (value !== undefined) output[key] = value; + } + return output; +} + +export function safeEnvironment(home: string): NodeJS.ProcessEnv { + const output = selectedEnvironment(["PATH", "TMPDIR", "TEMP", "TMP", "LANG", "LC_ALL", "CI"]); + output.HOME = home; + output.NO_COLOR = "1"; + output.CI = output.CI ?? "1"; + return output; +} + +export function dockerClientEnvironment(): NodeJS.ProcessEnv { + const output = selectedEnvironment([ + "PATH", + "TMPDIR", + "TEMP", + "TMP", + "LANG", + "LC_ALL", + "CI", + "HOME", + "XDG_RUNTIME_DIR", + "DOCKER_HOST", + "DOCKER_CONTEXT", + "DOCKER_CONFIG", + "SSH_AUTH_SOCK", + "DOCKER_TLS_VERIFY", + "DOCKER_CERT_PATH", + ]); + output.NO_COLOR = "1"; + output.CI = output.CI ?? "1"; + return output; +} diff --git a/evolve-agent/src/execution/types.ts b/evolve-agent/src/execution/types.ts new file mode 100644 index 0000000..2720f31 --- /dev/null +++ b/evolve-agent/src/execution/types.ts @@ -0,0 +1,70 @@ +export type ExecutorKind = "docker" | "local"; +export type WorkspaceAccess = "read-only" | "read-write"; + +export interface ResourceLimits { + memoryMb: number; + cpus: number; + pids: number; + tmpfsMb: number; +} + +export interface ExecutionRequest { + runId: string; + command: string; + args: string[]; + workspace: string; + cwd: string; + timeoutMs: number; + maxOutputBytes: number; + workspaceAccess: WorkspaceAccess; + network: string; + secretNames: string[]; + limits: ResourceLimits; + image?: string; +} + +export interface ExecutionResult { + executor: ExecutorKind; + success: boolean; + exitCode: number | null; + signal: string | null; + timedOut: boolean; + truncated: boolean; + stdout: string; + stderr: string; + durationMs: number; + image?: string; + network: string; + workspaceAccess: WorkspaceAccess; + secretNames: string[]; + receipt: { + runId: string; + sandboxId: string; + commandHash: string; + policyHash: string; + }; + isolation: { + boundary: string; + readOnlyRoot: boolean; + capabilitiesDropped: boolean; + noNewPrivileges: boolean; + resourceLimits: boolean; + networkPolicy: string; + secretDelivery: string; + }; +} + +export interface ExecutorProbe { + kind: ExecutorKind; + available: boolean; + ready: boolean; + summary: string; + warnings: string[]; + details: Record; +} + +export interface Executor { + readonly kind: ExecutorKind; + execute(request: ExecutionRequest): Promise; + probe(): Promise; +} diff --git a/evolve-agent/src/factory.ts b/evolve-agent/src/factory.ts new file mode 100644 index 0000000..2018739 --- /dev/null +++ b/evolve-agent/src/factory.ts @@ -0,0 +1,182 @@ +import path from "node:path"; +import type { EvolveConfig } from "./config.js"; +import { EvolveError } from "./core/errors.js"; +import { ContextCompiler } from "./context/context-compiler.js"; +import { EvaluationEngine, type EvaluationEngineOptions } from "./evaluation/evaluation-engine.js"; +import { EvolutionOrchestrator } from "./evaluation/evolution-orchestrator.js"; +import { FixtureStore } from "./evaluation/fixture-store.js"; +import { ProvenanceSigner } from "./evaluation/provenance-signer.js"; +import { EvaluationReportStore } from "./evaluation/report-store.js"; +import { ShadowStore } from "./evaluation/shadow-store.js"; +import { DockerExecutor } from "./execution/docker-executor.js"; +import { ExecutorRegistry } from "./execution/executor-registry.js"; +import { ImagePolicy } from "./execution/image-policy.js"; +import { LocalExecutor } from "./execution/local-executor.js"; +import { DockerNetworkPolicy } from "./execution/network-policy.js"; +import { NodeProcessRunner, type ProcessRunner } from "./execution/process-runner.js"; +import type { Executor } from "./execution/types.js"; +import { ArtifactStore } from "./ledger/artifact-store.js"; +import { JsonlLedger } from "./ledger/jsonl-ledger.js"; +import { LearningEngine } from "./learning/learning-engine.js"; +import { MemoryStore } from "./memory/memory-store.js"; +import { InteractiveApprover, type Approver } from "./policy/approver.js"; +import { CapabilityAuthority } from "./policy/capability.js"; +import { RiskEngine } from "./policy/risk-engine.js"; +import { OpenAIResponsesProvider } from "./providers/openai-responses.js"; +import type { + AgentPrompt, + AgentProvider, + DecisionResult, + VerificationInput, + VerificationResult, +} from "./providers/provider.js"; +import { AgentRuntime } from "./runtime/agent-runtime.js"; +import { CheckpointStore } from "./runtime/checkpoint-store.js"; +import { EpisodeLeaseManager } from "./runtime/lease-manager.js"; +import { FileSecretBroker, type SecretBroker, type SecretSource } from "./secrets/secret-broker.js"; +import { SkillStore } from "./skills/skill-store.js"; +import { ToolRegistry } from "./tools/registry.js"; +import { FinalVerifier } from "./verification/final-verifier.js"; + +class MissingApiKeyProvider implements AgentProvider { + public async decide(_prompt: AgentPrompt): Promise { + throw new EvolveError("PROVIDER_AUTH", "OPENAI_API_KEY is required to run or resume an episode"); + } + public async verify(_input: VerificationInput): Promise { + throw new EvolveError("PROVIDER_AUTH", "OPENAI_API_KEY is required to verify an answer"); + } +} + +export interface RuntimeBundle { + runtime: AgentRuntime; + ledger: JsonlLedger; + artifacts: ArtifactStore; + checkpoints: CheckpointStore; + memory: MemoryStore; + skills: SkillStore; + tools: ToolRegistry; + provider: AgentProvider; + executors: ExecutorRegistry; + secrets: SecretBroker; + leases: EpisodeLeaseManager; + fixtures: FixtureStore; + reports: EvaluationReportStore; + evaluations: EvaluationEngine; + evolution: EvolutionOrchestrator; +} + +export interface RuntimeOverrides { + provider?: AgentProvider; + approver?: Approver; + processRunner?: ProcessRunner; + secretSource?: SecretSource; + secretBroker?: SecretBroker; + executors?: ExecutorRegistry; + leases?: EpisodeLeaseManager; + evaluations?: EvaluationEngine; + evolution?: EvolutionOrchestrator; +} + +function createExecutors( + config: EvolveConfig, + runner: ProcessRunner, + secrets: SecretBroker, +): ExecutorRegistry { + const executors: Executor[] = [ + new DockerExecutor(runner, secrets, { + binary: config.docker.binary, + user: config.docker.user, + imagePolicy: new ImagePolicy(config.docker.allowedImages, config.docker.defaultImage), + networkPolicy: new DockerNetworkPolicy(config.docker.allowedNetworks), + maximums: config.docker.maximums, + requireRootless: config.docker.requireRootless, + }), + ]; + if (config.allowLocalExecutor) executors.push(new LocalExecutor(runner)); + return new ExecutorRegistry(config.defaultExecutor, executors); +} + +export function createRuntime(config: EvolveConfig, overrides: RuntimeOverrides = {}): RuntimeBundle { + const runner = overrides.processRunner ?? new NodeProcessRunner(); + const secrets = + overrides.secretBroker ?? + new FileSecretBroker( + path.join(config.home, "runtime", "secrets"), + config.secretAllowlist, + config.secretTtlMs, + overrides.secretSource, + ); + const executors = overrides.executors ?? createExecutors(config, runner, secrets); + const leases = + overrides.leases ?? + new EpisodeLeaseManager(config.home, { + ttlMs: config.leaseTtlMs, + heartbeatMs: config.leaseHeartbeatMs, + }); + + const capabilities = new CapabilityAuthority(path.join(config.home, "capability.key")); + const tools = new ToolRegistry(capabilities, { + workspace: config.workspace, + allowedCommands: config.allowedCommands, + executors, + }); + const ledger = new JsonlLedger(path.join(config.home, "episodes.jsonl")); + const artifacts = new ArtifactStore(config.home); + const checkpoints = new CheckpointStore(config.home); + const memory = new MemoryStore(config.home); + const signer = new ProvenanceSigner(config.home); + const reports = new EvaluationReportStore(config.home, signer); + const skills = new SkillStore(config.home, reports); + const provider = + overrides.provider ?? + (config.openAiApiKey + ? new OpenAIResponsesProvider({ + apiKey: config.openAiApiKey, + model: config.model, + verifierModel: config.verifierModel, + reasoningEffort: config.reasoningEffort, + }) + : new MissingApiKeyProvider()); + const context = new ContextCompiler(memory, skills, tools); + const verifier = new FinalVerifier(artifacts, provider); + const learning = new LearningEngine(config.home, skills); + const fixtures = new FixtureStore(config.home, checkpoints, ledger, artifacts); + const shadow = new ShadowStore(config.home, config.evaluation.monitorWindow); + const evaluationOptions: EvaluationEngineOptions = { + policy: config.evaluation.policy, + canaryMinSamples: config.evaluation.canaryMinSamples, + monitorMinSamples: config.evaluation.monitorMinSamples, + monitorMaxSuccessDrop: config.evaluation.monitorMaxSuccessDrop, + monitorMaxScoreDrop: config.evaluation.monitorMaxScoreDrop, + monitorMaxTokenRatio: config.evaluation.monitorMaxTokenRatio, + }; + const evaluations = + overrides.evaluations ?? new EvaluationEngine(fixtures, reports, shadow, skills, provider, tools, evaluationOptions); + const evolution = + overrides.evolution ?? + new EvolutionOrchestrator(fixtures, evaluations, skills, ledger, { + captureCommitted: config.evaluation.captureCommitted, + shadowPercent: config.evaluation.shadowPercent, + monitorPromoted: config.evaluation.monitorPromoted, + }); + const risk = new RiskEngine(); + const approver = overrides.approver ?? new InteractiveApprover(config.nonInteractive); + const runtime = new AgentRuntime({ + provider, + ledger, + artifacts, + checkpoints, + tools, + risk, + approver, + capabilities, + context, + verifier, + memory, + skills, + learning, + leases, + evolution, + }); + return { runtime, ledger, artifacts, checkpoints, memory, skills, tools, provider, executors, secrets, leases, fixtures, reports, evaluations, evolution }; +} diff --git a/evolve-agent/src/index.ts b/evolve-agent/src/index.ts new file mode 100644 index 0000000..8d9d53e --- /dev/null +++ b/evolve-agent/src/index.ts @@ -0,0 +1,66 @@ +export { + loadConfig, + type ConfigOverrides, + type DockerConfig, + type EvaluationConfig, + type EvolveConfig, + type ReasoningEffort, +} from "./config.js"; +export { createRuntime, type RuntimeBundle, type RuntimeOverrides } from "./factory.js"; +export { AgentRuntime, type RuntimeDependencies } from "./runtime/agent-runtime.js"; +export { EpisodeLeaseManager, type EpisodeLease, type EpisodeLeaseRecord } from "./runtime/lease-manager.js"; +export { DockerExecutor, buildDockerRunArgs, type DockerExecutorOptions } from "./execution/docker-executor.js"; +export { ExecutorRegistry } from "./execution/executor-registry.js"; +export { ImagePolicy } from "./execution/image-policy.js"; +export { DockerNetworkPolicy } from "./execution/network-policy.js"; +export { LocalExecutor } from "./execution/local-executor.js"; +export { NodeProcessRunner, type ProcessRunner } from "./execution/process-runner.js"; +export type { + Executor, + ExecutorKind, + ExecutorProbe, + ExecutionRequest, + ExecutionResult, + ResourceLimits, + WorkspaceAccess, +} from "./execution/types.js"; +export { FileSecretBroker, type SecretBroker, type SecretLease, type SecretSource } from "./secrets/secret-broker.js"; +export { MockProvider } from "./providers/mock-provider.js"; +export type { + AgentDecision, + AgentPrompt, + AgentProvider, + FinalDecision, + MemoryProposal, + Observation, + ToolDecision, + VerificationVerdict, +} from "./providers/provider.js"; +export { StaticApprover, InteractiveApprover, type Approver } from "./policy/approver.js"; +export { FixtureStore } from "./evaluation/fixture-store.js"; +export { ReplayHarness } from "./evaluation/replay-harness.js"; +export { EvaluationEngine, type EvaluationEngineOptions } from "./evaluation/evaluation-engine.js"; +export { EvaluationReportStore } from "./evaluation/report-store.js"; +export { ProvenanceSigner } from "./evaluation/provenance-signer.js"; +export { ShadowStore } from "./evaluation/shadow-store.js"; +export { EvolutionOrchestrator, type EvolutionOrchestratorOptions } from "./evaluation/evolution-orchestrator.js"; +export { aggregateRuns, compareRuns, defaultEvaluationPolicy } from "./evaluation/metrics.js"; +export type { + EvaluationAggregate, + EvaluationArm, + EvaluationComparison, + EvaluationDecision, + EvaluationPolicy, + EvaluationReportKind, + EvaluationReportPayload, + FixtureSplit, + ProductionOutcome, + PromotionAuthorization, + ReplayFixture, + ReplayFixturePayload, + ReplayRunResult, + ReplayTraceStep, + ShadowObservation, + SignedEvaluationReport, +} from "./evaluation/types.js"; +export type { RunResult, TaskBudget, TaskInput, TaskSpec } from "./core/types.js"; diff --git a/evolve-agent/src/learning/learning-engine.ts b/evolve-agent/src/learning/learning-engine.ts new file mode 100644 index 0000000..4289dda --- /dev/null +++ b/evolve-agent/src/learning/learning-engine.ts @@ -0,0 +1,65 @@ +import path from "node:path"; +import { atomicWriteJson, readJsonFile } from "../core/fs.js"; +import { sha256Json } from "../core/hash.js"; +import type { EpisodeCheckpoint } from "../core/types.js"; +import type { SkillStore } from "../skills/skill-store.js"; + +interface PatternRecord { + fingerprint: string; + toolSequence: string[]; + episodes: string[]; + evidenceIds: string[]; + triggerTerms: string[]; +} + +interface PatternFile { + version: 1; + records: PatternRecord[]; +} + +function triggerTerms(goal: string): string[] { + return [...new Set(goal.toLowerCase().split(/[^\p{L}\p{N}_-]+/u).filter((term) => term.length >= 4))].slice(0, 12); +} + +export class LearningEngine { + private readonly filePath: string; + + public constructor(home: string, private readonly skills: SkillStore) { + this.filePath = path.join(home, "patterns.json"); + } + + public async observeCommitted(checkpoint: EpisodeCheckpoint): Promise { + if (checkpoint.status !== "committed" || checkpoint.toolSequence.length === 0) return; + const sequence = checkpoint.toolSequence.slice(0, 30); + const fingerprint = sha256Json({ version: 1, sequence }); + const file = await readJsonFile(this.filePath, { version: 1, records: [] }); + let pattern = file.records.find((record) => record.fingerprint === fingerprint); + if (!pattern) { + pattern = { + fingerprint, + toolSequence: sequence, + episodes: [], + evidenceIds: [], + triggerTerms: [], + }; + file.records.push(pattern); + } + pattern.episodes = [...new Set([...pattern.episodes, checkpoint.episodeId])]; + pattern.evidenceIds = [...new Set([...pattern.evidenceIds, ...checkpoint.evidenceIds])].slice(-100); + pattern.triggerTerms = [...new Set([...pattern.triggerTerms, ...triggerTerms(checkpoint.task.goal)])].slice(0, 20); + await atomicWriteJson(this.filePath, file, 0o600); + + if (pattern.episodes.length >= 2) { + await this.skills.upsertCandidate({ + fingerprint, + name: `Learned flow: ${sequence.join(" → ")}`.slice(0, 120), + description: `Candidate procedure derived from ${pattern.episodes.length} committed episodes. It remains inactive until evaluation and canary gates pass.`, + triggers: pattern.triggerTerms, + steps: sequence.map((toolName, index) => ({ toolName, purpose: `Step ${index + 1} in the observed successful flow` })), + allowedTools: [...new Set(sequence)], + supportingEpisodes: pattern.episodes, + provenanceEvidenceIds: pattern.evidenceIds, + }); + } + } +} diff --git a/evolve-agent/src/ledger/artifact-store.ts b/evolve-agent/src/ledger/artifact-store.ts new file mode 100644 index 0000000..b35fd9a --- /dev/null +++ b/evolve-agent/src/ledger/artifact-store.ts @@ -0,0 +1,103 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { ensureDir, pathExists } from "../core/fs.js"; +import { sha256Bytes, sha256Json } from "../core/hash.js"; +import { stableStringify } from "../core/stable-json.js"; +import type { EvidenceRecord, JsonValue } from "../core/types.js"; + +interface ArtifactEnvelope { + version: 1; + mediaType: "application/json"; + data: JsonValue; +} + +export class ArtifactStore { + private readonly artifactsDir: string; + private readonly evidenceDir: string; + + public constructor(home: string) { + this.artifactsDir = path.join(home, "artifacts"); + this.evidenceDir = path.join(home, "evidence"); + } + + public async put(input: { + episodeId: string; + toolName: string; + args: Record; + data: JsonValue; + summary: string; + success: boolean; + }): Promise { + await ensureDir(this.artifactsDir); + await ensureDir(this.evidenceDir); + const envelope: ArtifactEnvelope = { version: 1, mediaType: "application/json", data: input.data }; + const serialized = `${stableStringify(envelope)}\n`; + const artifactHash = sha256Bytes(serialized); + const artifactPath = path.join(this.artifactsDir, `${artifactHash}.json`); + if (!(await pathExists(artifactPath))) await writeFile(artifactPath, serialized, { encoding: "utf8", flag: "wx" }); + + const createdAt = new Date().toISOString(); + const argsHash = sha256Json(input.args); + const id = `ev_${sha256Json({ + episodeId: input.episodeId, + toolName: input.toolName, + argsHash, + artifactHash, + createdAt, + }).slice(0, 24)}`; + const evidence: EvidenceRecord = { + id, + episodeId: input.episodeId, + toolName: input.toolName, + argsHash, + artifactHash, + summary: input.summary.slice(0, 500), + success: input.success, + createdAt, + }; + await writeFile(path.join(this.evidenceDir, `${id}.json`), `${JSON.stringify(evidence, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + }); + return evidence; + } + + public async getEvidence(id: string): Promise { + if (!/^ev_[a-f0-9]{24}$/.test(id)) return undefined; + try { + return JSON.parse(await readFile(path.join(this.evidenceDir, `${id}.json`), "utf8")) as EvidenceRecord; + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + } + + public async readArtifact(artifactHash: string): Promise { + if (!/^[a-f0-9]{64}$/.test(artifactHash)) return undefined; + try { + const envelope = JSON.parse(await readFile(path.join(this.artifactsDir, `${artifactHash}.json`), "utf8")) as ArtifactEnvelope; + return envelope.data; + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } + } + + public async validateEvidenceIds(episodeId: string, ids: string[]): Promise<{ valid: boolean; invalid: string[] }> { + const invalid: string[] = []; + for (const id of new Set(ids)) { + const evidence = await this.getEvidence(id); + if (!evidence || evidence.episodeId !== episodeId) invalid.push(id); + } + return { valid: invalid.length === 0, invalid }; + } + + public async summaries(episodeId: string, ids: string[]): Promise> { + const output: Array<{ id: string; summary: string; success: boolean }> = []; + for (const id of ids) { + const evidence = await this.getEvidence(id); + if (evidence?.episodeId === episodeId) output.push({ id, summary: evidence.summary, success: evidence.success }); + } + return output; + } +} diff --git a/evolve-agent/src/ledger/jsonl-ledger.ts b/evolve-agent/src/ledger/jsonl-ledger.ts new file mode 100644 index 0000000..481ce28 --- /dev/null +++ b/evolve-agent/src/ledger/jsonl-ledger.ts @@ -0,0 +1,82 @@ +import { appendFile, readFile } from "node:fs/promises"; +import path from "node:path"; +import { ensureDir } from "../core/fs.js"; +import { sha256Json } from "../core/hash.js"; +import type { JsonValue, LedgerEvent, LedgerVerification } from "../core/types.js"; + +interface LedgerUnsignedEvent { + version: 1; + index: number; + timestamp: string; + episodeId: string; + type: string; + payload: JsonValue; + prevHash: string | null; +} + +export class JsonlLedger { + private appendQueue: Promise = Promise.resolve(); + + public constructor(private readonly filePath: string) {} + + public async append(episodeId: string, type: string, payload: JsonValue): Promise { + let result: LedgerEvent | undefined; + this.appendQueue = this.appendQueue.then(async () => { + await ensureDir(path.dirname(this.filePath)); + const events = await this.readAll(); + const previous = events.at(-1); + const unsigned: LedgerUnsignedEvent = { + version: 1, + index: events.length, + timestamp: new Date().toISOString(), + episodeId, + type, + payload, + prevHash: previous?.hash ?? null, + }; + result = { ...unsigned, hash: sha256Json(unsigned) }; + await appendFile(this.filePath, `${JSON.stringify(result)}\n`, "utf8"); + }); + await this.appendQueue; + if (!result) throw new Error("Ledger append did not produce an event"); + return result; + } + + public async readAll(): Promise { + try { + const text = await readFile(this.filePath, "utf8"); + return text + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as LedgerEvent); + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + } + + public async forEpisode(episodeId: string): Promise { + return (await this.readAll()).filter((event) => event.episodeId === episodeId); + } + + public async verify(): Promise { + let events: LedgerEvent[]; + try { + events = await this.readAll(); + } catch (error: unknown) { + return { valid: false, events: 0, error: error instanceof Error ? error.message : String(error) }; + } + + let previousHash: string | null = null; + for (let index = 0; index < events.length; index += 1) { + const event = events[index]; + if (!event) return { valid: false, events: index, error: `Missing event at index ${index}` }; + if (event.index !== index) return { valid: false, events: index, error: `Index mismatch at ${index}` }; + if (event.prevHash !== previousHash) return { valid: false, events: index, error: `Chain mismatch at ${index}` }; + const { hash, ...unsigned } = event; + if (sha256Json(unsigned) !== hash) return { valid: false, events: index, error: `Hash mismatch at ${index}` }; + previousHash = event.hash; + } + return { valid: true, events: events.length }; + } +} diff --git a/evolve-agent/src/memory/memory-store.ts b/evolve-agent/src/memory/memory-store.ts new file mode 100644 index 0000000..22adaca --- /dev/null +++ b/evolve-agent/src/memory/memory-store.ts @@ -0,0 +1,99 @@ +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { atomicWriteJson, readJsonFile } from "../core/fs.js"; +import { EvolveError } from "../core/errors.js"; +import type { MemoryRecord } from "../core/types.js"; + +interface MemoryFile { + version: 1; + records: MemoryRecord[]; +} + +function terms(text: string): Set { + return new Set( + text + .toLowerCase() + .split(/[^\p{L}\p{N}_-]+/u) + .map((term) => term.trim()) + .filter((term) => term.length >= 2), + ); +} + +export class MemoryStore { + private readonly filePath: string; + + public constructor(home: string) { + this.filePath = path.join(home, "memory.json"); + } + + private async load(): Promise { + const file = await readJsonFile(this.filePath, { version: 1, records: [] }); + if (file.version !== 1 || !Array.isArray(file.records)) throw new EvolveError("MEMORY_CORRUPT", "Unsupported memory file"); + return file; + } + + public async add(input: { + text: string; + tags: string[]; + confidence: number; + sourceEpisodeId: string; + evidenceIds: string[]; + validEvidenceIds: Set; + }): Promise { + const text = input.text.trim(); + if (text.length < 4 || text.length > 2_000) throw new EvolveError("MEMORY_INVALID", "Memory text length is invalid"); + if (!Number.isFinite(input.confidence) || input.confidence < 0 || input.confidence > 1) { + throw new EvolveError("MEMORY_INVALID", "Memory confidence must be between 0 and 1"); + } + const evidenceIds = [...new Set(input.evidenceIds)]; + const invalid = evidenceIds.filter((id) => !input.validEvidenceIds.has(id)); + if (invalid.length > 0) throw new EvolveError("MEMORY_EVIDENCE_INVALID", `Invalid memory evidence: ${invalid.join(", ")}`); + if (input.confidence >= 0.8 && evidenceIds.length === 0) { + throw new EvolveError("MEMORY_EVIDENCE_REQUIRED", "High-confidence memory requires current-episode evidence"); + } + + const file = await this.load(); + const normalized = text.toLowerCase(); + const existing = file.records.find( + (record) => record.sourceEpisodeId === input.sourceEpisodeId && record.text.toLowerCase() === normalized, + ); + if (existing) return existing; + + const record: MemoryRecord = { + id: `mem_${randomUUID().replaceAll("-", "").slice(0, 24)}`, + text, + tags: [...new Set(input.tags.map((tag) => tag.trim().toLowerCase()).filter(Boolean))].slice(0, 12), + confidence: input.confidence, + sourceEpisodeId: input.sourceEpisodeId, + evidenceIds, + createdAt: new Date().toISOString(), + }; + file.records.push(record); + if (file.records.length > 10_000) file.records.splice(0, file.records.length - 10_000); + await atomicWriteJson(this.filePath, file, 0o600); + return record; + } + + public async search(query: string, limit = 8): Promise { + const queryTerms = terms(query); + if (queryTerms.size === 0) return []; + const records = (await this.load()).records; + return records + .map((record) => { + const recordTerms = terms(`${record.text} ${record.tags.join(" ")}`); + let overlap = 0; + for (const term of queryTerms) if (recordTerms.has(term)) overlap += 1; + const recencyDays = Math.max(0, (Date.now() - Date.parse(record.createdAt)) / 86_400_000); + const recency = 1 / (1 + recencyDays / 30); + return { record, score: overlap * 2 + record.confidence + recency * 0.25 }; + }) + .filter((entry) => entry.score > 0.5) + .sort((left, right) => right.score - left.score || right.record.createdAt.localeCompare(left.record.createdAt)) + .slice(0, Math.max(0, Math.min(limit, 50))) + .map((entry) => entry.record); + } + + public async list(): Promise { + return (await this.load()).records; + } +} diff --git a/evolve-agent/src/policy/approver.ts b/evolve-agent/src/policy/approver.ts new file mode 100644 index 0000000..9fee536 --- /dev/null +++ b/evolve-agent/src/policy/approver.ts @@ -0,0 +1,39 @@ +import { createInterface } from "node:readline/promises"; +import { stdin, stdout } from "node:process"; +import { stableStringify } from "../core/stable-json.js"; + +export interface ApprovalRequest { + episodeId: string; + toolName: string; + risk: "write" | "execute" | "external"; + args: Record; + reason: string; +} + +export interface Approver { + approve(request: ApprovalRequest): Promise; +} + +export class InteractiveApprover implements Approver { + public constructor(private readonly nonInteractive: boolean) {} + + public async approve(request: ApprovalRequest): Promise { + if (this.nonInteractive || !stdin.isTTY || !stdout.isTTY) return false; + const rl = createInterface({ input: stdin, output: stdout }); + try { + stdout.write(`\nApproval required [${request.risk}] ${request.toolName}\n`); + stdout.write(`${request.reason}\n${stableStringify(request.args)}\n`); + const answer = (await rl.question("Execute this exact action? [y/N] ")).trim().toLowerCase(); + return answer === "y" || answer === "yes"; + } finally { + rl.close(); + } + } +} + +export class StaticApprover implements Approver { + public constructor(private readonly decision: boolean) {} + public async approve(): Promise { + return this.decision; + } +} diff --git a/evolve-agent/src/policy/capability.ts b/evolve-agent/src/policy/capability.ts new file mode 100644 index 0000000..7b09746 --- /dev/null +++ b/evolve-agent/src/policy/capability.ts @@ -0,0 +1,102 @@ +import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { ensureDir } from "../core/fs.js"; +import { sha256Json } from "../core/hash.js"; +import { stableStringify } from "../core/stable-json.js"; +import { EvolveError } from "../core/errors.js"; + +interface CapabilityPayload { + version: 1; + episodeId: string; + toolName: string; + argsHash: string; + issuedAt: number; + expiresAt: number; + nonce: string; +} + +function base64url(value: string | Buffer): string { + return Buffer.from(value).toString("base64url"); +} + +export class CapabilityAuthority { + private keyPromise?: Promise; + + public constructor(private readonly keyPath: string) {} + + private async key(): Promise { + this.keyPromise ??= (async () => { + await ensureDir(path.dirname(this.keyPath)); + try { + const encoded = (await readFile(this.keyPath, "utf8")).trim(); + return Buffer.from(encoded, "base64url"); + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + const generated = randomBytes(32); + await writeFile(this.keyPath, `${generated.toString("base64url")}\n`, { mode: 0o600, flag: "wx" }); + return generated; + } + })(); + return this.keyPromise; + } + + public async issue(input: { + episodeId: string; + toolName: string; + args: Record; + ttlMs?: number; + }): Promise { + const now = Date.now(); + const payload: CapabilityPayload = { + version: 1, + episodeId: input.episodeId, + toolName: input.toolName, + argsHash: sha256Json(input.args), + issuedAt: now, + expiresAt: now + Math.max(1, Math.min(input.ttlMs ?? 60_000, 5 * 60_000)), + nonce: randomBytes(16).toString("hex"), + }; + const encoded = base64url(stableStringify(payload)); + const signature = createHmac("sha256", await this.key()).update(encoded).digest("base64url"); + return `${encoded}.${signature}`; + } + + public async verify(token: string, expected: { + episodeId: string; + toolName: string; + args: Record; + now?: number; + }): Promise { + const [encoded, signature] = token.split("."); + if (!encoded || !signature) throw new EvolveError("CAPABILITY_INVALID", "Malformed capability token"); + const expectedSignature = createHmac("sha256", await this.key()).update(encoded).digest(); + let provided: Buffer; + try { + provided = Buffer.from(signature, "base64url"); + } catch { + throw new EvolveError("CAPABILITY_INVALID", "Malformed capability signature"); + } + if (provided.length !== expectedSignature.length || !timingSafeEqual(provided, expectedSignature)) { + throw new EvolveError("CAPABILITY_INVALID", "Capability signature mismatch"); + } + + let payload: CapabilityPayload; + try { + payload = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")) as CapabilityPayload; + } catch { + throw new EvolveError("CAPABILITY_INVALID", "Malformed capability payload"); + } + if (payload.version !== 1) throw new EvolveError("CAPABILITY_INVALID", "Unsupported capability version"); + if (payload.episodeId !== expected.episodeId || payload.toolName !== expected.toolName) { + throw new EvolveError("CAPABILITY_SCOPE_MISMATCH", "Capability scope does not match the requested action"); + } + if (payload.argsHash !== sha256Json(expected.args)) { + throw new EvolveError("CAPABILITY_ARGS_MISMATCH", "Tool arguments changed after approval"); + } + const now = expected.now ?? Date.now(); + if (payload.expiresAt < now || payload.issuedAt > now + 5_000) { + throw new EvolveError("CAPABILITY_EXPIRED", "Capability token has expired or is not yet valid"); + } + } +} diff --git a/evolve-agent/src/policy/risk-engine.ts b/evolve-agent/src/policy/risk-engine.ts new file mode 100644 index 0000000..d3b73be --- /dev/null +++ b/evolve-agent/src/policy/risk-engine.ts @@ -0,0 +1,42 @@ +import { EvolveError } from "../core/errors.js"; +import type { TaskSpec } from "../core/types.js"; +import type { ToolDefinition, ToolRisk } from "../tools/types.js"; + +export interface PolicyDecision { + allowed: boolean; + approvalRequired: boolean; + reason: string; +} + +export class RiskEngine { + public evaluate(task: TaskSpec, tool: ToolDefinition, args: Record): PolicyDecision { + if (!task.requestedTools.includes(tool.name)) { + return { + allowed: false, + approvalRequired: false, + reason: `Tool ${tool.name} is outside the task's requested tool boundary`, + }; + } + + if (Object.keys(args).length > 64) { + return { allowed: false, approvalRequired: false, reason: "Tool argument object is unexpectedly large" }; + } + + const approvalRequired = tool.risk !== "read"; + return { + allowed: true, + approvalRequired, + reason: approvalRequired + ? `${tool.name} is classified as ${tool.risk} and requires explicit approval` + : `${tool.name} is a workspace-confined read operation`, + }; + } + + public assertAllowed(decision: PolicyDecision): void { + if (!decision.allowed) throw new EvolveError("POLICY_DENIED", decision.reason); + } +} + +export function protectedRisk(risk: ToolRisk): risk is Exclude { + return risk !== "read"; +} diff --git a/evolve-agent/src/providers/mock-provider.ts b/evolve-agent/src/providers/mock-provider.ts new file mode 100644 index 0000000..63f00b7 --- /dev/null +++ b/evolve-agent/src/providers/mock-provider.ts @@ -0,0 +1,43 @@ +import type { + AgentDecision, + AgentPrompt, + AgentProvider, + DecisionResult, + VerificationInput, + VerificationResult, + VerificationVerdict, +} from "./provider.js"; + +export type MockDecision = AgentDecision | ((prompt: AgentPrompt) => AgentDecision); + +export class MockProvider implements AgentProvider { + public readonly prompts: AgentPrompt[] = []; + public readonly verifications: VerificationInput[] = []; + + public constructor( + private readonly decisions: MockDecision[], + private readonly verdicts: VerificationVerdict[] = [{ passed: true, score: 1, feedback: "accepted" }], + ) {} + + public async decide(prompt: AgentPrompt): Promise { + this.prompts.push(prompt); + const next = this.decisions.shift(); + if (!next) throw new Error("MockProvider decision queue exhausted"); + const decision = typeof next === "function" ? next(prompt) : next; + return { + decision, + usage: { inputTokens: 11, outputTokens: 7, totalTokens: 18 }, + providerResponseId: `mock_${this.prompts.length}`, + }; + } + + public async verify(input: VerificationInput): Promise { + this.verifications.push(input); + const verdict = this.verdicts.shift() ?? { passed: true, score: 1, feedback: "accepted" }; + return { + verdict, + usage: { inputTokens: 5, outputTokens: 3, totalTokens: 8 }, + providerResponseId: `mock_verify_${this.verifications.length}`, + }; + } +} diff --git a/evolve-agent/src/providers/openai-responses.ts b/evolve-agent/src/providers/openai-responses.ts new file mode 100644 index 0000000..c8e896f --- /dev/null +++ b/evolve-agent/src/providers/openai-responses.ts @@ -0,0 +1,281 @@ +import { EvolveError } from "../core/errors.js"; +import type { JsonObject, Usage } from "../core/types.js"; +import type { ReasoningEffort } from "../config.js"; +import type { + AgentDecision, + AgentPrompt, + AgentProvider, + DecisionResult, + FinalDecision, + MemoryProposal, + VerificationInput, + VerificationResult, + VerificationVerdict, +} from "./provider.js"; + +interface ResponsesOutputItem { + type?: string; + name?: string; + arguments?: string; + call_id?: string; +} + +interface ResponsesPayload { + id?: string; + output?: ResponsesOutputItem[]; + usage?: { + input_tokens?: number; + output_tokens?: number; + total_tokens?: number; + }; + error?: { message?: string; code?: string }; +} + +const SYSTEM_INSTRUCTIONS = `You are the decision engine inside Evolve Agent, an evidence-gated autonomous runtime. +Return exactly one function call per turn. Never claim that a tool ran unless its result appears in observations. +Use the minimum necessary tool. Respect the task's requested tools, remaining budgets, and constraints. +When the task is complete, call commit_answer. Every externally checkable claim in the final answer must cite a current-episode evidence ID using [evidence:ev_...]. +Do not invent evidence IDs. Do not expose hidden chain-of-thought; rationale must be a short operational reason. +Memory proposals must be durable, generalizable facts or procedures, not temporary conversation details. High-confidence memory requires evidence.`; + +const VERIFIER_INSTRUCTIONS = `You are an independent acceptance verifier. Judge only the proposed answer, task criteria, and supplied evidence metadata. Do not assume missing facts. Return exactly one verification_verdict call. Pass only when the answer satisfies the goal and constraints, the evidence supports factual claims, and no material claim is unsupported. A score below 0.8 must fail.`; + +function usageOf(payload: ResponsesPayload): Usage { + const inputTokens = payload.usage?.input_tokens ?? 0; + const outputTokens = payload.usage?.output_tokens ?? 0; + return { + inputTokens, + outputTokens, + totalTokens: payload.usage?.total_tokens ?? inputTokens + outputTokens, + }; +} + +function parseObject(value: string | undefined, label: string): Record { + if (value === undefined) throw new EvolveError("PROVIDER_PROTOCOL", `${label} omitted function arguments`); + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new EvolveError("PROVIDER_PROTOCOL", `${label} returned invalid JSON arguments`); + } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new EvolveError("PROVIDER_PROTOCOL", `${label} arguments must be an object`); + } + return parsed as Record; +} + +function requiredString(value: unknown, label: string, max = 200_000): string { + if (typeof value !== "string" || value.length === 0 || value.length > max) { + throw new EvolveError("PROVIDER_PROTOCOL", `${label} must be a non-empty string no longer than ${max}`); + } + return value; +} + +function stringList(value: unknown, label: string, maxItems = 100): string[] { + if (!Array.isArray(value) || value.length > maxItems || !value.every((entry) => typeof entry === "string")) { + throw new EvolveError("PROVIDER_PROTOCOL", `${label} must be an array of strings`); + } + return [...value]; +} + +function numberValue(value: unknown, label: string, min: number, max: number): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) { + throw new EvolveError("PROVIDER_PROTOCOL", `${label} must be between ${min} and ${max}`); + } + return value; +} + +function booleanValue(value: unknown, label: string): boolean { + if (typeof value !== "boolean") throw new EvolveError("PROVIDER_PROTOCOL", `${label} must be boolean`); + return value; +} + +function parseMemoryProposals(value: unknown): MemoryProposal[] { + if (value === undefined) return []; + if (!Array.isArray(value) || value.length > 3) { + throw new EvolveError("PROVIDER_PROTOCOL", "memory_proposals must be an array of at most 3 items"); + } + return value.map((entry, index) => { + if (entry === null || typeof entry !== "object" || Array.isArray(entry)) { + throw new EvolveError("PROVIDER_PROTOCOL", `memory_proposals[${index}] must be an object`); + } + const object = entry as Record; + return { + text: requiredString(object.text, `memory_proposals[${index}].text`, 2_000), + tags: stringList(object.tags, `memory_proposals[${index}].tags`, 12).map((tag) => tag.slice(0, 80)), + confidence: numberValue(object.confidence, `memory_proposals[${index}].confidence`, 0, 1), + evidenceIds: stringList(object.evidence_ids, `memory_proposals[${index}].evidence_ids`, 20), + }; + }); +} + +export class OpenAIResponsesProvider implements AgentProvider { + public constructor( + private readonly options: { + apiKey: string; + model: string; + verifierModel: string; + reasoningEffort: ReasoningEffort; + endpoint?: string; + timeoutMs?: number; + }, + ) {} + + private async request(body: Record): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), this.options.timeoutMs ?? 180_000); + timeout.unref(); + try { + const response = await fetch(this.options.endpoint ?? "https://api.openai.com/v1/responses", { + method: "POST", + headers: { + authorization: `Bearer ${this.options.apiKey}`, + "content-type": "application/json", + }, + body: JSON.stringify(body), + signal: controller.signal, + }); + const payload = (await response.json()) as ResponsesPayload; + if (!response.ok) { + throw new EvolveError( + "PROVIDER_HTTP", + payload.error?.message ?? `OpenAI Responses API returned HTTP ${response.status}`, + { status: response.status, code: payload.error?.code }, + ); + } + return payload; + } catch (error: unknown) { + if (error instanceof EvolveError) throw error; + if ((error as Error).name === "AbortError") throw new EvolveError("PROVIDER_TIMEOUT", "OpenAI request timed out"); + throw new EvolveError("PROVIDER_NETWORK", error instanceof Error ? error.message : String(error)); + } finally { + clearTimeout(timeout); + } + } + + public async decide(prompt: AgentPrompt): Promise { + const tools = prompt.tools.map((tool) => ({ + type: "function", + name: tool.name, + description: `${tool.description} Risk classification: ${tool.risk}.`, + parameters: tool.inputSchema, + strict: false, + })); + tools.push({ + type: "function", + name: "commit_answer", + description: "Submit the final answer, its exact evidence set, and optional durable memory proposals.", + parameters: { + type: "object", + properties: { + answer: { type: "string" }, + evidence_ids: { type: "array", items: { type: "string" } }, + memory_proposals: { + type: "array", + maxItems: 3, + items: { + type: "object", + properties: { + text: { type: "string" }, + tags: { type: "array", items: { type: "string" } }, + confidence: { type: "number", minimum: 0, maximum: 1 }, + evidence_ids: { type: "array", items: { type: "string" } }, + }, + required: ["text", "tags", "confidence", "evidence_ids"], + additionalProperties: false, + }, + }, + }, + required: ["answer", "evidence_ids", "memory_proposals"], + additionalProperties: false, + }, + strict: true, + }); + + const payload = await this.request({ + model: this.options.model, + store: false, + reasoning: { effort: this.options.reasoningEffort }, + instructions: SYSTEM_INSTRUCTIONS, + input: JSON.stringify(prompt), + tools, + tool_choice: "required", + parallel_tool_calls: false, + }); + const calls = (payload.output ?? []).filter((item) => item.type === "function_call"); + if (calls.length !== 1) { + throw new EvolveError("PROVIDER_PROTOCOL", `Expected exactly one function call, received ${calls.length}`); + } + const call = calls[0] as ResponsesOutputItem; + const parsed = parseObject(call.arguments, call.name ?? "function_call"); + let decision: AgentDecision; + if (call.name === "commit_answer") { + const final: FinalDecision = { + kind: "final", + answer: requiredString(parsed.answer, "answer"), + evidenceIds: stringList(parsed.evidence_ids, "evidence_ids", 100), + memoryProposals: parseMemoryProposals(parsed.memory_proposals), + }; + decision = final; + } else { + const tool = prompt.tools.find((candidate) => candidate.name === call.name); + if (!tool) throw new EvolveError("PROVIDER_PROTOCOL", `Model called unavailable tool ${String(call.name)}`); + decision = { + kind: "tool", + toolName: tool.name, + args: parsed as JsonObject, + rationale: `Use ${tool.name} to advance the task`, + }; + } + return { + decision, + usage: usageOf(payload), + ...(payload.id ? { providerResponseId: payload.id } : {}), + }; + } + + public async verify(input: VerificationInput): Promise { + const payload = await this.request({ + model: this.options.verifierModel, + store: false, + reasoning: { effort: this.options.reasoningEffort }, + instructions: VERIFIER_INSTRUCTIONS, + input: JSON.stringify(input), + tools: [ + { + type: "function", + name: "verification_verdict", + description: "Return the independent acceptance verdict.", + parameters: { + type: "object", + properties: { + passed: { type: "boolean" }, + score: { type: "number", minimum: 0, maximum: 1 }, + feedback: { type: "string" }, + }, + required: ["passed", "score", "feedback"], + additionalProperties: false, + }, + strict: true, + }, + ], + tool_choice: { type: "function", name: "verification_verdict" }, + parallel_tool_calls: false, + }); + const calls = (payload.output ?? []).filter((item) => item.type === "function_call" && item.name === "verification_verdict"); + if (calls.length !== 1) throw new EvolveError("PROVIDER_PROTOCOL", "Verifier did not return one verdict"); + const parsed = parseObject(calls[0]?.arguments, "verification_verdict"); + const score = numberValue(parsed.score, "score", 0, 1); + const passed = booleanValue(parsed.passed, "passed") && score >= 0.8; + const verdict: VerificationVerdict = { + passed, + score, + feedback: requiredString(parsed.feedback, "feedback", 4_000), + }; + return { + verdict, + usage: usageOf(payload), + ...(payload.id ? { providerResponseId: payload.id } : {}), + }; + } +} diff --git a/evolve-agent/src/providers/provider.ts b/evolve-agent/src/providers/provider.ts new file mode 100644 index 0000000..5c74033 --- /dev/null +++ b/evolve-agent/src/providers/provider.ts @@ -0,0 +1,79 @@ +import type { EvidenceRecord, JsonObject, MemoryRecord, SkillRecord, TaskSpec, Usage } from "../core/types.js"; +import type { ToolDescription } from "../tools/types.js"; + +export interface Observation { + id: string; + kind: "tool" | "error" | "verification" | "system"; + content: string; + evidenceId?: string; + toolName?: string; +} + +export interface AgentPrompt { + task: TaskSpec; + observations: Observation[]; + memories: MemoryRecord[]; + skills: SkillRecord[]; + tools: ToolDescription[]; + remainingBudget: { + turns: number; + toolCalls: number; + inputTokens: number; + outputTokens: number; + wallTimeMs: number; + }; +} + +export interface ToolDecision { + kind: "tool"; + toolName: string; + args: JsonObject; + rationale: string; +} + +export interface MemoryProposal { + text: string; + tags: string[]; + confidence: number; + evidenceIds: string[]; +} + +export interface FinalDecision { + kind: "final"; + answer: string; + evidenceIds: string[]; + memoryProposals: MemoryProposal[]; +} + +export type AgentDecision = ToolDecision | FinalDecision; + +export interface DecisionResult { + decision: AgentDecision; + usage: Usage; + providerResponseId?: string; +} + +export interface VerificationInput { + task: TaskSpec; + answer: string; + evidence: EvidenceRecord[]; +} + +export interface VerificationVerdict { + passed: boolean; + score: number; + feedback: string; +} + +export interface VerificationResult { + verdict: VerificationVerdict; + usage: Usage; + providerResponseId?: string; +} + +export interface AgentProvider { + decide(prompt: AgentPrompt): Promise; + verify(input: VerificationInput): Promise; +} + +export const ZERO_USAGE: Usage = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; diff --git a/evolve-agent/src/runtime/agent-runtime.ts b/evolve-agent/src/runtime/agent-runtime.ts new file mode 100644 index 0000000..f580137 --- /dev/null +++ b/evolve-agent/src/runtime/agent-runtime.ts @@ -0,0 +1,472 @@ +import { randomUUID } from "node:crypto"; +import { errorMessage, EvolveError } from "../core/errors.js"; +import { sha256Json } from "../core/hash.js"; +import type { EpisodeCheckpoint, RunResult, TaskBudget, TaskInput, TaskSpec, Usage } from "../core/types.js"; +import type { JsonlLedger } from "../ledger/jsonl-ledger.js"; +import type { ArtifactStore } from "../ledger/artifact-store.js"; +import type { MemoryStore } from "../memory/memory-store.js"; +import type { Approver } from "../policy/approver.js"; +import type { CapabilityAuthority } from "../policy/capability.js"; +import type { RiskEngine } from "../policy/risk-engine.js"; +import type { AgentProvider, Observation } from "../providers/provider.js"; +import type { SkillStore } from "../skills/skill-store.js"; +import type { LearningEngine } from "../learning/learning-engine.js"; +import type { ContextCompiler } from "../context/context-compiler.js"; +import type { ToolRegistry } from "../tools/registry.js"; +import type { FinalVerifier } from "../verification/final-verifier.js"; +import type { CheckpointStore } from "./checkpoint-store.js"; +import type { EpisodeLease, EpisodeLeaseManager } from "./lease-manager.js"; + +const DEFAULT_BUDGET: TaskBudget = { + maxTurns: 12, + maxToolCalls: 8, + maxInputTokens: 200_000, + maxOutputTokens: 40_000, + maxWallTimeMs: 5 * 60_000, +}; + +function episodeId(): string { + return `ep_${randomUUID().replaceAll("-", "").slice(0, 24)}`; +} + +function observationId(): string { + return `obs_${randomUUID().replaceAll("-", "").slice(0, 16)}`; +} + +function addUsage(left: Usage, right: Usage): Usage { + return { + inputTokens: left.inputTokens + right.inputTokens, + outputTokens: left.outputTokens + right.outputTokens, + totalTokens: left.totalTokens + right.totalTokens, + }; +} + +function resultOf(checkpoint: EpisodeCheckpoint): RunResult { + return { + episodeId: checkpoint.episodeId, + status: checkpoint.status, + ...(checkpoint.answer !== undefined ? { answer: checkpoint.answer } : {}), + evidenceIds: checkpoint.evidenceIds, + usage: checkpoint.usage, + turns: checkpoint.turns, + toolCalls: checkpoint.toolCalls, + ...(checkpoint.stopReason !== undefined ? { reason: checkpoint.stopReason } : {}), + }; +} + +function preTurnBudgetReason(checkpoint: EpisodeCheckpoint): string | undefined { + const budget = checkpoint.task.budget; + if (checkpoint.turns >= budget.maxTurns) return `Turn budget exhausted (${budget.maxTurns})`; + if (checkpoint.usage.inputTokens >= budget.maxInputTokens) return `Input-token budget exhausted (${budget.maxInputTokens})`; + if (checkpoint.usage.outputTokens >= budget.maxOutputTokens) return `Output-token budget exhausted (${budget.maxOutputTokens})`; + if (checkpoint.elapsedMs >= budget.maxWallTimeMs) return `Wall-time budget exhausted (${budget.maxWallTimeMs} ms)`; + return undefined; +} + +function hardOverrunReason(checkpoint: EpisodeCheckpoint): string | undefined { + const budget = checkpoint.task.budget; + if (checkpoint.usage.inputTokens > budget.maxInputTokens) return `Input-token budget exceeded (${budget.maxInputTokens})`; + if (checkpoint.usage.outputTokens > budget.maxOutputTokens) return `Output-token budget exceeded (${budget.maxOutputTokens})`; + if (checkpoint.elapsedMs > budget.maxWallTimeMs) return `Wall-time budget exceeded (${budget.maxWallTimeMs} ms)`; + return undefined; +} + +export interface RuntimeDependencies { + provider: AgentProvider; + ledger: JsonlLedger; + artifacts: ArtifactStore; + checkpoints: CheckpointStore; + tools: ToolRegistry; + risk: RiskEngine; + approver: Approver; + capabilities: CapabilityAuthority; + context: ContextCompiler; + verifier: FinalVerifier; + memory: MemoryStore; + skills: SkillStore; + learning: LearningEngine; + leases: EpisodeLeaseManager; + evolution?: { observeTerminal(checkpoint: EpisodeCheckpoint): Promise }; +} + +export class AgentRuntime { + public constructor(private readonly dependencies: RuntimeDependencies) {} + + private createTask(input: TaskInput): TaskSpec { + const goal = input.goal.trim(); + if (goal.length < 3 || goal.length > 20_000) throw new EvolveError("TASK_INVALID", "Goal length is invalid"); + const requestedTools = [...new Set(input.requestedTools ?? this.dependencies.tools.readOnlyNames())]; + const unknown = requestedTools.filter((name) => !this.dependencies.tools.has(name)); + if (unknown.length > 0) throw new EvolveError("TASK_TOOL_UNKNOWN", `Unknown requested tools: ${unknown.join(", ")}`); + const budget: TaskBudget = { ...DEFAULT_BUDGET, ...(input.budget ?? {}) }; + for (const [key, value] of Object.entries(budget)) { + if (!Number.isFinite(value) || value <= 0) throw new EvolveError("TASK_BUDGET_INVALID", `${key} must be positive`); + } + return { + id: `task_${randomUUID().replaceAll("-", "").slice(0, 24)}`, + goal, + constraints: [...new Set((input.constraints ?? []).map((value) => value.trim()).filter(Boolean))].slice(0, 50), + successCriteria: [...new Set((input.successCriteria ?? ["The answer satisfies the stated goal without unsupported factual claims"]).map((value) => value.trim()).filter(Boolean))].slice(0, 50), + requestedTools, + budget, + createdAt: new Date().toISOString(), + }; + } + + public async run(input: TaskInput): Promise { + const task = this.createTask(input); + const checkpoint: EpisodeCheckpoint = { + version: 1, + episodeId: episodeId(), + task, + status: "running", + observations: [], + evidenceIds: [], + toolSequence: [], + activeSkillIds: [], + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + turns: 0, + toolCalls: 0, + elapsedMs: 0, + updatedAt: new Date().toISOString(), + }; + return this.dependencies.leases.withLease(checkpoint.episodeId, async (lease) => { + await this.dependencies.ledger.append(checkpoint.episodeId, "episode.started", { + task_id: task.id, + goal_hash: sha256Json(task.goal), + requested_tools: task.requestedTools, + budget: { + max_turns: task.budget.maxTurns, + max_tool_calls: task.budget.maxToolCalls, + max_input_tokens: task.budget.maxInputTokens, + max_output_tokens: task.budget.maxOutputTokens, + max_wall_time_ms: task.budget.maxWallTimeMs, + }, + }); + await this.recordLease(checkpoint.episodeId, lease, "run"); + await this.dependencies.checkpoints.save(checkpoint); + return this.execute(checkpoint, false); + }); + } + + public async resume(episode: string): Promise { + return this.dependencies.leases.withLease(episode, async (lease) => { + const checkpoint = await this.dependencies.checkpoints.load(episode); + if (checkpoint.status === "committed" || checkpoint.status === "budget_exhausted") { + throw new EvolveError("EPISODE_TERMINAL", `Cannot resume ${checkpoint.status} episode ${episode}`); + } + await this.recordLease(episode, lease, "resume"); + checkpoint.status = "running"; + delete checkpoint.stopReason; + await this.dependencies.ledger.append(episode, "episode.resumed", { + turns: checkpoint.turns, + tool_calls: checkpoint.toolCalls, + elapsed_ms: checkpoint.elapsedMs, + }); + await this.dependencies.checkpoints.save(checkpoint); + return this.execute(checkpoint, true); + }); + } + + private async recordLease(episode: string, lease: EpisodeLease, phase: "run" | "resume"): Promise { + await this.dependencies.ledger.append(episode, "episode.lease_acquired", { + phase, + owner_id: lease.ownerId, + }); + if (lease.recovered) { + await this.dependencies.ledger.append(episode, "episode.stale_lock_recovered", { + previous_owner_id: lease.recovered.ownerId, + previous_pid: lease.recovered.pid, + previous_acquired_at: lease.recovered.acquiredAt, + }); + } + } + + private async stopForBudget(checkpoint: EpisodeCheckpoint, reason: string): Promise { + checkpoint.status = "budget_exhausted"; + checkpoint.stopReason = reason; + await this.dependencies.ledger.append(checkpoint.episodeId, "episode.budget_exhausted", { + reason, + turns: checkpoint.turns, + tool_calls: checkpoint.toolCalls, + usage: { + input_tokens: checkpoint.usage.inputTokens, + output_tokens: checkpoint.usage.outputTokens, + total_tokens: checkpoint.usage.totalTokens, + }, + elapsed_ms: checkpoint.elapsedMs, + }); + await this.dependencies.checkpoints.save(checkpoint); + await this.observeEvolution(checkpoint); + return resultOf(checkpoint); + } + + private async observeEvolution(checkpoint: EpisodeCheckpoint): Promise { + if (!this.dependencies.evolution) return; + try { + await this.dependencies.evolution.observeTerminal(checkpoint); + } catch (error: unknown) { + await this.dependencies.ledger.append(checkpoint.episodeId, "evaluation.observer_failed", { + reason: errorMessage(error), + }); + } + } + + private async execute(checkpoint: EpisodeCheckpoint, resumed: boolean): Promise { + const segmentStartedAt = Date.now(); + const elapsedBeforeSegment = checkpoint.elapsedMs; + const updateElapsed = (): void => { + checkpoint.elapsedMs = elapsedBeforeSegment + (Date.now() - segmentStartedAt); + }; + + try { + while (checkpoint.status === "running") { + updateElapsed(); + const preReason = preTurnBudgetReason(checkpoint); + if (preReason) return this.stopForBudget(checkpoint, preReason); + + const prompt = await this.dependencies.context.compile({ + task: checkpoint.task, + observations: checkpoint.observations as Observation[], + usage: checkpoint.usage, + turns: checkpoint.turns, + toolCalls: checkpoint.toolCalls, + elapsedMs: checkpoint.elapsedMs, + }); + checkpoint.activeSkillIds = [...new Set([...(checkpoint.activeSkillIds ?? []), ...prompt.skills.map((skill) => skill.id)])]; + const decisionResult = await this.dependencies.provider.decide(prompt); + checkpoint.turns += 1; + checkpoint.usage = addUsage(checkpoint.usage, decisionResult.usage); + updateElapsed(); + + if (decisionResult.decision.kind === "tool") { + const decision = decisionResult.decision; + if (checkpoint.toolCalls >= checkpoint.task.budget.maxToolCalls) { + return this.stopForBudget( + checkpoint, + `Tool-call budget exhausted (${checkpoint.task.budget.maxToolCalls})`, + ); + } + await this.dependencies.ledger.append(checkpoint.episodeId, "model.tool_proposed", { + turn: checkpoint.turns, + tool: decision.toolName, + args_hash: sha256Json(decision.args), + ...(decisionResult.providerResponseId ? { provider_response_id: decisionResult.providerResponseId } : {}), + }); + + const tool = this.dependencies.tools.get(decision.toolName); + let validatedArgs: Record; + try { + validatedArgs = tool.validate(decision.args); + } catch (error: unknown) { + checkpoint.observations.push({ + id: observationId(), + kind: "error", + toolName: decision.toolName, + content: `Tool arguments rejected: ${errorMessage(error)}`, + }); + await this.dependencies.ledger.append(checkpoint.episodeId, "tool.arguments_rejected", { + tool: decision.toolName, + error: errorMessage(error), + }); + await this.dependencies.checkpoints.save(checkpoint); + continue; + } + + const policy = this.dependencies.risk.evaluate(checkpoint.task, tool, validatedArgs); + if (!policy.allowed) { + checkpoint.observations.push({ + id: observationId(), + kind: "error", + toolName: decision.toolName, + content: `Policy denied tool call: ${policy.reason}`, + }); + await this.dependencies.ledger.append(checkpoint.episodeId, "tool.policy_denied", { + tool: decision.toolName, + reason: policy.reason, + }); + await this.dependencies.checkpoints.save(checkpoint); + continue; + } + + if (policy.approvalRequired) { + const approved = await this.dependencies.approver.approve({ + episodeId: checkpoint.episodeId, + toolName: decision.toolName, + risk: tool.risk as "write" | "execute" | "external", + args: validatedArgs, + reason: policy.reason, + }); + await this.dependencies.ledger.append(checkpoint.episodeId, approved ? "approval.granted" : "approval.denied", { + tool: decision.toolName, + args_hash: sha256Json(validatedArgs), + }); + if (!approved) { + checkpoint.observations.push({ + id: observationId(), + kind: "error", + toolName: decision.toolName, + content: "Human approval was denied. Choose a safer action or explain the limitation.", + }); + await this.dependencies.checkpoints.save(checkpoint); + continue; + } + } + + const capabilityToken = await this.dependencies.capabilities.issue({ + episodeId: checkpoint.episodeId, + toolName: decision.toolName, + args: validatedArgs, + }); + checkpoint.toolCalls += 1; + let evidence; + try { + const { execution } = await this.dependencies.tools.execute({ + episodeId: checkpoint.episodeId, + toolName: decision.toolName, + rawArgs: validatedArgs, + capabilityToken, + }); + evidence = await this.dependencies.artifacts.put({ + episodeId: checkpoint.episodeId, + toolName: decision.toolName, + args: validatedArgs, + data: execution.data, + summary: execution.summary, + success: execution.success, + }); + checkpoint.evidenceIds.push(evidence.id); + if (execution.success) checkpoint.toolSequence.push(decision.toolName); + checkpoint.observations.push({ + id: observationId(), + kind: execution.success ? "tool" : "error", + toolName: decision.toolName, + evidenceId: evidence.id, + content: `${execution.summary} [evidence:${evidence.id}]`, + }); + await this.dependencies.ledger.append(checkpoint.episodeId, "tool.executed", { + tool: decision.toolName, + success: execution.success, + evidence_id: evidence.id, + artifact_hash: evidence.artifactHash, + args_hash: evidence.argsHash, + }); + } catch (error: unknown) { + evidence = await this.dependencies.artifacts.put({ + episodeId: checkpoint.episodeId, + toolName: decision.toolName, + args: validatedArgs, + data: { error: errorMessage(error) }, + summary: `${decision.toolName} failed: ${errorMessage(error)}`, + success: false, + }); + checkpoint.evidenceIds.push(evidence.id); + checkpoint.observations.push({ + id: observationId(), + kind: "error", + toolName: decision.toolName, + evidenceId: evidence.id, + content: `${decision.toolName} failed: ${errorMessage(error)} [evidence:${evidence.id}]`, + }); + await this.dependencies.ledger.append(checkpoint.episodeId, "tool.failed", { + tool: decision.toolName, + evidence_id: evidence.id, + error: errorMessage(error), + }); + } + updateElapsed(); + await this.dependencies.checkpoints.save(checkpoint); + continue; + } + + const decision = decisionResult.decision; + await this.dependencies.ledger.append(checkpoint.episodeId, "model.final_proposed", { + turn: checkpoint.turns, + answer_hash: sha256Json(decision.answer), + evidence_ids: decision.evidenceIds, + ...(decisionResult.providerResponseId ? { provider_response_id: decisionResult.providerResponseId } : {}), + }); + const verification = await this.dependencies.verifier.verify(checkpoint.episodeId, checkpoint.task, decision); + checkpoint.usage = addUsage(checkpoint.usage, verification.usage); + updateElapsed(); + + const postReason = hardOverrunReason(checkpoint); + if (postReason) return this.stopForBudget(checkpoint, postReason); + + if (!verification.passed) { + checkpoint.observations.push({ + id: observationId(), + kind: "verification", + content: `Final answer rejected (score ${verification.verdict.score.toFixed(2)}): ${verification.verdict.feedback}`, + }); + await this.dependencies.ledger.append(checkpoint.episodeId, "verification.rejected", { + score: verification.verdict.score, + feedback: verification.verdict.feedback, + }); + await this.dependencies.checkpoints.save(checkpoint); + continue; + } + + checkpoint.answer = decision.answer; + checkpoint.finalScore = verification.verdict.score; + checkpoint.status = "committed"; + const validEvidence = new Set(verification.validEvidenceIds); + for (const proposal of decision.memoryProposals) { + try { + const memory = await this.dependencies.memory.add({ + text: proposal.text, + tags: proposal.tags, + confidence: proposal.confidence, + sourceEpisodeId: checkpoint.episodeId, + evidenceIds: proposal.evidenceIds, + validEvidenceIds: validEvidence, + }); + await this.dependencies.ledger.append(checkpoint.episodeId, "memory.accepted", { + memory_id: memory.id, + evidence_ids: memory.evidenceIds, + confidence: memory.confidence, + }); + } catch (error: unknown) { + await this.dependencies.ledger.append(checkpoint.episodeId, "memory.rejected", { + reason: errorMessage(error), + text_hash: sha256Json(proposal.text), + }); + } + } + + await this.dependencies.ledger.append(checkpoint.episodeId, "episode.committed", { + answer_hash: sha256Json(decision.answer), + evidence_ids: verification.validEvidenceIds, + score: verification.verdict.score, + turns: checkpoint.turns, + tool_calls: checkpoint.toolCalls, + usage: { + input_tokens: checkpoint.usage.inputTokens, + output_tokens: checkpoint.usage.outputTokens, + total_tokens: checkpoint.usage.totalTokens, + }, + resumed, + }); + await this.dependencies.checkpoints.save(checkpoint); + await this.observeEvolution(checkpoint); + await this.dependencies.learning.observeCommitted(checkpoint); + return resultOf(checkpoint); + } + return resultOf(checkpoint); + } catch (error: unknown) { + updateElapsed(); + const isProviderInterruption = + error instanceof EvolveError && ["PROVIDER_AUTH", "PROVIDER_HTTP", "PROVIDER_TIMEOUT", "PROVIDER_NETWORK"].includes(error.code); + checkpoint.status = isProviderInterruption ? "interrupted" : "failed"; + checkpoint.stopReason = errorMessage(error); + await this.dependencies.ledger.append(checkpoint.episodeId, isProviderInterruption ? "episode.interrupted" : "episode.failed", { + reason: checkpoint.stopReason, + turns: checkpoint.turns, + tool_calls: checkpoint.toolCalls, + }); + await this.dependencies.checkpoints.save(checkpoint); + await this.observeEvolution(checkpoint); + return resultOf(checkpoint); + } + } +} diff --git a/evolve-agent/src/runtime/checkpoint-store.ts b/evolve-agent/src/runtime/checkpoint-store.ts new file mode 100644 index 0000000..9d4fac3 --- /dev/null +++ b/evolve-agent/src/runtime/checkpoint-store.ts @@ -0,0 +1,32 @@ +import path from "node:path"; +import { atomicWriteJson, readJsonFile } from "../core/fs.js"; +import { EvolveError } from "../core/errors.js"; +import type { EpisodeCheckpoint } from "../core/types.js"; + +export class CheckpointStore { + private readonly directory: string; + + public constructor(home: string) { + this.directory = path.join(home, "checkpoints"); + } + + private pathFor(episodeId: string): string { + if (!/^ep_[a-f0-9]{24}$/.test(episodeId)) throw new EvolveError("EPISODE_ID_INVALID", "Invalid episode ID"); + return path.join(this.directory, `${episodeId}.json`); + } + + public async save(checkpoint: EpisodeCheckpoint): Promise { + checkpoint.updatedAt = new Date().toISOString(); + await atomicWriteJson(this.pathFor(checkpoint.episodeId), checkpoint, 0o600); + } + + public async load(episodeId: string): Promise { + const checkpoint = await readJsonFile(this.pathFor(episodeId), null); + if (!checkpoint) throw new EvolveError("EPISODE_NOT_FOUND", `No checkpoint for ${episodeId}`); + if (checkpoint.version !== 1 || checkpoint.episodeId !== episodeId) { + throw new EvolveError("CHECKPOINT_CORRUPT", `Invalid checkpoint for ${episodeId}`); + } + checkpoint.activeSkillIds ??= []; + return checkpoint; + } +} diff --git a/evolve-agent/src/runtime/lease-manager.ts b/evolve-agent/src/runtime/lease-manager.ts new file mode 100644 index 0000000..ac8acb0 --- /dev/null +++ b/evolve-agent/src/runtime/lease-manager.ts @@ -0,0 +1,188 @@ +import { randomUUID } from "node:crypto"; +import { hostname } from "node:os"; +import { open, readFile, rename, rm, stat, unlink } from "node:fs/promises"; +import path from "node:path"; +import type { FileHandle } from "node:fs/promises"; +import { EvolveError } from "../core/errors.js"; +import { ensureDir } from "../core/fs.js"; + +export interface EpisodeLeaseRecord { + version: 1; + episodeId: string; + ownerId: string; + pid: number; + hostname: string; + acquiredAt: string; +} + +export interface EpisodeLease { + episodeId: string; + ownerId: string; + recovered?: EpisodeLeaseRecord; + release(): Promise; +} + +export interface EpisodeLeaseOptions { + ttlMs: number; + heartbeatMs: number; + ownerId?: string; + hostname?: string; +} + +function validateEpisodeId(episodeId: string): void { + if (!/^ep_[a-f0-9]{24}$/.test(episodeId)) throw new EvolveError("EPISODE_ID_INVALID", "Invalid episode ID"); +} + +async function readRecord(target: string): Promise { + try { + const value = JSON.parse(await readFile(target, "utf8")) as Partial; + if ( + value.version !== 1 || + typeof value.ownerId !== "string" || + typeof value.episodeId !== "string" || + typeof value.pid !== "number" || + typeof value.hostname !== "string" || + typeof value.acquiredAt !== "string" + ) { + return undefined; + } + return value as EpisodeLeaseRecord; + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + return undefined; + } +} + +function processIsAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error: unknown) { + const code = (error as NodeJS.ErrnoException).code; + return code === "EPERM"; + } +} + +export class EpisodeLeaseManager { + private readonly directory: string; + private readonly ownerId: string; + private readonly host: string; + + public constructor(home: string, private readonly options: EpisodeLeaseOptions) { + if (!Number.isFinite(options.ttlMs) || options.ttlMs < 1_000) throw new Error("Lease TTL must be at least 1000 ms"); + if (!Number.isFinite(options.heartbeatMs) || options.heartbeatMs < 100 || options.heartbeatMs * 2 >= options.ttlMs) { + throw new Error("Lease heartbeat must be at least 100 ms and less than half of the TTL"); + } + this.directory = path.join(home, "leases"); + this.ownerId = options.ownerId ?? `${process.pid}-${randomUUID()}`; + this.host = options.hostname ?? hostname(); + } + + private pathFor(episodeId: string): string { + validateEpisodeId(episodeId); + return path.join(this.directory, `${episodeId}.lock`); + } + + public async acquire(episodeId: string): Promise { + const target = this.pathFor(episodeId); + await ensureDir(this.directory); + let recovered: EpisodeLeaseRecord | undefined; + + for (let attempt = 0; attempt < 16; attempt += 1) { + let handle: FileHandle; + try { + handle = await open(target, "wx", 0o600); + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + let currentStats; + try { + currentStats = await stat(target); + } catch (statError: unknown) { + if ((statError as NodeJS.ErrnoException).code === "ENOENT") continue; + throw statError; + } + const ageMs = Date.now() - currentStats.mtimeMs; + const current = await readRecord(target); + if (ageMs < this.options.ttlMs) { + throw new EvolveError( + "EPISODE_LOCKED", + `Episode ${episodeId} is leased${current ? ` by ${current.ownerId}` : ""}; retry after the lease TTL`, + ); + } + if (current?.hostname === this.host && processIsAlive(current.pid)) { + throw new EvolveError( + "EPISODE_LOCK_HEARTBEAT_STALE", + `Episode ${episodeId} has a stale heartbeat but its local owner process ${current.pid} is still alive`, + ); + } + + recovered = current; + const quarantine = path.join( + this.directory, + `${episodeId}.stale.${Date.now()}.${randomUUID().replaceAll("-", "")}`, + ); + try { + await rename(target, quarantine); + await rm(quarantine, { force: true }); + } catch (renameError: unknown) { + if ((renameError as NodeJS.ErrnoException).code === "ENOENT") continue; + throw renameError; + } + continue; + } + + const record: EpisodeLeaseRecord = { + version: 1, + episodeId, + ownerId: this.ownerId, + pid: process.pid, + hostname: this.host, + acquiredAt: new Date().toISOString(), + }; + try { + await handle.writeFile(`${JSON.stringify(record)}\n`, "utf8"); + await handle.sync(); + } catch (error: unknown) { + await handle.close().catch(() => undefined); + await unlink(target).catch(() => undefined); + throw error; + } + + let released = false; + const heartbeat = setInterval(() => { + const now = new Date(); + void handle.utimes(now, now).catch(() => undefined); + }, this.options.heartbeatMs); + heartbeat.unref(); + + return { + episodeId, + ownerId: this.ownerId, + ...(recovered !== undefined ? { recovered } : {}), + release: async () => { + if (released) return; + released = true; + clearInterval(heartbeat); + await handle.close().catch(() => undefined); + const current = await readRecord(target); + if (current?.ownerId !== this.ownerId) return; + await unlink(target).catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + }); + }, + }; + } + + throw new EvolveError("LEASE_ACQUIRE_FAILED", `Could not acquire a stable lease for ${episodeId}`); + } + + public async withLease(episodeId: string, callback: (lease: EpisodeLease) => Promise): Promise { + const lease = await this.acquire(episodeId); + try { + return await callback(lease); + } finally { + await lease.release(); + } + } +} diff --git a/evolve-agent/src/secrets/secret-broker.ts b/evolve-agent/src/secrets/secret-broker.ts new file mode 100644 index 0000000..7effa2d --- /dev/null +++ b/evolve-agent/src/secrets/secret-broker.ts @@ -0,0 +1,149 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { EvolveError } from "../core/errors.js"; +import { ensureDir } from "../core/fs.js"; + +const SECRET_NAME = /^[A-Z][A-Z0-9_]{0,63}$/; +const MAX_SECRET_BYTES = 64 * 1024; + +export interface SecretMount { + name: string; + hostPath: string; + containerPath: string; +} + +export interface SecretLease { + mounts: SecretMount[]; + redact(text: string): string; + release(): Promise; +} + +export interface SecretBroker { + materialize(runId: string, names: string[]): Promise; + sweepExpired(now?: number): Promise; + allowedNames(): string[]; +} + +export type SecretSource = (name: string) => string | undefined; + +interface SecretLeaseMetadata { + version: 1; + createdAt: string; + expiresAt: string; +} + +function redactWith(values: Array<{ name: string; value: string }>, text: string): string { + let output = text; + for (const secret of [...values].sort((left, right) => right.value.length - left.value.length)) { + output = output.replaceAll(secret.value, `[REDACTED_SECRET:${secret.name}]`); + } + return output; +} + +export class FileSecretBroker implements SecretBroker { + private readonly allowed: Set; + + public constructor( + private readonly root: string, + allowedNames: Iterable, + private readonly ttlMs: number, + private readonly source: SecretSource = (name) => process.env[name], + ) { + if (!Number.isFinite(ttlMs) || ttlMs < 1_000) throw new Error("Secret TTL must be at least 1000 ms"); + this.allowed = new Set([...allowedNames].map((value) => value.trim()).filter(Boolean)); + } + + public allowedNames(): string[] { + return [...this.allowed].sort(); + } + + public async materialize(runId: string, names: string[]): Promise { + const unique = [...new Set(names)].sort(); + if (unique.length === 0) { + return { mounts: [], redact: (text) => text, release: async () => undefined }; + } + for (const name of unique) { + if (!SECRET_NAME.test(name)) throw new EvolveError("SECRET_NAME_INVALID", `Invalid secret name: ${name}`); + if (!this.allowed.has(name)) throw new EvolveError("SECRET_NOT_ALLOWED", `${name} is not in EVOLVE_SECRET_ALLOWLIST`); + } + + await this.sweepExpired(); + await ensureDir(this.root); + const safeRunId = runId.replace(/[^a-zA-Z0-9_.-]/g, "_").slice(0, 80); + const directory = path.join(this.root, `${safeRunId}-${randomUUID().replaceAll("-", "")}`); + await mkdir(directory, { mode: 0o700 }); + + let released = false; + try { + const mounts: SecretMount[] = []; + const values: Array<{ name: string; value: string }> = []; + for (const name of unique) { + const value = this.source(name); + if (value === undefined) throw new EvolveError("SECRET_UNAVAILABLE", `${name} is allowlisted but not configured`); + if (value.length === 0) throw new EvolveError("SECRET_EMPTY", `${name} is configured with an empty value`); + if (Buffer.byteLength(value, "utf8") > MAX_SECRET_BYTES) { + throw new EvolveError("SECRET_TOO_LARGE", `${name} exceeds the ${MAX_SECRET_BYTES}-byte secret limit`); + } + const hostPath = path.join(directory, name); + await writeFile(hostPath, value, { encoding: "utf8", mode: 0o600, flag: "wx" }); + mounts.push({ name, hostPath, containerPath: `/run/secrets/${name}` }); + values.push({ name, value }); + } + const metadata: SecretLeaseMetadata = { + version: 1, + createdAt: new Date().toISOString(), + expiresAt: new Date(Date.now() + this.ttlMs).toISOString(), + }; + await writeFile(path.join(directory, ".lease.json"), `${JSON.stringify(metadata)}\n`, { + encoding: "utf8", + mode: 0o600, + flag: "wx", + }); + return { + mounts, + redact: (text) => redactWith(values, text), + release: async () => { + if (released) return; + released = true; + await rm(directory, { recursive: true, force: true }); + }, + }; + } catch (error: unknown) { + await rm(directory, { recursive: true, force: true }); + throw error; + } + } + + private async expired(target: string, now: number): Promise { + try { + const metadata = JSON.parse(await readFile(path.join(target, ".lease.json"), "utf8")) as Partial; + if (metadata.version === 1 && typeof metadata.expiresAt === "string") { + const expiresAt = Date.parse(metadata.expiresAt); + if (Number.isFinite(expiresAt)) return expiresAt <= now; + } + } catch { + // Fall back to directory age for interrupted or malformed materialization. + } + const stats = await stat(target); + return stats.mtimeMs + this.ttlMs <= now; + } + + public async sweepExpired(now = Date.now()): Promise { + await ensureDir(this.root); + const entries = await readdir(this.root, { withFileTypes: true }); + let removed = 0; + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const target = path.join(this.root, entry.name); + try { + if (!(await this.expired(target, now))) continue; + await rm(target, { recursive: true, force: true }); + removed += 1; + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + return removed; + } +} diff --git a/evolve-agent/src/skills/skill-store.ts b/evolve-agent/src/skills/skill-store.ts new file mode 100644 index 0000000..f91660a --- /dev/null +++ b/evolve-agent/src/skills/skill-store.ts @@ -0,0 +1,203 @@ +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { atomicWriteJson, readJsonFile } from "../core/fs.js"; +import { EvolveError } from "../core/errors.js"; +import type { SkillRecord, SkillStep } from "../core/types.js"; +import type { PromotionAuthorization } from "../evaluation/types.js"; + +export interface SkillPromotionVerifier { + verifyPromotion(skill: SkillRecord, authorization: PromotionAuthorization): Promise; +} + +interface SkillFileV1 { + version: 1; + records: Array>>; +} + +interface SkillFile { + version: 2; + records: SkillRecord[]; +} + +function migrate(file: SkillFile | SkillFileV1): SkillFile { + if (file.version === 2) return file; + return { + version: 2, + records: file.records.map((record) => ({ + ...record, + evaluationReportIds: [...(record.evaluationReportIds ?? [])], + canaryReportIds: [...(record.canaryReportIds ?? [])], + })), + }; +} + +export class SkillStore { + private readonly filePath: string; + + public constructor(home: string, private readonly promotionVerifier?: SkillPromotionVerifier) { + this.filePath = path.join(home, "skills.json"); + } + + private async load(): Promise { + const raw = await readJsonFile(this.filePath, { version: 2, records: [] }); + if ((raw.version !== 1 && raw.version !== 2) || !Array.isArray(raw.records)) { + throw new EvolveError("SKILLS_CORRUPT", "Unsupported skills file"); + } + const file = migrate(raw); + if (raw.version === 1) await this.save(file); + return file; + } + + private async save(file: SkillFile): Promise { + await atomicWriteJson(this.filePath, file, 0o600); + } + + public async list(): Promise { + return (await this.load()).records; + } + + public async promoted(): Promise { + return (await this.list()).filter((record) => record.status === "promoted"); + } + + public async get(id: string): Promise { + const skill = (await this.list()).find((record) => record.id === id); + if (!skill) throw new EvolveError("SKILL_NOT_FOUND", `Unknown skill: ${id}`); + return skill; + } + + public async upsertCandidate(input: { + fingerprint: string; + name: string; + description: string; + triggers: string[]; + steps: SkillStep[]; + allowedTools: string[]; + supportingEpisodes: string[]; + provenanceEvidenceIds: string[]; + }): Promise { + const file = await this.load(); + const now = new Date().toISOString(); + let skill = file.records.find((record) => record.fingerprint === input.fingerprint && record.status !== "rolled_back"); + if (skill) { + // Once evaluation starts, freeze the training provenance. New Episodes become independent holdout material. + if (skill.status === "candidate") { + skill.supportingEpisodes = [...new Set([...skill.supportingEpisodes, ...input.supportingEpisodes])]; + skill.provenanceEvidenceIds = [...new Set([...skill.provenanceEvidenceIds, ...input.provenanceEvidenceIds])]; + } + skill.triggers = [...new Set([...skill.triggers, ...input.triggers])].slice(0, 20); + skill.updatedAt = now; + } else { + skill = { + id: `skill_${randomUUID().replaceAll("-", "").slice(0, 24)}`, + fingerprint: input.fingerprint, + name: input.name.slice(0, 120), + description: input.description.slice(0, 1_000), + triggers: [...new Set(input.triggers)].slice(0, 20), + steps: input.steps.slice(0, 30), + allowedTools: [...new Set(input.allowedTools)].slice(0, 30), + supportingEpisodes: [...new Set(input.supportingEpisodes)], + provenanceEvidenceIds: [...new Set(input.provenanceEvidenceIds)], + status: "candidate", + createdAt: now, + updatedAt: now, + evaluations: [], + canaries: [], + evaluationReportIds: [], + canaryReportIds: [], + }; + file.records.push(skill); + } + await this.save(file); + return skill; + } + + public async attachEvaluationReport(id: string, reportId: string, passed: boolean): Promise { + const file = await this.load(); + const skill = file.records.find((record) => record.id === id); + if (!skill) throw new EvolveError("SKILL_NOT_FOUND", `Unknown skill: ${id}`); + if (skill.status === "promoted" || skill.status === "rolled_back") { + throw new EvolveError("SKILL_STATE", `Cannot attach an offline evaluation to a ${skill.status} Skill`); + } + skill.evaluationReportIds = [...new Set([...skill.evaluationReportIds, reportId])]; + skill.status = passed ? "evaluated" : "quarantined"; + skill.updatedAt = new Date().toISOString(); + await this.save(file); + return skill; + } + + public async attachCanaryReport(id: string, reportId: string, passed: boolean): Promise { + const file = await this.load(); + const skill = file.records.find((record) => record.id === id); + if (!skill) throw new EvolveError("SKILL_NOT_FOUND", `Unknown skill: ${id}`); + if (skill.status === "promoted" || skill.status === "rolled_back") { + throw new EvolveError("SKILL_STATE", `Cannot attach a canary evaluation to a ${skill.status} Skill`); + } + if (skill.evaluationReportIds.length === 0) { + throw new EvolveError("SKILL_GATE", "Canary requires a signed offline evaluation report"); + } + skill.canaryReportIds = [...new Set([...skill.canaryReportIds, reportId])]; + skill.status = passed ? "canary" : "quarantined"; + skill.updatedAt = new Date().toISOString(); + await this.save(file); + return skill; + } + + public async promote(id: string, authorization: PromotionAuthorization): Promise { + const file = await this.load(); + const skill = file.records.find((record) => record.id === id); + if (!skill) throw new EvolveError("SKILL_NOT_FOUND", `Unknown skill: ${id}`); + if (skill.status !== "canary") throw new EvolveError("SKILL_GATE", "Promotion requires a passing canary state"); + if (!skill.evaluationReportIds.includes(authorization.offlineReportId)) { + throw new EvolveError("SKILL_GATE", "Offline report is not attached to this Skill"); + } + if (!skill.canaryReportIds.includes(authorization.canaryReportId)) { + throw new EvolveError("SKILL_GATE", "Canary report is not attached to this Skill"); + } + if (!this.promotionVerifier) { + throw new EvolveError("SKILL_AUTHORITY", "A signed-report promotion verifier is required"); + } + await this.promotionVerifier.verifyPromotion(skill, authorization); + const at = new Date().toISOString(); + skill.status = "promoted"; + skill.promotion = { at, ...authorization }; + delete skill.rollback; + skill.updatedAt = at; + await this.save(file); + return skill; + } + + public async quarantine(id: string, reason: string): Promise { + const file = await this.load(); + const skill = file.records.find((record) => record.id === id); + if (!skill) throw new EvolveError("SKILL_NOT_FOUND", `Unknown skill: ${id}`); + if (skill.status === "promoted") throw new EvolveError("SKILL_STATE", "Use rollback for a promoted Skill"); + skill.status = "quarantined"; + skill.updatedAt = new Date().toISOString(); + skill.canaries.push({ at: skill.updatedAt, passed: false, score: 0, note: `Quarantine: ${reason.slice(0, 1_000)}` }); + await this.save(file); + return skill; + } + + public async rollback( + id: string, + reason: string, + options: { automatic?: boolean; reportId?: string } = {}, + ): Promise { + const file = await this.load(); + const skill = file.records.find((record) => record.id === id); + if (!skill) throw new EvolveError("SKILL_NOT_FOUND", `Unknown skill: ${id}`); + const at = new Date().toISOString(); + skill.status = "rolled_back"; + skill.updatedAt = at; + skill.rollback = { + at, + reason: reason.slice(0, 2_000), + automatic: options.automatic ?? false, + ...(options.reportId ? { reportId: options.reportId } : {}), + }; + skill.canaries.push({ at, passed: false, score: 0, note: `Rollback: ${reason.slice(0, 1_000)}` }); + await this.save(file); + return skill; + } +} diff --git a/evolve-agent/src/tools/list-files.ts b/evolve-agent/src/tools/list-files.ts new file mode 100644 index 0000000..f385d7a --- /dev/null +++ b/evolve-agent/src/tools/list-files.ts @@ -0,0 +1,59 @@ +import { readdir } from "node:fs/promises"; +import path from "node:path"; +import type { ToolDefinition, ToolExecution } from "./types.js"; +import { numberArg, rejectUnknownKeys, stringArg } from "./validate.js"; +import { displayPath, resolveWorkspacePath } from "./workspace.js"; + +const ignored = new Set([".git", ".evolve", "node_modules", "dist", ".test-dist"]); + +export const listFilesTool: ToolDefinition = { + name: "list_files", + description: "List workspace files and directories recursively with strict depth and entry limits.", + risk: "read", + inputSchema: { + type: "object", + properties: { + path: { type: "string", description: "Workspace-relative directory" }, + depth: { type: "integer", minimum: 0, maximum: 6 }, + max_entries: { type: "integer", minimum: 1, maximum: 1000 }, + }, + additionalProperties: false, + }, + validate(args) { + rejectUnknownKeys(args, ["path", "depth", "max_entries"]); + return { + path: stringArg(args, "path", { fallback: ".", max: 4096 }) as string, + depth: numberArg(args, "depth", { fallback: 2, min: 0, max: 6, integer: true }) as number, + max_entries: numberArg(args, "max_entries", { fallback: 300, min: 1, max: 1000, integer: true }) as number, + }; + }, + async execute(args, context): Promise { + const target = await resolveWorkspacePath(context.workspace, args.path as string); + const output: string[] = []; + let truncated = false; + const maxEntries = args.max_entries as number; + + async function visit(directory: string, depth: number): Promise { + if (truncated) return; + const entries = await readdir(directory, { withFileTypes: true }); + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + if (ignored.has(entry.name)) continue; + const absolute = path.join(directory, entry.name); + output.push(`${displayPath(context.workspace, absolute)}${entry.isDirectory() ? "/" : ""}`); + if (output.length >= maxEntries) { + truncated = true; + return; + } + if (entry.isDirectory() && depth > 0) await visit(absolute, depth - 1); + } + } + + await visit(target, args.depth as number); + return { + success: true, + summary: `Listed ${output.length} entries under ${displayPath(context.workspace, target)}${truncated ? " (truncated)" : ""}`, + data: { entries: output, truncated }, + }; + }, +}; diff --git a/evolve-agent/src/tools/read-file.ts b/evolve-agent/src/tools/read-file.ts new file mode 100644 index 0000000..58a7d34 --- /dev/null +++ b/evolve-agent/src/tools/read-file.ts @@ -0,0 +1,47 @@ +import { readFile, stat } from "node:fs/promises"; +import { sha256Bytes } from "../core/hash.js"; +import type { ToolDefinition, ToolExecution } from "./types.js"; +import { numberArg, rejectUnknownKeys, stringArg } from "./validate.js"; +import { displayPath, resolveWorkspacePath } from "./workspace.js"; + +export const readFileTool: ToolDefinition = { + name: "read_file", + description: "Read a UTF-8 workspace file with a byte cap and return its content hash.", + risk: "read", + inputSchema: { + type: "object", + properties: { + path: { type: "string" }, + max_bytes: { type: "integer", minimum: 1, maximum: 500000 }, + }, + required: ["path"], + additionalProperties: false, + }, + validate(args) { + rejectUnknownKeys(args, ["path", "max_bytes"]); + return { + path: stringArg(args, "path", { required: true, min: 1, max: 4096 }) as string, + max_bytes: numberArg(args, "max_bytes", { fallback: 200_000, min: 1, max: 500_000, integer: true }) as number, + }; + }, + async execute(args, context): Promise { + const target = await resolveWorkspacePath(context.workspace, args.path as string); + const metadata = await stat(target); + if (!metadata.isFile()) throw new Error("read_file target is not a regular file"); + const buffer = await readFile(target); + const limit = args.max_bytes as number; + const slice = buffer.subarray(0, limit); + const content = slice.toString("utf8"); + return { + success: true, + summary: `Read ${displayPath(context.workspace, target)} (${buffer.length} bytes${buffer.length > limit ? ", truncated" : ""})`, + data: { + path: displayPath(context.workspace, target), + content, + sha256: sha256Bytes(buffer), + bytes: buffer.length, + truncated: buffer.length > limit, + }, + }; + }, +}; diff --git a/evolve-agent/src/tools/registry.ts b/evolve-agent/src/tools/registry.ts new file mode 100644 index 0000000..ce6e748 --- /dev/null +++ b/evolve-agent/src/tools/registry.ts @@ -0,0 +1,78 @@ +import { EvolveError } from "../core/errors.js"; +import { ExecutorRegistry } from "../execution/executor-registry.js"; +import { LocalExecutor } from "../execution/local-executor.js"; +import { NodeProcessRunner } from "../execution/process-runner.js"; +import type { CapabilityAuthority } from "../policy/capability.js"; +import { listFilesTool } from "./list-files.js"; +import { readFileTool } from "./read-file.js"; +import { replaceTextTool } from "./replace-text.js"; +import { runProcessTool } from "./run-process.js"; +import { searchTextTool } from "./search-text.js"; +import type { ToolDefinition, ToolDescription, ToolExecution } from "./types.js"; +import { assertObject } from "./validate.js"; +import { writeFileTool } from "./write-file.js"; + +export interface ToolRegistryContext { + workspace: string; + allowedCommands: Set; + executors?: ExecutorRegistry; +} + +export class ToolRegistry { + private readonly definitions = new Map(); + private readonly executors: ExecutorRegistry; + + public constructor( + private readonly capabilityAuthority: CapabilityAuthority, + private readonly context: ToolRegistryContext, + tools: ToolDefinition[] = [listFilesTool, readFileTool, searchTextTool, writeFileTool, replaceTextTool, runProcessTool], + ) { + this.executors = context.executors ?? new ExecutorRegistry("local", [new LocalExecutor(new NodeProcessRunner())]); + for (const tool of tools) { + if (this.definitions.has(tool.name)) throw new Error(`Duplicate tool name: ${tool.name}`); + this.definitions.set(tool.name, tool); + } + } + + public has(name: string): boolean { + return this.definitions.has(name); + } + + public get(name: string): ToolDefinition { + const tool = this.definitions.get(name); + if (!tool) throw new EvolveError("TOOL_UNKNOWN", `Unknown tool: ${name}`); + return tool; + } + + public modelDescriptions(): ToolDescription[] { + return [...this.definitions.values()] + .map(({ name, description, risk, inputSchema }) => ({ name, description, risk, inputSchema })) + .sort((left, right) => left.name.localeCompare(right.name)); + } + + public readOnlyNames(): string[] { + return [...this.definitions.values()].filter((tool) => tool.risk === "read").map((tool) => tool.name).sort(); + } + + public async execute(input: { + episodeId: string; + toolName: string; + rawArgs: unknown; + capabilityToken: string; + }): Promise<{ args: Record; execution: ToolExecution }> { + const tool = this.get(input.toolName); + const args = tool.validate(assertObject(input.rawArgs)); + await this.capabilityAuthority.verify(input.capabilityToken, { + episodeId: input.episodeId, + toolName: input.toolName, + args, + }); + const execution = await tool.execute(args, { + episodeId: input.episodeId, + workspace: this.context.workspace, + allowedCommands: this.context.allowedCommands, + executors: this.executors, + }); + return { args, execution }; + } +} diff --git a/evolve-agent/src/tools/replace-text.ts b/evolve-agent/src/tools/replace-text.ts new file mode 100644 index 0000000..02965e7 --- /dev/null +++ b/evolve-agent/src/tools/replace-text.ts @@ -0,0 +1,91 @@ +import { readFile, stat } from "node:fs/promises"; +import { atomicWriteFile } from "../core/fs.js"; +import { sha256Bytes } from "../core/hash.js"; +import { EvolveError } from "../core/errors.js"; +import type { ToolDefinition, ToolExecution } from "./types.js"; +import { numberArg, rejectUnknownKeys, stringArg } from "./validate.js"; +import { displayPath, resolveWorkspacePath } from "./workspace.js"; + +function countOccurrences(text: string, needle: string): number { + let count = 0; + let offset = 0; + while (true) { + const index = text.indexOf(needle, offset); + if (index === -1) return count; + count += 1; + offset = index + Math.max(1, needle.length); + } +} + +export const replaceTextTool: ToolDefinition = { + name: "replace_text", + description: + "Replace an exact text fragment in one UTF-8 workspace file. Fails unless the occurrence count and optional file hash match.", + risk: "write", + inputSchema: { + type: "object", + properties: { + path: { type: "string" }, + old_text: { type: "string" }, + new_text: { type: "string" }, + expected_replacements: { type: "integer", minimum: 1, maximum: 1000 }, + expected_sha256: { type: "string" }, + }, + required: ["path", "old_text", "new_text"], + additionalProperties: false, + }, + validate(args) { + rejectUnknownKeys(args, ["path", "old_text", "new_text", "expected_replacements", "expected_sha256"]); + const oldText = stringArg(args, "old_text", { required: true, min: 1, max: 200_000 }) as string; + const expected = stringArg(args, "expected_sha256", { min: 64, max: 64 }); + if (expected !== undefined && !/^[a-f0-9]{64}$/.test(expected)) { + throw new EvolveError("TOOL_ARGS_INVALID", "expected_sha256 must be a lowercase SHA-256 hex digest"); + } + return { + path: stringArg(args, "path", { required: true, min: 1, max: 4096 }) as string, + old_text: oldText, + new_text: stringArg(args, "new_text", { required: true, max: 200_000 }) as string, + expected_replacements: numberArg(args, "expected_replacements", { + fallback: 1, + min: 1, + max: 1000, + integer: true, + }) as number, + ...(expected !== undefined ? { expected_sha256: expected } : {}), + }; + }, + async execute(args, context): Promise { + const target = await resolveWorkspacePath(context.workspace, args.path as string); + const metadata = await stat(target); + if (!metadata.isFile() || metadata.size > 1_000_000) { + throw new EvolveError("REPLACE_TARGET_INVALID", "replace_text requires a regular file no larger than 1 MB"); + } + const before = await readFile(target, "utf8"); + const beforeSha = sha256Bytes(Buffer.from(before, "utf8")); + const expectedSha = args.expected_sha256 as string | undefined; + if (expectedSha !== undefined && expectedSha !== beforeSha) { + throw new EvolveError("REPLACE_CAS_MISMATCH", "File changed since it was read"); + } + const oldText = args.old_text as string; + const occurrences = countOccurrences(before, oldText); + const expectedCount = args.expected_replacements as number; + if (occurrences !== expectedCount) { + throw new EvolveError( + "REPLACE_COUNT_MISMATCH", + `Expected ${expectedCount} occurrence(s), found ${occurrences}; no write was performed`, + ); + } + const after = before.split(oldText).join(args.new_text as string); + await atomicWriteFile(target, after); + return { + success: true, + summary: `Replaced ${occurrences} occurrence(s) in ${displayPath(context.workspace, target)}`, + data: { + path: displayPath(context.workspace, target), + replacements: occurrences, + before_sha256: beforeSha, + after_sha256: sha256Bytes(Buffer.from(after, "utf8")), + }, + }; + }, +}; diff --git a/evolve-agent/src/tools/run-process.ts b/evolve-agent/src/tools/run-process.ts new file mode 100644 index 0000000..7a0e6b5 --- /dev/null +++ b/evolve-agent/src/tools/run-process.ts @@ -0,0 +1,172 @@ +import { randomUUID } from "node:crypto"; +import { stat } from "node:fs/promises"; +import { EvolveError } from "../core/errors.js"; +import type { ExecutorKind, WorkspaceAccess } from "../execution/types.js"; +import type { ToolDefinition, ToolExecution } from "./types.js"; +import { numberArg, rejectUnknownKeys, stringArg, stringArrayArg } from "./validate.js"; +import { displayPath, resolveWorkspacePath } from "./workspace.js"; + +function enumValue(value: string, allowed: readonly T[], label: string): T { + if (!allowed.includes(value as T)) throw new EvolveError("TOOL_ARGS_INVALID", `${label} must be one of ${allowed.join(", ")}`); + return value as T; +} + +function validateSecrets(names: string[]): string[] { + for (const name of names) { + if (!/^[A-Z][A-Z0-9_]{0,63}$/.test(name)) { + throw new EvolveError("TOOL_ARGS_INVALID", `Invalid secret name: ${name}`); + } + } + return [...new Set(names)].sort(); +} + +export const runProcessTool: ToolDefinition = { + name: "run_process", + description: + "Run an allowlisted executable through the configured executor. Docker is deny-by-default: pinned allowlisted image, no network, read-only root, dropped capabilities, resource limits, and ephemeral file secrets.", + risk: "execute", + inputSchema: { + type: "object", + properties: { + command: { type: "string", description: "Allowlisted executable name; paths are rejected" }, + args: { type: "array", items: { type: "string" }, maxItems: 128 }, + cwd: { type: "string", description: "Workspace-relative working directory" }, + timeout_ms: { type: "integer", minimum: 100, maximum: 120000 }, + max_output_bytes: { type: "integer", minimum: 1024, maximum: 500000 }, + executor: { type: "string", enum: ["default", "docker", "local"] }, + image: { type: "string", description: "Exact allowlisted sha256-pinned Docker image" }, + network: { type: "string", description: "none or an operator-managed allowlisted Docker network" }, + workspace_access: { type: "string", enum: ["read-only", "read-write"] }, + secrets: { type: "array", items: { type: "string" }, maxItems: 16 }, + memory_mb: { type: "integer", minimum: 64, maximum: 8192 }, + cpus: { type: "number", minimum: 0.1, maximum: 8 }, + pids_limit: { type: "integer", minimum: 16, maximum: 1024 }, + tmpfs_mb: { type: "integer", minimum: 16, maximum: 1024 }, + }, + required: ["command"], + additionalProperties: false, + }, + validate(args) { + rejectUnknownKeys(args, [ + "command", + "args", + "cwd", + "timeout_ms", + "max_output_bytes", + "executor", + "image", + "network", + "workspace_access", + "secrets", + "memory_mb", + "cpus", + "pids_limit", + "tmpfs_mb", + ]); + const command = stringArg(args, "command", { required: true, min: 1, max: 128 }) as string; + if (command.includes("/") || command.includes("\\") || command === "." || command === "..") { + throw new EvolveError("TOOL_ARGS_INVALID", "command must be an executable name, not a path"); + } + const executor = enumValue( + stringArg(args, "executor", { fallback: "default", max: 16 }) as string, + ["default", "docker", "local"] as const, + "executor", + ); + const workspaceAccess = enumValue( + stringArg(args, "workspace_access", { fallback: "read-only", max: 16 }) as string, + ["read-only", "read-write"] as const, + "workspace_access", + ); + const image = stringArg(args, "image", { max: 512 }); + return { + command, + args: stringArrayArg(args, "args", { fallback: [], maxItems: 128, maxItemLength: 10_000 }) as string[], + cwd: stringArg(args, "cwd", { fallback: ".", max: 4096 }) as string, + timeout_ms: numberArg(args, "timeout_ms", { fallback: 60_000, min: 100, max: 120_000, integer: true }) as number, + max_output_bytes: numberArg(args, "max_output_bytes", { + fallback: 200_000, + min: 1024, + max: 500_000, + integer: true, + }) as number, + executor, + ...(image !== undefined ? { image } : {}), + network: stringArg(args, "network", { fallback: "none", min: 1, max: 128 }) as string, + workspace_access: workspaceAccess, + secrets: validateSecrets( + stringArrayArg(args, "secrets", { fallback: [], maxItems: 16, maxItemLength: 64 }) as string[], + ), + memory_mb: numberArg(args, "memory_mb", { fallback: 512, min: 64, max: 8_192, integer: true }) as number, + cpus: numberArg(args, "cpus", { fallback: 1, min: 0.1, max: 8 }) as number, + pids_limit: numberArg(args, "pids_limit", { fallback: 128, min: 16, max: 1_024, integer: true }) as number, + tmpfs_mb: numberArg(args, "tmpfs_mb", { fallback: 64, min: 16, max: 1_024, integer: true }) as number, + }; + }, + async execute(args, context): Promise { + const command = args.command as string; + if (!context.allowedCommands.has(command)) { + throw new EvolveError("COMMAND_NOT_ALLOWED", `${command} is not in EVOLVE_ALLOWED_COMMANDS`); + } + const workspace = await resolveWorkspacePath(context.workspace, "."); + const cwd = await resolveWorkspacePath(workspace, args.cwd as string); + if (!(await stat(cwd)).isDirectory()) throw new EvolveError("PROCESS_CWD_INVALID", "cwd is not a directory"); + + const requestedExecutor = args.executor as "default" | ExecutorKind; + const result = await context.executors.execute(requestedExecutor, { + runId: `${context.episodeId}-${randomUUID()}`, + command, + args: args.args as string[], + workspace, + cwd, + timeoutMs: args.timeout_ms as number, + maxOutputBytes: args.max_output_bytes as number, + workspaceAccess: args.workspace_access as WorkspaceAccess, + network: args.network as string, + secretNames: args.secrets as string[], + limits: { + memoryMb: args.memory_mb as number, + cpus: args.cpus as number, + pids: args.pids_limit as number, + tmpfsMb: args.tmpfs_mb as number, + }, + ...(args.image !== undefined ? { image: args.image as string } : {}), + }); + + return { + success: result.success, + summary: `${command} exited ${result.timedOut ? "after timeout" : `with code ${String(result.exitCode)}`} via ${result.executor} in ${displayPath(workspace, cwd)}`, + data: { + command, + args: args.args as string[], + cwd: displayPath(workspace, cwd), + executor: result.executor, + exit_code: result.exitCode, + signal: result.signal, + timed_out: result.timedOut, + truncated: result.truncated, + duration_ms: result.durationMs, + stdout: result.stdout, + stderr: result.stderr, + ...(result.image !== undefined ? { image: result.image } : {}), + network: result.network, + workspace_access: result.workspaceAccess, + secret_names: result.secretNames, + execution_receipt: { + run_id: result.receipt.runId, + sandbox_id: result.receipt.sandboxId, + command_hash: result.receipt.commandHash, + policy_hash: result.receipt.policyHash, + }, + isolation: { + boundary: result.isolation.boundary, + read_only_root: result.isolation.readOnlyRoot, + capabilities_dropped: result.isolation.capabilitiesDropped, + no_new_privileges: result.isolation.noNewPrivileges, + resource_limits: result.isolation.resourceLimits, + network_policy: result.isolation.networkPolicy, + secret_delivery: result.isolation.secretDelivery, + }, + }, + }; + }, +}; diff --git a/evolve-agent/src/tools/search-text.ts b/evolve-agent/src/tools/search-text.ts new file mode 100644 index 0000000..4e82a06 --- /dev/null +++ b/evolve-agent/src/tools/search-text.ts @@ -0,0 +1,76 @@ +import { readFile, readdir, stat } from "node:fs/promises"; +import path from "node:path"; +import type { ToolDefinition, ToolExecution } from "./types.js"; +import { numberArg, rejectUnknownKeys, stringArg } from "./validate.js"; +import { displayPath, resolveWorkspacePath } from "./workspace.js"; + +const ignored = new Set([".git", ".evolve", "node_modules", "dist", ".test-dist"]); + +export const searchTextTool: ToolDefinition = { + name: "search_text", + description: "Search UTF-8 workspace files for a literal string and return line-level matches.", + risk: "read", + inputSchema: { + type: "object", + properties: { + query: { type: "string" }, + path: { type: "string" }, + max_results: { type: "integer", minimum: 1, maximum: 200 }, + }, + required: ["query"], + additionalProperties: false, + }, + validate(args) { + rejectUnknownKeys(args, ["query", "path", "max_results"]); + return { + query: stringArg(args, "query", { required: true, min: 1, max: 1000 }) as string, + path: stringArg(args, "path", { fallback: ".", max: 4096 }) as string, + max_results: numberArg(args, "max_results", { fallback: 50, min: 1, max: 200, integer: true }) as number, + }; + }, + async execute(args, context): Promise { + const root = await resolveWorkspacePath(context.workspace, args.path as string); + const query = args.query as string; + const maxResults = args.max_results as number; + const matches: Array<{ path: string; line: number; text: string }> = []; + let filesScanned = 0; + let truncated = false; + + async function scan(target: string): Promise { + if (truncated) return; + const metadata = await stat(target); + if (metadata.isDirectory()) { + const entries = await readdir(target, { withFileTypes: true }); + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + if (ignored.has(entry.name)) continue; + await scan(path.join(target, entry.name)); + if (truncated) return; + } + return; + } + if (!metadata.isFile() || metadata.size > 1_000_000 || filesScanned >= 1000) return; + filesScanned += 1; + const buffer = await readFile(target); + if (buffer.includes(0)) return; + const lines = buffer.toString("utf8").split(/\r?\n/); + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + if (line.includes(query)) { + matches.push({ path: displayPath(context.workspace, target), line: index + 1, text: line.slice(0, 500) }); + if (matches.length >= maxResults) { + truncated = true; + return; + } + } + } + } + + await scan(root); + return { + success: true, + summary: `Found ${matches.length} literal matches for ${JSON.stringify(query)} in ${filesScanned} files${truncated ? " (truncated)" : ""}`, + data: { matches, files_scanned: filesScanned, truncated }, + }; + }, +}; diff --git a/evolve-agent/src/tools/types.ts b/evolve-agent/src/tools/types.ts new file mode 100644 index 0000000..3fa115a --- /dev/null +++ b/evolve-agent/src/tools/types.ts @@ -0,0 +1,33 @@ +import type { JsonObject, JsonValue } from "../core/types.js"; +import type { ExecutorRegistry } from "../execution/executor-registry.js"; + +export type ToolRisk = "read" | "write" | "execute" | "external"; + +export interface ToolContext { + episodeId: string; + workspace: string; + allowedCommands: Set; + executors: ExecutorRegistry; +} + +export interface ToolExecution { + success: boolean; + summary: string; + data: JsonValue; +} + +export interface ToolDefinition { + name: string; + description: string; + risk: ToolRisk; + inputSchema: JsonObject; + validate(args: Record): Record; + execute(args: Record, context: ToolContext): Promise; +} + +export interface ToolDescription { + name: string; + description: string; + risk: ToolRisk; + inputSchema: JsonObject; +} diff --git a/evolve-agent/src/tools/validate.ts b/evolve-agent/src/tools/validate.ts new file mode 100644 index 0000000..a5353a6 --- /dev/null +++ b/evolve-agent/src/tools/validate.ts @@ -0,0 +1,72 @@ +import { EvolveError } from "../core/errors.js"; + +export function assertObject(value: unknown, label = "arguments"): Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new EvolveError("TOOL_ARGS_INVALID", `${label} must be an object`); + } + return value as Record; +} + +export function stringArg( + args: Record, + key: string, + options: { required?: boolean; min?: number; max?: number; fallback?: string } = {}, +): string | undefined { + const value = args[key] ?? options.fallback; + if (value === undefined && !options.required) return undefined; + if (typeof value !== "string") throw new EvolveError("TOOL_ARGS_INVALID", `${key} must be a string`); + const min = options.min ?? 0; + const max = options.max ?? 1_000_000; + if (value.length < min || value.length > max) { + throw new EvolveError("TOOL_ARGS_INVALID", `${key} length must be between ${min} and ${max}`); + } + return value; +} + +export function numberArg( + args: Record, + key: string, + options: { required?: boolean; min?: number; max?: number; integer?: boolean; fallback?: number } = {}, +): number | undefined { + const value = args[key] ?? options.fallback; + if (value === undefined && !options.required) return undefined; + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new EvolveError("TOOL_ARGS_INVALID", `${key} must be a finite number`); + } + if (options.integer && !Number.isInteger(value)) throw new EvolveError("TOOL_ARGS_INVALID", `${key} must be an integer`); + if (options.min !== undefined && value < options.min) throw new EvolveError("TOOL_ARGS_INVALID", `${key} must be >= ${options.min}`); + if (options.max !== undefined && value > options.max) throw new EvolveError("TOOL_ARGS_INVALID", `${key} must be <= ${options.max}`); + return value; +} + +export function booleanArg( + args: Record, + key: string, + options: { required?: boolean; fallback?: boolean } = {}, +): boolean | undefined { + const value = args[key] ?? options.fallback; + if (value === undefined && !options.required) return undefined; + if (typeof value !== "boolean") throw new EvolveError("TOOL_ARGS_INVALID", `${key} must be a boolean`); + return value; +} + +export function stringArrayArg( + args: Record, + key: string, + options: { required?: boolean; maxItems?: number; maxItemLength?: number; fallback?: string[] } = {}, +): string[] | undefined { + const value = args[key] ?? options.fallback; + if (value === undefined && !options.required) return undefined; + if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) { + throw new EvolveError("TOOL_ARGS_INVALID", `${key} must be an array of strings`); + } + if (value.length > (options.maxItems ?? 128)) throw new EvolveError("TOOL_ARGS_INVALID", `${key} has too many items`); + const maxLength = options.maxItemLength ?? 10_000; + if (value.some((item) => item.length > maxLength)) throw new EvolveError("TOOL_ARGS_INVALID", `${key} contains an oversized item`); + return [...value]; +} + +export function rejectUnknownKeys(args: Record, allowed: string[]): void { + const unknown = Object.keys(args).filter((key) => !allowed.includes(key)); + if (unknown.length > 0) throw new EvolveError("TOOL_ARGS_INVALID", `Unknown arguments: ${unknown.join(", ")}`); +} diff --git a/evolve-agent/src/tools/workspace.ts b/evolve-agent/src/tools/workspace.ts new file mode 100644 index 0000000..6d6c2b3 --- /dev/null +++ b/evolve-agent/src/tools/workspace.ts @@ -0,0 +1,62 @@ +import { lstat, mkdir, realpath } from "node:fs/promises"; +import path from "node:path"; +import { EvolveError } from "../core/errors.js"; + +function contained(root: string, candidate: string): boolean { + return candidate === root || candidate.startsWith(`${root}${path.sep}`); +} + +async function nearestExistingParent(target: string): Promise { + let current = target; + while (true) { + try { + await lstat(current); + return current; + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + const parent = path.dirname(current); + if (parent === current) throw error; + current = parent; + } + } +} + +async function rejectSymlinks(root: string, target: string): Promise { + const relative = path.relative(root, target); + if (relative === "") return; + let current = root; + for (const part of relative.split(path.sep)) { + current = path.join(current, part); + try { + const stats = await lstat(current); + if (stats.isSymbolicLink()) throw new EvolveError("WORKSPACE_SYMLINK", `Symlink traversal is not allowed: ${current}`); + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + } +} + +export async function resolveWorkspacePath(workspace: string, requested = ".", options: { createParent?: boolean } = {}): Promise { + await mkdir(workspace, { recursive: true }); + const root = await realpath(workspace); + const lexical = path.resolve(root, requested); + if (!contained(root, lexical)) throw new EvolveError("WORKSPACE_ESCAPE", `Path escapes workspace: ${requested}`); + await rejectSymlinks(root, lexical); + + const existing = await nearestExistingParent(lexical); + const canonicalParent = await realpath(existing); + if (!contained(root, canonicalParent)) throw new EvolveError("WORKSPACE_ESCAPE", `Canonical path escapes workspace: ${requested}`); + + if (options.createParent) { + const parent = path.dirname(lexical); + await mkdir(parent, { recursive: true }); + await rejectSymlinks(root, parent); + } + return lexical; +} + +export function displayPath(workspace: string, absolute: string): string { + const relative = path.relative(workspace, absolute); + return relative === "" ? "." : relative.split(path.sep).join("/"); +} diff --git a/evolve-agent/src/tools/write-file.ts b/evolve-agent/src/tools/write-file.ts new file mode 100644 index 0000000..fed291e --- /dev/null +++ b/evolve-agent/src/tools/write-file.ts @@ -0,0 +1,76 @@ +import { readFile, stat } from "node:fs/promises"; +import { atomicWriteFile, pathExists } from "../core/fs.js"; +import { sha256Bytes } from "../core/hash.js"; +import { EvolveError } from "../core/errors.js"; +import type { ToolDefinition, ToolExecution } from "./types.js"; +import { booleanArg, rejectUnknownKeys, stringArg } from "./validate.js"; +import { displayPath, resolveWorkspacePath } from "./workspace.js"; + +export const writeFileTool: ToolDefinition = { + name: "write_file", + description: + "Create or replace one UTF-8 workspace file. Supports create-only and SHA-256 compare-and-swap guards.", + risk: "write", + inputSchema: { + type: "object", + properties: { + path: { type: "string", description: "Workspace-relative file path" }, + content: { type: "string", description: "Complete UTF-8 file contents" }, + create_only: { type: "boolean", description: "Fail if the file already exists" }, + expected_sha256: { + type: "string", + description: "Optional SHA-256 of the current file. The write fails if it changed.", + }, + }, + required: ["path", "content"], + additionalProperties: false, + }, + validate(args) { + rejectUnknownKeys(args, ["path", "content", "create_only", "expected_sha256"]); + const expected = stringArg(args, "expected_sha256", { min: 64, max: 64 }); + if (expected !== undefined && !/^[a-f0-9]{64}$/.test(expected)) { + throw new EvolveError("TOOL_ARGS_INVALID", "expected_sha256 must be a lowercase SHA-256 hex digest"); + } + return { + path: stringArg(args, "path", { required: true, min: 1, max: 4096 }) as string, + content: stringArg(args, "content", { required: true, max: 500_000 }) as string, + create_only: booleanArg(args, "create_only", { fallback: false }) as boolean, + ...(expected !== undefined ? { expected_sha256: expected } : {}), + }; + }, + async execute(args, context): Promise { + const target = await resolveWorkspacePath(context.workspace, args.path as string, { createParent: true }); + const exists = await pathExists(target); + if (exists) { + const metadata = await stat(target); + if (!metadata.isFile()) throw new EvolveError("WRITE_TARGET_INVALID", "write_file target is not a regular file"); + } + if ((args.create_only as boolean) && exists) { + throw new EvolveError("WRITE_CREATE_ONLY", "Target already exists and create_only is true"); + } + + let beforeSha: string | null = null; + if (exists) beforeSha = sha256Bytes(await readFile(target)); + const expected = args.expected_sha256 as string | undefined; + if (expected !== undefined && beforeSha !== expected) { + throw new EvolveError( + "WRITE_CAS_MISMATCH", + `Current file hash ${beforeSha ?? ""} does not match expected_sha256`, + ); + } + + const content = args.content as string; + await atomicWriteFile(target, content); + const afterSha = sha256Bytes(Buffer.from(content, "utf8")); + return { + success: true, + summary: `${exists ? "Replaced" : "Created"} ${displayPath(context.workspace, target)} (${Buffer.byteLength(content)} bytes)`, + data: { + path: displayPath(context.workspace, target), + before_sha256: beforeSha, + after_sha256: afterSha, + bytes: Buffer.byteLength(content), + }, + }; + }, +}; diff --git a/evolve-agent/src/verification/final-verifier.ts b/evolve-agent/src/verification/final-verifier.ts new file mode 100644 index 0000000..5f612e9 --- /dev/null +++ b/evolve-agent/src/verification/final-verifier.ts @@ -0,0 +1,79 @@ +import { EvolveError } from "../core/errors.js"; +import type { ArtifactStore } from "../ledger/artifact-store.js"; +import type { AgentProvider, FinalDecision, VerificationVerdict } from "../providers/provider.js"; +import type { TaskSpec, Usage } from "../core/types.js"; + +const EVIDENCE_PATTERN = /ev_[a-f0-9]{24}/g; + +export interface FinalVerificationResult { + passed: boolean; + verdict: VerificationVerdict; + usage: Usage; + validEvidenceIds: string[]; +} + +export class FinalVerifier { + public constructor( + private readonly artifacts: ArtifactStore, + private readonly provider: AgentProvider, + ) {} + + public async verify(episodeId: string, task: TaskSpec, decision: FinalDecision): Promise { + const declared = [...new Set(decision.evidenceIds)]; + const declarationCheck = await this.artifacts.validateEvidenceIds(episodeId, declared); + if (!declarationCheck.valid) { + return { + passed: false, + verdict: { + passed: false, + score: 0, + feedback: `The answer declared invalid or cross-episode evidence IDs: ${declarationCheck.invalid.join(", ")}`, + }, + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + validEvidenceIds: declared.filter((id) => !declarationCheck.invalid.includes(id)), + }; + } + + const cited = [...new Set(decision.answer.match(EVIDENCE_PATTERN) ?? [])]; + const undeclared = cited.filter((id) => !declared.includes(id)); + if (undeclared.length > 0) { + return { + passed: false, + verdict: { + passed: false, + score: 0, + feedback: `The answer cited evidence that was not declared: ${undeclared.join(", ")}`, + }, + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + validEvidenceIds: declared, + }; + } + + if (declared.length > 0 && cited.length === 0) { + return { + passed: false, + verdict: { + passed: false, + score: 0.25, + feedback: "Evidence was declared but the answer contains no [evidence:ev_...] citations.", + }, + usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + validEvidenceIds: declared, + }; + } + + const evidence = []; + for (const id of declared) { + const record = await this.artifacts.getEvidence(id); + if (!record) throw new EvolveError("EVIDENCE_MISSING", `Validated evidence disappeared: ${id}`); + evidence.push(record); + } + const independent = await this.provider.verify({ task, answer: decision.answer, evidence }); + return { + passed: independent.verdict.passed && independent.verdict.score >= 0.8, + verdict: independent.verdict, + usage: independent.usage, + validEvidenceIds: declared, + }; + } +} diff --git a/evolve-agent/tests/capability.test.ts b/evolve-agent/tests/capability.test.ts new file mode 100644 index 0000000..c7123e9 --- /dev/null +++ b/evolve-agent/tests/capability.test.ts @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { CapabilityAuthority } from "../src/policy/capability.js"; +import { ToolRegistry } from "../src/tools/registry.js"; + +async function temporary(prefix: string): Promise { + return mkdtemp(path.join(os.tmpdir(), prefix)); +} + +test("capability tokens bind exact arguments and expire", async () => { + const root = await temporary("evolve-capability-"); + try { + const authority = new CapabilityAuthority(path.join(root, "capability.key")); + const args = { path: "note.txt", content: "alpha", create_only: true }; + const token = await authority.issue({ episodeId: "ep_aaaaaaaaaaaaaaaaaaaaaaaa", toolName: "write_file", args, ttlMs: 1_000 }); + await authority.verify(token, { episodeId: "ep_aaaaaaaaaaaaaaaaaaaaaaaa", toolName: "write_file", args }); + await assert.rejects( + authority.verify(token, { + episodeId: "ep_aaaaaaaaaaaaaaaaaaaaaaaa", + toolName: "write_file", + args: { ...args, content: "changed" }, + }), + /arguments changed/i, + ); + await assert.rejects( + authority.verify(token, { + episodeId: "ep_aaaaaaaaaaaaaaaaaaaaaaaa", + toolName: "write_file", + args, + now: Date.now() + 10_000, + }), + /expired/i, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("registry revalidates the exact approved action before execution", async () => { + const root = await temporary("evolve-registry-"); + try { + const workspace = path.join(root, "workspace"); + const authority = new CapabilityAuthority(path.join(root, "capability.key")); + const registry = new ToolRegistry(authority, { workspace, allowedCommands: new Set(["node"]) }); + const validated = registry.get("write_file").validate({ path: "note.txt", content: "alpha", create_only: true }); + const token = await authority.issue({ + episodeId: "ep_bbbbbbbbbbbbbbbbbbbbbbbb", + toolName: "write_file", + args: validated, + }); + await assert.rejects( + registry.execute({ + episodeId: "ep_bbbbbbbbbbbbbbbbbbbbbbbb", + toolName: "write_file", + rawArgs: { path: "note.txt", content: "beta", create_only: true }, + capabilityToken: token, + }), + /arguments changed/i, + ); + await registry.execute({ + episodeId: "ep_bbbbbbbbbbbbbbbbbbbbbbbb", + toolName: "write_file", + rawArgs: validated, + capabilityToken: token, + }); + assert.equal(await readFile(path.join(workspace, "note.txt"), "utf8"), "alpha"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/config-hardening.test.ts b/evolve-agent/tests/config-hardening.test.ts new file mode 100644 index 0000000..8360018 --- /dev/null +++ b/evolve-agent/tests/config-hardening.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { loadConfig } from "../src/config.js"; +import { createRuntime } from "../src/factory.js"; + +const IMAGE = `node:22@sha256:${"c".repeat(64)}`; + +test("hardened execution is Docker-first and local execution is explicit", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-config-test-")); + try { + const hardened = loadConfig({ home: path.join(root, "home"), workspace: path.join(root, "workspace") }); + assert.equal(hardened.defaultExecutor, "docker"); + assert.equal(hardened.allowLocalExecutor, false); + assert.throws( + () => loadConfig({ defaultExecutor: "local", allowLocalExecutor: false }), + /requires EVOLVE_ALLOW_LOCAL_EXECUTOR=true/, + ); + + const local = loadConfig({ defaultExecutor: "local", allowLocalExecutor: true }); + assert.equal(local.defaultExecutor, "local"); + assert.equal(local.allowLocalExecutor, true); + assert.throws( + () => loadConfig({ workspace: path.join(root, "unsafe"), home: path.join(root, "unsafe", ".evolve") }), + /EVOLVE_HOME must be outside EVOLVE_WORKSPACE/, + ); + assert.throws( + () => loadConfig({ evaluation: { canaryMinSamples: 10, monitorWindow: 5 } }), + /EVOLVE_EVAL_MONITOR_WINDOW/, + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("runtime construction rejects unpinned images, unsafe networks, and container root", () => { + assert.throws( + () => createRuntime(loadConfig({ docker: { defaultImage: "node:22", allowedImages: ["node:22"] } })), + /not sha256-pinned/, + ); + assert.throws( + () => createRuntime(loadConfig({ docker: { defaultImage: IMAGE, allowedImages: [IMAGE], allowedNetworks: ["host"] } })), + /Unsafe network cannot be allowlisted/, + ); + assert.throws( + () => createRuntime(loadConfig({ docker: { defaultImage: IMAGE, allowedImages: [IMAGE], user: "0:0" } })), + /non-root numeric uid:gid/, + ); +}); diff --git a/evolve-agent/tests/docker-executor.test.ts b/evolve-agent/tests/docker-executor.test.ts new file mode 100644 index 0000000..be7a7c3 --- /dev/null +++ b/evolve-agent/tests/docker-executor.test.ts @@ -0,0 +1,204 @@ +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { DockerExecutor, buildDockerRunArgs } from "../src/execution/docker-executor.js"; +import { ImagePolicy } from "../src/execution/image-policy.js"; +import { DockerNetworkPolicy } from "../src/execution/network-policy.js"; +import type { ProcessRunner, ProcessRunRequest, ProcessRunResult } from "../src/execution/process-runner.js"; +import type { ExecutionRequest, ResourceLimits } from "../src/execution/types.js"; +import { FileSecretBroker } from "../src/secrets/secret-broker.js"; + +const IMAGE = `node:22-bookworm-slim@sha256:${"a".repeat(64)}`; +const LIMITS: ResourceLimits = { memoryMb: 512, cpus: 1, pids: 128, tmpfsMb: 64 }; + +class RecordingRunner implements ProcessRunner { + public readonly calls: ProcessRunRequest[] = []; + + public async run(request: ProcessRunRequest): Promise { + this.calls.push(request); + if (request.args[0] === "run") { + return { + code: 0, + signal: null, + timedOut: false, + truncated: false, + stdout: "token=super-secret-token\n", + stderr: "", + durationMs: 12, + }; + } + return { + code: 0, + signal: null, + timedOut: false, + truncated: false, + stdout: "", + stderr: "", + durationMs: 2, + }; + } +} + +function request(workspace: string): ExecutionRequest { + return { + runId: "ep_aaaaaaaaaaaaaaaaaaaaaaaa-run-1", + command: "node", + args: ["--version"], + workspace, + cwd: workspace, + timeoutMs: 5_000, + maxOutputBytes: 32_000, + workspaceAccess: "read-only", + network: "none", + secretNames: ["TEST_TOKEN"], + limits: LIMITS, + image: IMAGE, + }; +} + +test("Docker image and network policy fail closed", () => { + assert.throws(() => new ImagePolicy(["node:22"]), /sha256-pinned/); + assert.throws(() => new DockerNetworkPolicy(["host"]), /Unsafe network/); + + const images = new ImagePolicy([IMAGE], IMAGE); + assert.equal(images.resolve(), IMAGE); + assert.throws(() => images.resolve(`node:20@sha256:${"b".repeat(64)}`), /not in EVOLVE_DOCKER_ALLOWED_IMAGES/); + + const networks = new DockerNetworkPolicy(["evolve-egress"]); + assert.deepEqual(networks.resolve("none"), { name: "none", enforcement: "deny-all" }); + assert.deepEqual(networks.resolve("evolve-egress"), { + name: "evolve-egress", + enforcement: "operator-managed:evolve-egress", + }); + assert.throws(() => networks.resolve("bridge"), /bypasses the hardened network boundary/); +}); + +test("Docker run arguments encode the hardened sandbox contract", () => { + const workspace = path.join(os.tmpdir(), "evolve-workspace"); + const args = buildDockerRunArgs({ + request: request(workspace), + image: IMAGE, + network: "none", + networkEnforcement: "deny-all", + user: "65532:65532", + containerName: "evolve-test", + secretMounts: [ + { name: "TEST_TOKEN", hostPath: "/tmp/secret-token", containerPath: "/run/secrets/TEST_TOKEN" }, + ], + maximums: { memoryMb: 2_048, cpus: 2, pids: 256, tmpfsMb: 256 }, + }); + const rendered = JSON.stringify(args); + for (const required of [ + "--pull", + "never", + "--read-only", + "--cap-drop", + "ALL", + "no-new-privileges:true", + "--network", + "none", + "--memory", + "512m", + "--pids-limit", + "128", + "--ipc", + "none", + "TEST_TOKEN_FILE=/run/secrets/TEST_TOKEN", + ]) { + assert.ok(args.includes(required), `missing Docker hardening argument: ${required}`); + } + assert.ok(rendered.includes("readonly")); + assert.ok(!rendered.includes("super-secret-token")); + assert.equal(args.at(-2), "node"); + assert.equal(args.at(-1), "--version"); +}); + +test("Docker executor redacts injected secrets and emits a policy receipt", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-docker-test-")); + const workspace = path.join(root, "workspace"); + await mkdir(workspace, { recursive: true }); + const runner = new RecordingRunner(); + const broker = new FileSecretBroker(path.join(root, "secrets"), ["TEST_TOKEN"], 60_000, (name) => + name === "TEST_TOKEN" ? "super-secret-token" : undefined, + ); + const executor = new DockerExecutor(runner, broker, { + binary: "docker", + user: "65532:65532", + imagePolicy: new ImagePolicy([IMAGE], IMAGE), + networkPolicy: new DockerNetworkPolicy([]), + maximums: { memoryMb: 2_048, cpus: 2, pids: 256, tmpfsMb: 256 }, + requireRootless: false, + }); + + try { + const result = await executor.execute(request(workspace)); + assert.equal(result.success, true); + assert.equal(result.stdout, "token=[REDACTED_SECRET:TEST_TOKEN]\n"); + assert.equal(result.isolation.boundary, "docker-container"); + assert.equal(result.isolation.networkPolicy, "deny-all"); + assert.equal(result.receipt.commandHash.length, 64); + assert.equal(result.receipt.policyHash.length, 64); + assert.equal(runner.calls.length, 2, "docker run plus best-effort cleanup"); + + const dockerRun = runner.calls[0]; + assert.ok(dockerRun); + const serialized = JSON.stringify({ args: dockerRun.args, env: dockerRun.env }); + assert.ok(!serialized.includes("super-secret-token"), "secret values must never enter Docker CLI args or env"); + assert.ok(serialized.includes("TEST_TOKEN_FILE=/run/secrets/TEST_TOKEN")); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + + +test("Docker readiness requires seccomp, cgroups, and the pinned local image", async () => { + class ProbeRunner implements ProcessRunner { + public constructor(private readonly securityOptions: string[]) {} + public async run(input: ProcessRunRequest): Promise { + const joined = input.args.join(" "); + let stdout = ""; + if (joined.startsWith("version ")) stdout = "29.0.1\n"; + else if (joined.includes("SecurityOptions")) stdout = `${JSON.stringify(this.securityOptions)}\n`; + else if (joined.includes("CgroupVersion")) stdout = "2\n"; + else if (joined.startsWith("image inspect")) stdout = `sha256:${"d".repeat(64)}\n`; + return { + code: 0, + signal: null, + timedOut: false, + truncated: false, + stdout, + stderr: "", + durationMs: 1, + }; + } + } + + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-probe-test-")); + try { + const broker = new FileSecretBroker(path.join(root, "secrets"), [], 60_000); + const options = { + binary: "docker", + user: "65532:65532", + imagePolicy: new ImagePolicy([IMAGE], IMAGE), + networkPolicy: new DockerNetworkPolicy([]), + maximums: { memoryMb: 2_048, cpus: 2, pids: 256, tmpfsMb: 256 }, + requireRootless: true, + }; + const ready = await new DockerExecutor( + new ProbeRunner(["name=seccomp,profile=builtin", "name=rootless"]), + broker, + options, + ).probe(); + assert.equal(ready.ready, true); + assert.equal(ready.details.seccomp, true); + assert.equal(ready.details.resource_limits_ready, true); + + const missingSeccomp = await new DockerExecutor(new ProbeRunner(["name=rootless"]), broker, options).probe(); + assert.equal(missingSeccomp.ready, false); + assert.ok(missingSeccomp.warnings.some((warning) => warning.includes("seccomp"))); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/evaluation-engine.test.ts b/evolve-agent/tests/evaluation-engine.test.ts new file mode 100644 index 0000000..b0a2e82 --- /dev/null +++ b/evolve-agent/tests/evaluation-engine.test.ts @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { loadConfig } from "../src/config.js"; +import { createRuntime } from "../src/factory.js"; +import { SkillAwareProvider, candidateInput, fixturePayload } from "./evaluation-helpers.js"; + +async function setup() { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-eval-engine-")); + const workspace = path.join(root, "workspace"); + const home = path.join(root, "home"); + const provider = new SkillAwareProvider(""); + const config = loadConfig({ + home, + workspace, + evaluation: { + canaryMinSamples: 3, + monitorMinSamples: 2, + monitorWindow: 10, + policy: { + minFixtures: 3, + repeats: 1, + bootstrapSamples: 200, + }, + }, + }); + const bundle = createRuntime(config, { provider }); + const skill = await bundle.skills.upsertCandidate(candidateInput()); + provider.candidateSkillId = skill.id; + return { root, workspace, home, provider, bundle, skill }; +} + +test("offline evaluation excludes training Episodes and creates a signed report", async () => { + const environment = await setup(); + try { + for (let index = 1; index <= 4; index += 1) await environment.bundle.fixtures.put(fixturePayload(index)); + // A fixture sourced from the candidate's training provenance must never enter the report. + const leaked = fixturePayload(50); + leaked.sourceEpisodeId = environment.skill.supportingEpisodes[0] as string; + leaked.trace[0]!.evidence.episodeId = leaked.sourceEpisodeId; + const leakedFixture = await environment.bundle.fixtures.put(leaked); + + const report = await environment.bundle.evaluations.evaluateSkill(environment.skill.id, { + fixtures: [...(await environment.bundle.fixtures.list()).map((fixture) => fixture.id)], + }); + assert.equal(report.payload.decision.passed, true); + assert.equal(await environment.bundle.evaluations.verifyReport(report.id), true); + assert.equal(report.payload.fixtureIds.includes(leakedFixture.id), false); + assert.ok(report.payload.comparison.verifierScoreDelta > 0.1); + assert.ok(report.payload.comparison.totalTokenRatio < 1); + const skill = await environment.bundle.skills.get(environment.skill.id); + assert.equal(skill.status, "evaluated"); + assert.deepEqual(skill.evaluationReportIds, [report.id]); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); + +test("shadow canary stays off the production path and explicit promotion requires both signed reports", async () => { + const environment = await setup(); + try { + for (let index = 1; index <= 6; index += 1) await environment.bundle.fixtures.put(fixturePayload(index)); + const offline = await environment.bundle.evaluations.evaluateSkill(environment.skill.id, { + fixtures: (await environment.bundle.fixtures.list()).slice(0, 3).map((fixture) => fixture.id), + }); + assert.equal(offline.payload.decision.passed, true); + + const holdouts = (await environment.bundle.fixtures.list()).slice(3, 6); + for (const fixture of holdouts) { + const observation = await environment.bundle.evaluations.shadowEpisode( + environment.skill.id, + fixture.sourceEpisodeId, + "production-baseline", + ); + assert.equal(observation.mode, "production-baseline"); + assert.equal(observation.baseline.activeSkillIds.includes(environment.skill.id), false); + assert.equal(observation.candidate.activeSkillIds.includes(environment.skill.id), true); + } + const skillAfterCanary = await environment.bundle.skills.get(environment.skill.id); + assert.equal(skillAfterCanary.status, "canary"); + assert.equal(skillAfterCanary.canaryReportIds.length, 1); + const promoted = await environment.bundle.evaluations.promoteSkill(environment.skill.id); + assert.equal(promoted.status, "promoted"); + assert.ok(promoted.promotion?.offlineReportId); + assert.ok(promoted.promotion?.canaryReportId); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); + +test("production regression monitor automatically rolls back a promoted Skill", async () => { + const environment = await setup(); + try { + for (let index = 1; index <= 6; index += 1) await environment.bundle.fixtures.put(fixturePayload(index)); + await environment.bundle.evaluations.evaluateSkill(environment.skill.id, { + fixtures: (await environment.bundle.fixtures.list()).slice(0, 3).map((fixture) => fixture.id), + }); + for (const fixture of (await environment.bundle.fixtures.list()).slice(3, 6)) { + await environment.bundle.evaluations.shadowEpisode(environment.skill.id, fixture.sourceEpisodeId, "production-baseline"); + } + await environment.bundle.evaluations.promoteSkill(environment.skill.id); + + for (let index = 0; index < 2; index += 1) { + await environment.bundle.evaluations.recordProduction({ + version: 1, + episodeId: `ep_${(900 + index).toString(16).padStart(24, "0")}`, + task: fixturePayload(index + 20).task, + status: "failed", + observations: [], + evidenceIds: [], + toolSequence: [], + activeSkillIds: [environment.skill.id], + usage: { inputTokens: 100, outputTokens: 100, totalTokens: 200 }, + turns: 3, + toolCalls: 1, + elapsedMs: 100, + finalScore: 0, + stopReason: "regression", + updatedAt: new Date().toISOString(), + }); + } + const rolledBack = await environment.bundle.skills.get(environment.skill.id); + assert.equal(rolledBack.status, "rolled_back"); + assert.equal(rolledBack.rollback?.automatic, true); + assert.ok(rolledBack.rollback?.reportId); + assert.equal(await environment.bundle.evaluations.verifyReport(rolledBack.rollback?.reportId as string), true); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); + + +test("promotion refuses a tampered signed evaluation report", async () => { + const environment = await setup(); + try { + for (let index = 1; index <= 6; index += 1) await environment.bundle.fixtures.put(fixturePayload(index)); + await environment.bundle.evaluations.evaluateSkill(environment.skill.id, { + fixtures: (await environment.bundle.fixtures.list()).slice(0, 3).map((fixture) => fixture.id), + }); + for (const fixture of (await environment.bundle.fixtures.list()).slice(3, 6)) { + await environment.bundle.evaluations.shadowEpisode(environment.skill.id, fixture.sourceEpisodeId, "production-baseline"); + } + const skill = await environment.bundle.skills.get(environment.skill.id); + const canaryId = skill.canaryReportIds.at(-1) as string; + const reportPath = path.join(environment.home, "evaluations", "reports", `${canaryId}.json`); + const { readFile, writeFile } = await import("node:fs/promises"); + const report = JSON.parse(await readFile(reportPath, "utf8")) as { payload: { decision: { passed: boolean } } }; + report.payload.decision.passed = false; + await writeFile(reportPath, JSON.stringify(report), "utf8"); + await assert.rejects(environment.bundle.evaluations.promoteSkill(environment.skill.id), /signature/i); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/evaluation-helpers.ts b/evolve-agent/tests/evaluation-helpers.ts new file mode 100644 index 0000000..9f36c9c --- /dev/null +++ b/evolve-agent/tests/evaluation-helpers.ts @@ -0,0 +1,143 @@ +import { sha256Json } from "../src/core/hash.js"; +import type { SkillRecord, Usage } from "../src/core/types.js"; +import type { + AgentPrompt, + AgentProvider, + DecisionResult, + VerificationInput, + VerificationResult, +} from "../src/providers/provider.js"; +import type { ReplayFixturePayload } from "../src/evaluation/types.js"; + +export class SkillAwareProvider implements AgentProvider { + public readonly prompts: AgentPrompt[] = []; + public readonly verifications: VerificationInput[] = []; + + public constructor(public candidateSkillId: string) {} + + public async decide(prompt: AgentPrompt): Promise { + this.prompts.push(prompt); + const candidate = prompt.skills.some((skill) => skill.id === this.candidateSkillId); + const usage: Usage = candidate + ? { inputTokens: 7, outputTokens: 3, totalTokens: 10 } + : { inputTokens: 12, outputTokens: 8, totalTokens: 20 }; + if (prompt.observations.length === 0) { + const path = prompt.task.goal.match(/file-\d+\.txt/)?.[0] ?? "file-0.txt"; + return { + decision: { kind: "tool", toolName: "read_file", args: { path }, rationale: "Replay the recorded read" }, + usage, + }; + } + const evidence = prompt.observations.findLast((observation) => observation.evidenceId)?.evidenceId; + if (!evidence) throw new Error("Expected replay evidence"); + return { + decision: { + kind: "final", + answer: `${candidate ? "Candidate" : "Baseline"} answer [evidence:${evidence}]`, + evidenceIds: [evidence], + memoryProposals: [], + }, + usage, + }; + } + + public async verify(input: VerificationInput): Promise { + this.verifications.push(input); + const candidate = input.answer.startsWith("Candidate"); + return { + verdict: { + passed: true, + score: candidate ? 0.96 : 0.82, + feedback: candidate ? "candidate accepted" : "baseline accepted", + }, + usage: candidate + ? { inputTokens: 3, outputTokens: 2, totalTokens: 5 } + : { inputTokens: 5, outputTokens: 3, totalTokens: 8 }, + }; + } +} + +export function candidateInput(index = 0): { + fingerprint: string; + name: string; + description: string; + triggers: string[]; + steps: Array<{ toolName: string; purpose: string }>; + allowedTools: string[]; + supportingEpisodes: string[]; + provenanceEvidenceIds: string[]; +} { + const suffix = index.toString(16).padStart(2, "0"); + return { + fingerprint: `${suffix}${"a".repeat(62)}`, + name: "Read one file accurately", + description: "Use read_file and produce an evidence-backed answer with less deliberation.", + triggers: ["inspect", "file"], + steps: [{ toolName: "read_file", purpose: "Read the requested file" }], + allowedTools: ["read_file"], + supportingEpisodes: ["ep_aaaaaaaaaaaaaaaaaaaaaaaa", "ep_bbbbbbbbbbbbbbbbbbbbbbbb"], + provenanceEvidenceIds: ["ev_aaaaaaaaaaaaaaaaaaaaaaaa", "ev_bbbbbbbbbbbbbbbbbbbbbbbb"], + }; +} + +export function fixturePayload(index: number, activeSkillIds: string[] = []): ReplayFixturePayload { + const hex = index.toString(16).padStart(24, "0").slice(-24); + const episodeId = `ep_${hex}`; + const evidenceId = `ev_${(index + 100).toString(16).padStart(24, "0").slice(-24)}`; + const file = `file-${index}.txt`; + const args = { path: file }; + return { + version: 1, + sourceEpisodeId: episodeId, + split: index % 2 === 0 ? "validation" : "holdout", + createdAt: new Date(1_700_000_000_000 + index).toISOString(), + task: { + id: `task_${(index + 200).toString(16).padStart(24, "0").slice(-24)}`, + goal: `Inspect ${file} and report the result.`, + constraints: ["Use replay evidence"], + successCriteria: ["Answer cites the read evidence"], + requestedTools: ["read_file"], + budget: { + maxTurns: 4, + maxToolCalls: 2, + maxInputTokens: 20_000, + maxOutputTokens: 5_000, + maxWallTimeMs: 60_000, + }, + createdAt: new Date(1_700_000_000_000 + index).toISOString(), + }, + trace: [ + { + index: 0, + toolName: "read_file", + argsHash: sha256Json(args), + evidence: { + id: evidenceId, + episodeId, + toolName: "read_file", + argsHash: sha256Json(args), + artifactHash: sha256Json({ file, content: `value-${index}` }), + summary: `${file} contains value-${index}`, + success: true, + createdAt: new Date(1_700_000_000_000 + index).toISOString(), + }, + observation: `${file} contains value-${index} [evidence:${evidenceId}]`, + success: true, + }, + ], + baseline: { + status: "committed", + answerHash: sha256Json(`Baseline answer ${index}`), + verifierScore: 0.82, + usage: { inputTokens: 29, outputTokens: 19, totalTokens: 48 }, + turns: 2, + toolCalls: 1, + elapsedMs: 100, + activeSkillIds, + }, + }; +} + +export function asSkill(record: SkillRecord): SkillRecord { + return record; +} diff --git a/evolve-agent/tests/evaluation-metrics.test.ts b/evolve-agent/tests/evaluation-metrics.test.ts new file mode 100644 index 0000000..222a8d5 --- /dev/null +++ b/evolve-agent/tests/evaluation-metrics.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { compareRuns, defaultEvaluationPolicy } from "../src/evaluation/metrics.js"; +import type { ReplayRunResult } from "../src/evaluation/types.js"; + +function run(fixture: number, arm: "baseline" | "candidate", input: Partial = {}): ReplayRunResult { + return { + fixtureId: `fixture_${fixture.toString(16).padStart(24, "0")}`, + fixtureHash: fixture.toString(16).padStart(64, "0"), + sourceEpisodeId: `ep_${fixture.toString(16).padStart(24, "0")}`, + arm, + repeat: 0, + success: true, + verifierScore: arm === "candidate" ? 0.95 : 0.82, + usage: arm === "candidate" + ? { inputTokens: 5, outputTokens: 5, totalTokens: 10 } + : { inputTokens: 10, outputTokens: 10, totalTokens: 20 }, + turns: 2, + toolCalls: 1, + durationMs: 10, + traceMatched: true, + safetyViolations: [], + activeSkillIds: [], + ...input, + }; +} + +test("evaluation gate accepts paired quality and efficiency improvement", () => { + const baseline = [run(1, "baseline"), run(2, "baseline"), run(3, "baseline")]; + const candidate = [run(1, "candidate"), run(2, "candidate"), run(3, "candidate")]; + const result = compareRuns(baseline, candidate, defaultEvaluationPolicy({ minFixtures: 3, bootstrapSamples: 200 })); + assert.equal(result.decision.passed, true); + assert.ok(result.comparison.verifierScoreDelta > 0.1); + assert.ok(result.comparison.totalTokenRatio < 1); +}); + +test("evaluation gate rejects one new failure even when average cost improves", () => { + const baseline = [run(1, "baseline"), run(2, "baseline"), run(3, "baseline")]; + const candidate = [run(1, "candidate"), run(2, "candidate", { success: false, verifierScore: 0 }), run(3, "candidate")]; + const result = compareRuns(baseline, candidate, defaultEvaluationPolicy({ minFixtures: 3, bootstrapSamples: 200 })); + assert.equal(result.decision.passed, false); + assert.equal(result.decision.gates.no_new_failures, false); +}); diff --git a/evolve-agent/tests/evolution-runtime.test.ts b/evolve-agent/tests/evolution-runtime.test.ts new file mode 100644 index 0000000..e78d3ed --- /dev/null +++ b/evolve-agent/tests/evolution-runtime.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { loadConfig } from "../src/config.js"; +import { createRuntime } from "../src/factory.js"; +import { StaticApprover } from "../src/policy/approver.js"; +import { SkillAwareProvider, candidateInput, fixturePayload } from "./evaluation-helpers.js"; + +test("runtime records active Skills and captures committed Episodes as replay fixtures when explicitly enabled", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-runtime-eval-")); + try { + const workspace = path.join(root, "workspace"); + const home = path.join(root, "home"); + await writeFile(path.join(root, "placeholder"), "x", "utf8"); + const provider = new SkillAwareProvider(""); + const config = loadConfig({ + home, + workspace, + evaluation: { + captureCommitted: true, + monitorPromoted: false, + canaryMinSamples: 3, + monitorWindow: 10, + policy: { minFixtures: 3, bootstrapSamples: 200 }, + }, + }); + const bundle = createRuntime(config, { provider, approver: new StaticApprover(true) }); + const skill = await bundle.skills.upsertCandidate(candidateInput(9)); + provider.candidateSkillId = skill.id; + for (let index = 1; index <= 6; index += 1) await bundle.fixtures.put(fixturePayload(index)); + await bundle.evaluations.evaluateSkill(skill.id, { + fixtures: (await bundle.fixtures.list()).slice(0, 3).map((fixture) => fixture.id), + }); + for (const fixture of (await bundle.fixtures.list()).slice(3, 6)) { + await bundle.evaluations.shadowEpisode(skill.id, fixture.sourceEpisodeId, "production-baseline"); + } + await bundle.evaluations.promoteSkill(skill.id); + const { mkdir } = await import("node:fs/promises"); + await mkdir(workspace, { recursive: true }); + await writeFile(path.join(workspace, "file-0.txt"), "alpha", "utf8"); + + const result = await bundle.runtime.run({ + goal: "Inspect file-0.txt and report it.", + requestedTools: ["read_file"], + }); + assert.equal(result.status, "committed"); + const checkpoint = await bundle.checkpoints.load(result.episodeId); + assert.ok(checkpoint.activeSkillIds.includes(skill.id)); + const fixtures = await bundle.fixtures.list(); + const captured = fixtures.find((fixture) => fixture.sourceEpisodeId === result.episodeId); + assert.ok(captured); + const events = await bundle.ledger.forEpisode(result.episodeId); + assert.ok(events.some((event) => event.type === "evaluation.fixture_captured")); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/fixture-store.test.ts b/evolve-agent/tests/fixture-store.test.ts new file mode 100644 index 0000000..f291153 --- /dev/null +++ b/evolve-agent/tests/fixture-store.test.ts @@ -0,0 +1,95 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { ArtifactStore } from "../src/ledger/artifact-store.js"; +import { JsonlLedger } from "../src/ledger/jsonl-ledger.js"; +import { CheckpointStore } from "../src/runtime/checkpoint-store.js"; +import { FixtureStore } from "../src/evaluation/fixture-store.js"; +import { sha256Json } from "../src/core/hash.js"; + +async function setupEpisode() { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-fixture-")); + const home = path.join(root, "home"); + const checkpoints = new CheckpointStore(home); + const ledger = new JsonlLedger(path.join(home, "episodes.jsonl")); + const artifacts = new ArtifactStore(home); + const fixtures = new FixtureStore(home, checkpoints, ledger, artifacts); + const episodeId = "ep_111111111111111111111111"; + const args = { path: "note.txt" }; + const evidence = await artifacts.put({ + episodeId, + toolName: "read_file", + args, + data: { content: "alpha" }, + summary: "note.txt contains alpha", + success: true, + }); + await ledger.append(episodeId, "model.tool_proposed", { tool: "read_file", args_hash: sha256Json(args) }); + await ledger.append(episodeId, "tool.executed", { + tool: "read_file", + success: true, + evidence_id: evidence.id, + artifact_hash: evidence.artifactHash, + args_hash: evidence.argsHash, + }); + await ledger.append(episodeId, "episode.committed", { + answer_hash: sha256Json("alpha"), + evidence_ids: [evidence.id], + score: 0.93, + }); + await checkpoints.save({ + version: 1, + episodeId, + task: { + id: "task_111111111111111111111111", + goal: "Inspect note.txt", + constraints: [], + successCriteria: ["Cite evidence"], + requestedTools: ["read_file"], + budget: { maxTurns: 4, maxToolCalls: 2, maxInputTokens: 1000, maxOutputTokens: 1000, maxWallTimeMs: 1000 }, + createdAt: new Date().toISOString(), + }, + status: "committed", + observations: [ + { + id: "obs_1111111111111111", + kind: "tool", + content: `note.txt contains alpha [evidence:${evidence.id}]`, + evidenceId: evidence.id, + toolName: "read_file", + }, + ], + evidenceIds: [evidence.id], + toolSequence: ["read_file"], + activeSkillIds: [], + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 }, + turns: 2, + toolCalls: 1, + elapsedMs: 50, + answer: `alpha [evidence:${evidence.id}]`, + finalScore: 0.93, + updatedAt: new Date().toISOString(), + }); + return { root, home, episodeId, fixtures }; +} + +test("fixture capture reconstructs a clean trace and detects tampering", async () => { + const environment = await setupEpisode(); + try { + const fixture = await environment.fixtures.capture(environment.episodeId, "holdout"); + assert.equal(fixture.trace.length, 1); + assert.equal(fixture.trace[0]?.toolName, "read_file"); + assert.equal(fixture.baseline.verifierScore, 0.93); + assert.equal(environment.fixtures.verify(fixture), true); + + const file = path.join(environment.home, "evaluations", "fixtures", `${fixture.id}.json`); + const tampered = JSON.parse(await readFile(file, "utf8")) as typeof fixture; + tampered.baseline.verifierScore = 0.1; + await writeFile(file, JSON.stringify(tampered), "utf8"); + await assert.rejects(environment.fixtures.get(fixture.id), /integrity/i); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/hardened-runtime.test.ts b/evolve-agent/tests/hardened-runtime.test.ts new file mode 100644 index 0000000..75b58bd --- /dev/null +++ b/evolve-agent/tests/hardened-runtime.test.ts @@ -0,0 +1,116 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { loadConfig } from "../src/config.js"; +import { createRuntime } from "../src/factory.js"; +import type { ProcessRunner, ProcessRunRequest, ProcessRunResult } from "../src/execution/process-runner.js"; +import { StaticApprover } from "../src/policy/approver.js"; +import { MockProvider } from "../src/providers/mock-provider.js"; +import type { AgentPrompt, FinalDecision } from "../src/providers/provider.js"; +import type { JsonValue } from "../src/core/types.js"; + +const IMAGE = `node:22-bookworm-slim@sha256:${"e".repeat(64)}`; + +class DockerRunner implements ProcessRunner { + public readonly calls: ProcessRunRequest[] = []; + public async run(input: ProcessRunRequest): Promise { + this.calls.push(input); + return { + code: 0, + signal: null, + timedOut: false, + truncated: false, + stdout: input.args[0] === "run" ? "v22.6.0\n" : "", + stderr: "", + durationMs: 3, + }; + } +} + +function finalFromEvidence(prompt: AgentPrompt): FinalDecision { + const evidence = prompt.observations.findLast((observation) => observation.evidenceId)?.evidenceId; + assert.ok(evidence); + return { + kind: "final", + answer: `The isolated process reported Node v22.6.0. [evidence:${evidence}]`, + evidenceIds: [evidence], + memoryProposals: [], + }; +} + +test("runtime carries exact approval through Docker execution, evidence, and final verification", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-hardened-runtime-")); + const workspace = path.join(root, "workspace"); + const home = path.join(root, "state"); + await mkdir(workspace, { recursive: true }); + const runner = new DockerRunner(); + const provider = new MockProvider([ + { + kind: "tool", + toolName: "run_process", + args: { + command: "node", + args: ["--version"], + executor: "docker", + image: IMAGE, + network: "none", + workspace_access: "read-only", + memory_mb: 256, + cpus: 0.5, + pids_limit: 32, + tmpfs_mb: 32, + }, + rationale: "Verify the runtime version in the isolated executor", + }, + finalFromEvidence, + ]); + + try { + const config = loadConfig({ + home, + workspace, + allowedCommands: ["node"], + docker: { + defaultImage: IMAGE, + allowedImages: [IMAGE], + maximums: { memoryMb: 512, cpus: 1, pids: 64, tmpfsMb: 64 }, + }, + nonInteractive: true, + }); + const bundle = createRuntime(config, { + provider, + approver: new StaticApprover(true), + processRunner: runner, + }); + const result = await bundle.runtime.run({ + goal: "Report the Node version using isolated execution.", + requestedTools: ["run_process"], + successCriteria: ["The version is supported by run_process evidence"], + }); + + assert.equal(result.status, "committed"); + assert.equal(result.toolCalls, 1); + assert.equal(runner.calls.length, 2, "docker run and orphan cleanup"); + assert.ok(runner.calls[0]?.args.includes("--read-only")); + assert.ok(runner.calls[0]?.args.includes("no-new-privileges:true")); + + const evidence = await bundle.artifacts.getEvidence(result.evidenceIds[0] as string); + assert.ok(evidence); + const artifact = (await bundle.artifacts.readArtifact(evidence.artifactHash)) as Record; + assert.equal(artifact.executor, "docker"); + assert.equal(artifact.network, "none"); + assert.equal(artifact.workspace_access, "read-only"); + const receipt = artifact.execution_receipt as Record; + assert.equal(typeof receipt.command_hash, "string"); + assert.equal(typeof receipt.policy_hash, "string"); + + const events = await bundle.ledger.forEpisode(result.episodeId); + assert.ok(events.some((event) => event.type === "approval.granted")); + assert.ok(events.some((event) => event.type === "tool.executed")); + assert.ok(events.some((event) => event.type === "episode.committed")); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/lease-manager.test.ts b/evolve-agent/tests/lease-manager.test.ts new file mode 100644 index 0000000..015aa02 --- /dev/null +++ b/evolve-agent/tests/lease-manager.test.ts @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import { hostname } from "node:os"; +import { mkdir, mkdtemp, rm, utimes, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { EpisodeLeaseManager, type EpisodeLeaseRecord } from "../src/runtime/lease-manager.js"; + +const EPISODE = "ep_aaaaaaaaaaaaaaaaaaaaaaaa"; + +async function writeStale(home: string, record: EpisodeLeaseRecord): Promise { + const directory = path.join(home, "leases"); + await mkdir(directory, { recursive: true }); + const target = path.join(directory, `${EPISODE}.lock`); + await writeFile(target, `${JSON.stringify(record)}\n`, { encoding: "utf8", mode: 0o600 }); + const old = new Date(Date.now() - 10_000); + await utimes(target, old, old); +} + +test("episode lease rejects concurrent ownership and releases cleanly", async () => { + const home = await mkdtemp(path.join(os.tmpdir(), "evolve-lease-test-")); + const first = new EpisodeLeaseManager(home, { ttlMs: 2_000, heartbeatMs: 200, ownerId: "owner-a" }); + const second = new EpisodeLeaseManager(home, { ttlMs: 2_000, heartbeatMs: 200, ownerId: "owner-b" }); + try { + const lease = await first.acquire(EPISODE); + await assert.rejects(second.acquire(EPISODE), /is leased by owner-a/); + await lease.release(); + const next = await second.acquire(EPISODE); + assert.equal(next.ownerId, "owner-b"); + await next.release(); + } finally { + await rm(home, { recursive: true, force: true }); + } +}); + +test("stale lease is recovered only when the recorded local process is gone", async () => { + const home = await mkdtemp(path.join(os.tmpdir(), "evolve-lease-stale-")); + const manager = new EpisodeLeaseManager(home, { + ttlMs: 1_000, + heartbeatMs: 100, + ownerId: "recovery-owner", + hostname: hostname(), + }); + try { + await writeStale(home, { + version: 1, + episodeId: EPISODE, + ownerId: "dead-owner", + pid: 999_999_999, + hostname: hostname(), + acquiredAt: new Date(Date.now() - 20_000).toISOString(), + }); + const recovered = await manager.acquire(EPISODE); + assert.equal(recovered.recovered?.ownerId, "dead-owner"); + await recovered.release(); + + await writeStale(home, { + version: 1, + episodeId: EPISODE, + ownerId: "live-owner", + pid: process.pid, + hostname: hostname(), + acquiredAt: new Date(Date.now() - 20_000).toISOString(), + }); + await assert.rejects(manager.acquire(EPISODE), /owner process .* is still alive/); + } finally { + await rm(home, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/ledger.test.ts b/evolve-agent/tests/ledger.test.ts new file mode 100644 index 0000000..e8691ba --- /dev/null +++ b/evolve-agent/tests/ledger.test.ts @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { JsonlLedger } from "../src/ledger/jsonl-ledger.js"; + +test("ledger detects post-hoc event mutation", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-ledger-")); + const ledgerPath = path.join(root, "episodes.jsonl"); + try { + const ledger = new JsonlLedger(ledgerPath); + await ledger.append("ep_aaaaaaaaaaaaaaaaaaaaaaaa", "episode.started", { value: 1 }); + await ledger.append("ep_aaaaaaaaaaaaaaaaaaaaaaaa", "episode.committed", { value: 2 }); + assert.deepEqual(await ledger.verify(), { valid: true, events: 2 }); + + const lines = (await readFile(ledgerPath, "utf8")).trim().split("\n"); + const second = JSON.parse(lines[1] as string) as { payload: { value: number } }; + second.payload.value = 999; + lines[1] = JSON.stringify(second); + await writeFile(ledgerPath, `${lines.join("\n")}\n`, "utf8"); + const result = await ledger.verify(); + assert.equal(result.valid, false); + assert.match(result.error ?? "", /hash mismatch/i); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/local-executor.test.ts b/evolve-agent/tests/local-executor.test.ts new file mode 100644 index 0000000..67b86c8 --- /dev/null +++ b/evolve-agent/tests/local-executor.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { LocalExecutor } from "../src/execution/local-executor.js"; +import type { ProcessRunner } from "../src/execution/process-runner.js"; +import type { ExecutionRequest } from "../src/execution/types.js"; + +const runner: ProcessRunner = { + async run() { + return { + code: 0, + signal: null, + timedOut: false, + truncated: false, + stdout: "ok", + stderr: "", + durationMs: 1, + }; + }, +}; + +function request(): ExecutionRequest { + return { + runId: "local-run", + command: "node", + args: ["--version"], + workspace: process.cwd(), + cwd: process.cwd(), + timeoutMs: 1_000, + maxOutputBytes: 2_000, + workspaceAccess: "read-only", + network: "none", + secretNames: [], + limits: { memoryMb: 128, cpus: 1, pids: 32, tmpfsMb: 16 }, + }; +} + +test("local executor requires explicit acknowledgement of unenforceable host access", async () => { + const executor = new LocalExecutor(runner); + await assert.rejects(executor.execute(request()), /cannot enforce network isolation/); + + const hostNetwork = { ...request(), network: "host" }; + await assert.rejects(executor.execute(hostNetwork), /cannot enforce a read-only workspace/); + + const explicit = { ...hostNetwork, workspaceAccess: "read-write" as const }; + const result = await executor.execute(explicit); + assert.equal(result.success, true); + assert.equal(result.isolation.boundary, "none"); + assert.equal(result.workspaceAccess, "read-write"); +}); diff --git a/evolve-agent/tests/memory.test.ts b/evolve-agent/tests/memory.test.ts new file mode 100644 index 0000000..83be82b --- /dev/null +++ b/evolve-agent/tests/memory.test.ts @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { MemoryStore } from "../src/memory/memory-store.js"; + +test("high-confidence memory is rejected without valid evidence", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-memory-")); + try { + const store = new MemoryStore(root); + await assert.rejects( + store.add({ + text: "Use compare-and-swap when replacing a file.", + tags: ["files"], + confidence: 0.9, + sourceEpisodeId: "ep_aaaaaaaaaaaaaaaaaaaaaaaa", + evidenceIds: [], + validEvidenceIds: new Set(), + }), + /requires current-episode evidence/i, + ); + const evidence = "ev_aaaaaaaaaaaaaaaaaaaaaaaa"; + const record = await store.add({ + text: "Use compare-and-swap when replacing a file.", + tags: ["files", "safety"], + confidence: 0.9, + sourceEpisodeId: "ep_aaaaaaaaaaaaaaaaaaaaaaaa", + evidenceIds: [evidence], + validEvidenceIds: new Set([evidence]), + }); + assert.equal(record.evidenceIds[0], evidence); + const matches = await store.search("safe file replace compare swap"); + assert.equal(matches[0]?.id, record.id); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/openai-provider.test.ts b/evolve-agent/tests/openai-provider.test.ts new file mode 100644 index 0000000..1474a1e --- /dev/null +++ b/evolve-agent/tests/openai-provider.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import test from "node:test"; +import { OpenAIResponsesProvider } from "../src/providers/openai-responses.js"; +import type { AgentPrompt } from "../src/providers/provider.js"; + +async function readBody(request: import("node:http").IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks).toString("utf8"); +} + +test("OpenAI adapter sends a Responses API function-tool request for gpt-5.6-sol", async () => { + let captured: Record | undefined; + const server = createServer(async (request, response) => { + captured = JSON.parse(await readBody(request)) as Record; + response.writeHead(200, { "content-type": "application/json" }); + response.end( + JSON.stringify({ + id: "resp_test", + output: [ + { + type: "function_call", + name: "commit_answer", + call_id: "call_test", + arguments: JSON.stringify({ answer: "done", evidence_ids: [], memory_proposals: [] }), + }, + ], + usage: { input_tokens: 10, output_tokens: 4, total_tokens: 14 }, + }), + ); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address() as AddressInfo; + const provider = new OpenAIResponsesProvider({ + apiKey: "test-key", + model: "gpt-5.6-sol", + verifierModel: "gpt-5.6-sol", + reasoningEffort: "high", + endpoint: `http://127.0.0.1:${address.port}/v1/responses`, + timeoutMs: 5_000, + }); + const prompt: AgentPrompt = { + task: { + id: "task_aaaaaaaaaaaaaaaaaaaaaaaa", + goal: "Acknowledge the task", + constraints: [], + successCriteria: ["Return an acknowledgement"], + requestedTools: ["read_file"], + budget: { + maxTurns: 3, + maxToolCalls: 1, + maxInputTokens: 10_000, + maxOutputTokens: 2_000, + maxWallTimeMs: 10_000, + }, + createdAt: new Date(0).toISOString(), + }, + observations: [], + memories: [], + skills: [], + tools: [ + { + name: "read_file", + description: "Read a file", + risk: "read", + inputSchema: { + type: "object", + properties: { path: { type: "string" } }, + required: ["path"], + additionalProperties: false, + }, + }, + ], + remainingBudget: { turns: 3, toolCalls: 1, inputTokens: 10_000, outputTokens: 2_000, wallTimeMs: 10_000 }, + }; + const result = await provider.decide(prompt); + assert.equal(result.decision.kind, "final"); + assert.equal(result.usage.totalTokens, 14); + assert.equal(captured?.model, "gpt-5.6-sol"); + assert.equal(captured?.store, false); + assert.equal(captured?.tool_choice, "required"); + assert.equal(captured?.parallel_tool_calls, false); + const tools = captured?.tools as Array<{ name: string; strict: boolean }>; + assert.ok(tools.some((tool) => tool.name === "read_file" && tool.strict === false)); + assert.ok(tools.some((tool) => tool.name === "commit_answer" && tool.strict === true)); + } finally { + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + } +}); diff --git a/evolve-agent/tests/process-runner.test.ts b/evolve-agent/tests/process-runner.test.ts new file mode 100644 index 0000000..f9d2284 --- /dev/null +++ b/evolve-agent/tests/process-runner.test.ts @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { NodeProcessRunner } from "../src/execution/process-runner.js"; + +test("process runner terminates output floods at the evidence limit", async () => { + const runner = new NodeProcessRunner(); + const result = await runner.run({ + command: process.execPath, + args: ["-e", "process.stdout.write('x'.repeat(10_000_000)); setInterval(() => {}, 1000)"], + cwd: process.cwd(), + env: { PATH: process.env.PATH }, + timeoutMs: 5_000, + maxOutputBytes: 1_024, + }); + assert.equal(result.truncated, true); + assert.equal(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr), 1_024); + assert.ok(result.durationMs < 5_000, `output cap should stop execution before timeout, got ${result.durationMs} ms`); +}); diff --git a/evolve-agent/tests/provenance-signer.test.ts b/evolve-agent/tests/provenance-signer.test.ts new file mode 100644 index 0000000..08eaf00 --- /dev/null +++ b/evolve-agent/tests/provenance-signer.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { ProvenanceSigner } from "../src/evaluation/provenance-signer.js"; +import { defaultEvaluationPolicy } from "../src/evaluation/metrics.js"; +import type { EvaluationReportPayload } from "../src/evaluation/types.js"; + +function payload(): EvaluationReportPayload { + const empty = { + samples: 0, + successes: 0, + successRate: 0, + meanVerifierScore: 0, + meanInputTokens: 0, + meanOutputTokens: 0, + meanTotalTokens: 0, + meanTurns: 0, + meanToolCalls: 0, + meanDurationMs: 0, + traceMatchRate: 0, + safetyViolations: 0, + }; + return { + version: 1, + engineVersion: "0.3.0", + kind: "offline", + skillId: "skill_aaaaaaaaaaaaaaaaaaaaaaaa", + skillFingerprint: "a".repeat(64), + createdAt: new Date(0).toISOString(), + fixtureIds: [], + fixtureHashes: [], + policy: defaultEvaluationPolicy(), + baselineRuns: [], + candidateRuns: [], + baseline: empty, + candidate: empty, + comparison: { + pairedSamples: 0, + wins: 0, + losses: 0, + ties: 0, + newFailures: 0, + successDelta: 0, + verifierScoreDelta: 0, + totalTokenRatio: 1, + toolCallRatio: 1, + durationRatio: 1, + confidence: { + successDeltaLower: 0, + successDeltaUpper: 0, + verifierScoreDeltaLower: 0, + verifierScoreDeltaUpper: 0, + }, + }, + decision: { passed: false, gates: {}, reasons: ["fixture"] }, + metadata: { provider: "test", notes: [] }, + }; +} + +test("Ed25519 evaluation provenance rejects report tampering", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-sign-")); + try { + const signer = new ProvenanceSigner(root); + const report = await signer.sign(payload()); + assert.equal(await signer.verify(report), true); + const tampered = structuredClone(report); + tampered.payload.decision.passed = true; + assert.equal(await signer.verify(tampered), false); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/replay-harness.test.ts b/evolve-agent/tests/replay-harness.test.ts new file mode 100644 index 0000000..794cc77 --- /dev/null +++ b/evolve-agent/tests/replay-harness.test.ts @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { ArtifactStore } from "../src/ledger/artifact-store.js"; +import { JsonlLedger } from "../src/ledger/jsonl-ledger.js"; +import { CheckpointStore } from "../src/runtime/checkpoint-store.js"; +import { FixtureStore } from "../src/evaluation/fixture-store.js"; +import { ReplayHarness } from "../src/evaluation/replay-harness.js"; +import { SkillStore } from "../src/skills/skill-store.js"; +import type { AgentProvider } from "../src/providers/provider.js"; +import { SkillAwareProvider, candidateInput, fixturePayload } from "./evaluation-helpers.js"; + +const tool = { + name: "read_file", + description: "Read a file", + risk: "read" as const, + inputSchema: { type: "object", properties: { path: { type: "string" } }, required: ["path"] }, +}; + +test("replay harness gives baseline and candidate the same recorded world", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-replay-")); + try { + const fixtureStore = new FixtureStore( + root, + new CheckpointStore(root), + new JsonlLedger(path.join(root, "episodes.jsonl")), + new ArtifactStore(root), + ); + const skillStore = new SkillStore(root); + const skill = await skillStore.upsertCandidate(candidateInput()); + const fixture = await fixtureStore.put(fixturePayload(5)); + const provider = new SkillAwareProvider(skill.id); + const harness = new ReplayHarness(provider, [tool]); + const baseline = await harness.run({ fixture, arm: "baseline", repeat: 0, skills: [] }); + const candidate = await harness.run({ fixture, arm: "candidate", repeat: 0, skills: [skill] }); + assert.equal(baseline.success, true); + assert.equal(candidate.success, true); + assert.equal(candidate.verifierScore, 0.96); + assert.ok(candidate.usage.totalTokens < baseline.usage.totalTokens); + assert.equal(candidate.fixtureHash, baseline.fixtureHash); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("replay harness fails closed when proposed arguments diverge from the fixture", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-replay-mismatch-")); + try { + const fixtureStore = new FixtureStore( + root, + new CheckpointStore(root), + new JsonlLedger(path.join(root, "episodes.jsonl")), + new ArtifactStore(root), + ); + const fixture = await fixtureStore.put(fixturePayload(6)); + const provider: AgentProvider = { + async decide() { + return { + decision: { kind: "tool", toolName: "read_file", args: { path: "wrong.txt" }, rationale: "wrong" }, + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }; + }, + async verify() { + throw new Error("Verifier must not run after trace mismatch"); + }, + }; + const result = await new ReplayHarness(provider, [tool]).run({ fixture, arm: "baseline", repeat: 0, skills: [] }); + assert.equal(result.success, false); + assert.ok(result.safetyViolations.includes("trace_mismatch")); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/runtime.test.ts b/evolve-agent/tests/runtime.test.ts new file mode 100644 index 0000000..beba05c --- /dev/null +++ b/evolve-agent/tests/runtime.test.ts @@ -0,0 +1,196 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { loadConfig } from "../src/config.js"; +import { createRuntime } from "../src/factory.js"; +import { StaticApprover } from "../src/policy/approver.js"; +import { MockProvider, type MockDecision } from "../src/providers/mock-provider.js"; +import type { AgentPrompt, FinalDecision } from "../src/providers/provider.js"; + +function latestEvidence(prompt: AgentPrompt): string { + const evidence = prompt.observations.findLast((observation) => observation.evidenceId)?.evidenceId; + assert.ok(evidence, "expected a tool evidence ID in the next prompt"); + return evidence; +} + +function evidenceFinal(text: string, memory = false): MockDecision { + return (prompt): FinalDecision => { + const evidence = latestEvidence(prompt); + return { + kind: "final", + answer: `${text} [evidence:${evidence}]`, + evidenceIds: [evidence], + memoryProposals: memory + ? [ + { + text: "The inspected workspace contains the requested source file.", + tags: ["workspace", "inspection"], + confidence: 0.9, + evidenceIds: [evidence], + }, + ] + : [], + }; + }; +} + +async function setup(decisions: MockDecision[], approver = new StaticApprover(true)) { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-runtime-")); + const workspace = path.join(root, "workspace"); + const home = path.join(root, "home"); + const provider = new MockProvider(decisions); + const config = loadConfig({ home, workspace, allowedCommands: ["node"], nonInteractive: true }); + const bundle = createRuntime(config, { provider, approver }); + return { root, workspace, home, provider, bundle }; +} + +test("runtime commits only after tool evidence and independent verification", async () => { + const environment = await setup( + [ + { kind: "tool", toolName: "read_file", args: { path: "note.txt" }, rationale: "Inspect the file" }, + evidenceFinal("The file contains alpha.", true), + ], + ); + try { + await writeFile(path.join(environment.workspace, "note.txt"), "alpha\n", { encoding: "utf8", flag: "wx" }).catch(async (error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + const { mkdir } = await import("node:fs/promises"); + await mkdir(environment.workspace, { recursive: true }); + await writeFile(path.join(environment.workspace, "note.txt"), "alpha\n", "utf8"); + return; + } + throw error; + }); + const result = await environment.bundle.runtime.run({ + goal: "Read note.txt and report its content.", + requestedTools: ["read_file"], + successCriteria: ["The answer cites the read_file evidence"], + }); + assert.equal(result.status, "committed"); + assert.equal(result.toolCalls, 1); + assert.match(result.answer ?? "", /\[evidence:ev_[a-f0-9]{24}\]/); + assert.equal(environment.provider.verifications.length, 1); + assert.deepEqual(await environment.bundle.ledger.verify(), { valid: true, events: (await environment.bundle.ledger.readAll()).length }); + const episodeEvents = await environment.bundle.ledger.forEpisode(result.episodeId); + assert.ok(episodeEvents.some((event) => event.type === "episode.lease_acquired")); + assert.equal((await environment.bundle.memory.list()).length, 1); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); + +test("invented evidence is rejected before the independent verifier runs", async () => { + const fake = "ev_ffffffffffffffffffffffff"; + const environment = await setup([ + { + kind: "final", + answer: `Fabricated claim [evidence:${fake}]`, + evidenceIds: [fake], + memoryProposals: [], + }, + { kind: "final", answer: "No external facts are needed for this response.", evidenceIds: [], memoryProposals: [] }, + ]); + try { + const result = await environment.bundle.runtime.run({ + goal: "Return a short acknowledgement.", + requestedTools: [], + budget: { maxTurns: 3 }, + }); + assert.equal(result.status, "committed"); + assert.equal(environment.provider.prompts.length, 2); + assert.equal(environment.provider.verifications.length, 1); + const events = await environment.bundle.ledger.forEpisode(result.episodeId); + assert.ok(events.some((event) => event.type === "verification.rejected")); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); + +test("denied approval prevents file mutation and allows a safe final response", async () => { + const environment = await setup( + [ + { + kind: "tool", + toolName: "write_file", + args: { path: "blocked.txt", content: "must not exist", create_only: true }, + rationale: "Attempt a write", + }, + { + kind: "final", + answer: "The requested write was not performed because approval was denied.", + evidenceIds: [], + memoryProposals: [], + }, + ], + new StaticApprover(false), + ); + try { + const result = await environment.bundle.runtime.run({ + goal: "Create blocked.txt only if approved.", + requestedTools: ["write_file"], + }); + assert.equal(result.status, "committed"); + assert.equal(result.toolCalls, 0); + await assert.rejects(readFile(path.join(environment.workspace, "blocked.txt"), "utf8"), /ENOENT/); + const events = await environment.bundle.ledger.forEpisode(result.episodeId); + assert.ok(events.some((event) => event.type === "approval.denied")); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); + +test("turn budget stops a loop durably after the allowed turn", async () => { + const environment = await setup([ + { kind: "tool", toolName: "read_file", args: { path: "note.txt" }, rationale: "Read once" }, + ]); + try { + const { mkdir } = await import("node:fs/promises"); + await mkdir(environment.workspace, { recursive: true }); + await writeFile(path.join(environment.workspace, "note.txt"), "alpha", "utf8"); + const result = await environment.bundle.runtime.run({ + goal: "Read note.txt but stop after one model turn.", + requestedTools: ["read_file"], + budget: { maxTurns: 1 }, + }); + assert.equal(result.status, "budget_exhausted"); + assert.equal(result.turns, 1); + assert.equal(result.toolCalls, 1); + const checkpoint = await environment.bundle.checkpoints.load(result.episodeId); + assert.equal(checkpoint.status, "budget_exhausted"); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); + +test("repeated committed flows create only an inactive skill candidate", async () => { + const environment = await setup([ + { kind: "tool", toolName: "read_file", args: { path: "note.txt" }, rationale: "Inspect" }, + evidenceFinal("First inspection complete."), + { kind: "tool", toolName: "read_file", args: { path: "note.txt" }, rationale: "Inspect" }, + evidenceFinal("Second inspection complete."), + ]); + try { + const { mkdir } = await import("node:fs/promises"); + await mkdir(environment.workspace, { recursive: true }); + await writeFile(path.join(environment.workspace, "note.txt"), "alpha", "utf8"); + const first = await environment.bundle.runtime.run({ + goal: "Inspect note.txt for run one.", + requestedTools: ["read_file"], + }); + const second = await environment.bundle.runtime.run({ + goal: "Inspect note.txt for run two.", + requestedTools: ["read_file"], + }); + assert.equal(first.status, "committed"); + assert.equal(second.status, "committed"); + const skills = await environment.bundle.skills.list(); + assert.equal(skills.length, 1); + assert.equal(skills[0]?.status, "candidate"); + assert.equal(skills[0]?.supportingEpisodes.length, 2); + assert.equal((await environment.bundle.skills.promoted()).length, 0); + } finally { + await rm(environment.root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/safe-environment.test.ts b/evolve-agent/tests/safe-environment.test.ts new file mode 100644 index 0000000..4c8e3f1 --- /dev/null +++ b/evolve-agent/tests/safe-environment.test.ts @@ -0,0 +1,26 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { dockerClientEnvironment, safeEnvironment } from "../src/execution/safe-environment.js"; + +test("child environments exclude model credentials while preserving Docker connection settings", () => { + const before = { + openai: process.env.OPENAI_API_KEY, + docker: process.env.DOCKER_HOST, + }; + process.env.OPENAI_API_KEY = "must-not-propagate"; + process.env.DOCKER_HOST = "unix:///tmp/docker.sock"; + try { + const local = safeEnvironment("/workspace"); + assert.equal(local.OPENAI_API_KEY, undefined); + assert.equal(local.HOME, "/workspace"); + + const docker = dockerClientEnvironment(); + assert.equal(docker.OPENAI_API_KEY, undefined); + assert.equal(docker.DOCKER_HOST, "unix:///tmp/docker.sock"); + } finally { + if (before.openai === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = before.openai; + if (before.docker === undefined) delete process.env.DOCKER_HOST; + else process.env.DOCKER_HOST = before.docker; + } +}); diff --git a/evolve-agent/tests/secret-broker.test.ts b/evolve-agent/tests/secret-broker.test.ts new file mode 100644 index 0000000..5fedbb3 --- /dev/null +++ b/evolve-agent/tests/secret-broker.test.ts @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, readdir, rm, stat } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { FileSecretBroker } from "../src/secrets/secret-broker.js"; + +test("secret broker uses restrictive files, redacts output, and destroys the lease", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-secret-test-")); + const secretRoot = path.join(root, "runtime-secrets"); + const broker = new FileSecretBroker(secretRoot, ["API_TOKEN"], 60_000, () => "very-private-value"); + try { + const lease = await broker.materialize("run-1", ["API_TOKEN", "API_TOKEN"]); + assert.equal(lease.mounts.length, 1); + const mount = lease.mounts[0]; + assert.ok(mount); + assert.equal(await readFile(mount.hostPath, "utf8"), "very-private-value"); + assert.equal((await stat(mount.hostPath)).mode & 0o777, 0o600); + assert.equal((await stat(path.dirname(mount.hostPath))).mode & 0o777, 0o700); + assert.equal(lease.redact("value=very-private-value"), "value=[REDACTED_SECRET:API_TOKEN]"); + + await lease.release(); + assert.equal((await readdir(secretRoot)).length, 0); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("secret broker denies unapproved and empty secrets and sweeps expired leases", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-secret-sweep-")); + const values = new Map([["GOOD_TOKEN", "abc"]]); + const broker = new FileSecretBroker(path.join(root, "secrets"), ["GOOD_TOKEN", "EMPTY_TOKEN"], 1_000, (name) => + values.get(name), + ); + try { + await assert.rejects(broker.materialize("run", ["DENIED_TOKEN"]), /not in EVOLVE_SECRET_ALLOWLIST/); + values.set("EMPTY_TOKEN", ""); + await assert.rejects(broker.materialize("run", ["EMPTY_TOKEN"]), /empty value/); + + const lease = await broker.materialize("run", ["GOOD_TOKEN"]); + assert.equal((await readdir(path.join(root, "secrets"))).length, 1); + assert.equal(await broker.sweepExpired(Date.now() + 2_000), 1); + assert.equal((await readdir(path.join(root, "secrets"))).length, 0); + await lease.release(); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/skills.test.ts b/evolve-agent/tests/skills.test.ts new file mode 100644 index 0000000..8113bae --- /dev/null +++ b/evolve-agent/tests/skills.test.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { SkillStore } from "../src/skills/skill-store.js"; + +async function candidate(store: SkillStore) { + return store.upsertCandidate({ + fingerprint: "a".repeat(64), + name: "Inspect package metadata", + description: "Read a package manifest and report its scripts.", + triggers: ["package", "scripts"], + steps: [{ toolName: "read_file", purpose: "Read the manifest" }], + allowedTools: ["read_file"], + supportingEpisodes: [ + "ep_aaaaaaaaaaaaaaaaaaaaaaaa", + "ep_bbbbbbbbbbbbbbbbbbbbbbbb", + ], + provenanceEvidenceIds: ["ev_aaaaaaaaaaaaaaaaaaaaaaaa", "ev_bbbbbbbbbbbbbbbbbbbbbbbb"], + }); +} + +test("skill promotion requires attached offline and canary reports", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-skills-")); + try { + const unauthorizedStore = new SkillStore(path.join(root, "unauthorized")); + const unauthorized = await candidate(unauthorizedStore); + await unauthorizedStore.attachEvaluationReport(unauthorized.id, "report_aaaaaaaaaaaaaaaaaaaaaaaa", true); + await unauthorizedStore.attachCanaryReport(unauthorized.id, "report_bbbbbbbbbbbbbbbbbbbbbbbb", true); + await assert.rejects( + unauthorizedStore.promote(unauthorized.id, { + offlineReportId: "report_aaaaaaaaaaaaaaaaaaaaaaaa", + canaryReportId: "report_bbbbbbbbbbbbbbbbbbbbbbbb", + policyHash: "c".repeat(64), + keyFingerprint: "d".repeat(32), + }), + /signed-report promotion verifier/i, + ); + + const store = new SkillStore(root, { async verifyPromotion() {} }); + const skill = await candidate(store); + await assert.rejects( + store.promote(skill.id, { + offlineReportId: "report_aaaaaaaaaaaaaaaaaaaaaaaa", + canaryReportId: "report_bbbbbbbbbbbbbbbbbbbbbbbb", + policyHash: "c".repeat(64), + keyFingerprint: "d".repeat(32), + }), + /canary state/i, + ); + await store.attachEvaluationReport(skill.id, "report_aaaaaaaaaaaaaaaaaaaaaaaa", true); + await store.attachCanaryReport(skill.id, "report_bbbbbbbbbbbbbbbbbbbbbbbb", true); + const promoted = await store.promote(skill.id, { + offlineReportId: "report_aaaaaaaaaaaaaaaaaaaaaaaa", + canaryReportId: "report_bbbbbbbbbbbbbbbbbbbbbbbb", + policyHash: "c".repeat(64), + keyFingerprint: "d".repeat(32), + }); + assert.equal(promoted.status, "promoted"); + assert.equal(promoted.promotion?.offlineReportId, "report_aaaaaaaaaaaaaaaaaaaaaaaa"); + const rolledBack = await store.rollback(skill.id, "Regression observed", { + automatic: true, + reportId: "report_cccccccccccccccccccccccc", + }); + assert.equal(rolledBack.status, "rolled_back"); + assert.equal(rolledBack.rollback?.automatic, true); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("evaluation freezes supporting Episodes so later runs remain holdout material", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-skills-freeze-")); + try { + const store = new SkillStore(root); + const skill = await candidate(store); + await store.attachEvaluationReport(skill.id, "report_aaaaaaaaaaaaaaaaaaaaaaaa", true); + const updated = await store.upsertCandidate({ + fingerprint: skill.fingerprint, + name: skill.name, + description: skill.description, + triggers: ["metadata"], + steps: skill.steps, + allowedTools: skill.allowedTools, + supportingEpisodes: ["ep_cccccccccccccccccccccccc"], + provenanceEvidenceIds: ["ev_cccccccccccccccccccccccc"], + }); + assert.deepEqual(updated.supportingEpisodes, skill.supportingEpisodes); + assert.ok(updated.triggers.includes("metadata")); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tests/workspace.test.ts b/evolve-agent/tests/workspace.test.ts new file mode 100644 index 0000000..463e066 --- /dev/null +++ b/evolve-agent/tests/workspace.test.ts @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { resolveWorkspacePath } from "../src/tools/workspace.js"; + +test("workspace path resolution rejects traversal and symlinks", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "evolve-workspace-")); + const workspace = path.join(root, "workspace"); + const outside = path.join(root, "outside"); + try { + await mkdir(workspace, { recursive: true }); + await mkdir(outside, { recursive: true }); + await writeFile(path.join(outside, "secret.txt"), "secret", "utf8"); + assert.equal(await resolveWorkspacePath(workspace, "safe.txt", { createParent: true }), path.join(workspace, "safe.txt")); + await assert.rejects(resolveWorkspacePath(workspace, "../outside/secret.txt"), /escapes workspace/i); + await symlink(outside, path.join(workspace, "link")); + await assert.rejects(resolveWorkspacePath(workspace, "link/secret.txt"), /symlink/i); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/evolve-agent/tsconfig.json b/evolve-agent/tsconfig.json new file mode 100644 index 0000000..d54cf97 --- /dev/null +++ b/evolve-agent/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023"], + "rootDir": "src", + "outDir": "dist", + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "useUnknownInCatchVariables": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "skipLibCheck": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules", "tests", ".test-dist"] +} diff --git a/evolve-agent/tsconfig.test.json b/evolve-agent/tsconfig.test.json new file mode 100644 index 0000000..03e0242 --- /dev/null +++ b/evolve-agent/tsconfig.test.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "outDir": ".test-dist", + "declaration": false, + "declarationMap": false, + "sourceMap": false + }, + "include": ["src/**/*.ts", "tests/**/*.ts"], + "exclude": ["dist", "node_modules", ".test-dist"] +}