diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..94cc1578 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,142 @@ +name: Benchmarks + +on: + workflow_dispatch: + pull_request: + branches: [main] + paths: + - .cargo/** + - .github/workflows/benchmarks.yml + - benches/** + - Cargo.toml + - Cargo.lock + - src/** + - symposium-install/** + - symposium-sdk/** + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + measure: + name: Measure benchmarks + runs-on: ubuntu-24.04 + timeout-minutes: 45 + + steps: + - uses: actions/checkout@v7 + + - name: Install Rust 1.98.0 + uses: dtolnay/rust-toolchain@stable + with: + toolchain: 1.98.0 + + - name: Cache Cargo dependencies + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: benchmark-rust-1.98.0-${{ hashFiles('Cargo.lock') }} + + - name: Record benchmark environment + shell: bash + run: | + { + printf 'Commit: %s\n\n' "$GITHUB_SHA" + printf 'Runner image: %s (%s)\n\n' \ + "${ImageOS:-unknown}" "${ImageVersion:-unknown}" + rustc --version --verbose + printf '\n' + cargo --version --verbose + printf '\n' + uname -a + printf '\n' + lscpu + } | tee benchmark-environment.txt + + - name: Smoke-test benchmark workloads + run: cargo test -p symposium-benchsuite --benches --locked + + - name: Measure workspace dependency resolution + run: cargo bench -p symposium-benchsuite --bench workspace_deps --locked + + - name: Measure hook dispatch + run: cargo bench -p symposium-benchsuite --bench hook_dispatch --locked + + - name: Write benchmark summary + shell: bash + run: | + set -euo pipefail + + median_ns() { + jq -er '.median.point_estimate' \ + "target/criterion/$1/new/estimates.json" + } + + format_ms() { + awk -v nanoseconds="$1" \ + 'BEGIN { printf "%.2f ms", nanoseconds / 1000000 }' + } + + cache_miss=$(median_ns \ + workspace_deps/symposium_cache_miss) + cache_hit=$(median_ns \ + workspace_deps/new_resolver_disk_cache_hit) + minimal_hook=$(median_ns \ + hook_dispatch/pre_tool_use_minimal_config) + registry_hook=$(median_ns \ + hook_dispatch/pre_tool_use_local_registry) + cache_speedup=$(awk \ + -v miss="$cache_miss" \ + -v hit="$cache_hit" \ + 'BEGIN { printf "%.2fx", miss / hit }') + os_name=$(. /etc/os-release && printf '%s' "$PRETTY_NAME") + cpu_name=$(lscpu | awk -F: \ + '$1 == "Model name" { sub(/^[[:space:]]+/, "", $2); print $2 }') + + { + echo '> These experimental measurements are informational.' + echo '> They do not gate merges or establish a regression.' + echo + echo '## Benchmark environment' + printf -- '- Commit: `%s`\n' "$GITHUB_SHA" + printf -- '- Rust: `%s`\n' "$(rustc --version)" + printf -- '- Cargo: `%s`\n' "$(cargo --version)" + printf -- '- Runner image: `%s` (`%s`)\n' \ + "${ImageOS:-unknown}" "${ImageVersion:-unknown}" + printf -- '- OS: `%s`\n' "$os_name" + printf -- '- CPU: `%s` (%s available cores)\n\n' \ + "$cpu_name" "$(nproc)" + echo '## Median estimates' + echo '| Benchmark | Median |' + echo '| --- | ---: |' + printf '| `workspace_deps/symposium_cache_miss` | %s |\n' \ + "$(format_ms "$cache_miss")" + printf '| `workspace_deps/new_resolver_disk_cache_hit` | %s |\n' \ + "$(format_ms "$cache_hit")" + printf '| `hook_dispatch/pre_tool_use_minimal_config` | %s |\n' \ + "$(format_ms "$minimal_hook")" + printf '| `hook_dispatch/pre_tool_use_local_registry` | %s |\n\n' \ + "$(format_ms "$registry_hook")" + printf '**Workspace cache speedup:** %s\n' "$cache_speedup" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v7 + with: + name: criterion-${{ github.sha }}-attempt-${{ github.run_attempt }} + path: | + benchmark-environment.txt + target/criterion/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f12696c..2e2f9ab8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,12 +25,12 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install Rust uses: dtolnay/rust-toolchain@stable - name: Cache Rust dependencies - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.cargo/registry/index/ @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -71,18 +71,22 @@ jobs: include: - name: ubuntu os: ubuntu-latest + native: true - name: macos os: macos-latest + native: true - name: musl os: ubuntu-latest target: x86_64-unknown-linux-musl + native: false - name: windows os: windows-latest + native: true runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -96,7 +100,7 @@ jobs: sudo apt-get install -y musl-tools - name: Cache Rust dependencies - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.cargo/registry/index/ @@ -125,6 +129,10 @@ jobs: - name: Test run: cargo test ${{ matrix.target && format('--target {0}', matrix.target) || '' }} + - name: Check benchmarks + if: matrix.native + run: cargo check -p symposium-benchsuite --all-targets + xtask: name: Run xtask checks needs: check @@ -132,13 +140,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install Rust uses: dtolnay/rust-toolchain@stable - name: Cache Rust dependencies - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/.cargo/registry/index/ diff --git a/.github/workflows/deploy-book.yml b/.github/workflows/deploy-book.yml index 5ec3336f..20a289ee 100644 --- a/.github/workflows/deploy-book.yml +++ b/.github/workflows/deploy-book.yml @@ -18,7 +18,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install mdbook and preprocessors env: @@ -31,7 +31,7 @@ jobs: run: mdbook build - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@v5 with: path: book @@ -44,4 +44,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@v5 diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index d06abce1..138d6a3f 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -31,7 +31,7 @@ jobs: if: runner.os == 'Windows' run: git config --system core.longpaths true - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -65,14 +65,14 @@ jobs: - name: Upload to release if: github.event_name == 'release' - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v3 with: files: dist/cargo-agents-${{ matrix.target }}.* env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: binary-${{ matrix.target }} path: dist/cargo-agents* diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml index a099ff1a..843cc843 100644 --- a/.github/workflows/release-plz.yml +++ b/.github/workflows/release-plz.yml @@ -16,7 +16,7 @@ jobs: steps: - &checkout name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v7 with: fetch-depth: 0 persist-credentials: true diff --git a/Cargo.lock b/Cargo.lock index af2134d1..67359120 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -74,6 +83,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -126,9 +141,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "assert_matches" @@ -261,6 +276,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.57" @@ -305,6 +326,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -468,6 +516,72 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -623,6 +737,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -916,6 +1036,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -1248,6 +1379,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1484,6 +1624,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -1496,6 +1642,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking" version = "2.2.1" @@ -1591,6 +1747,34 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -1746,6 +1930,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2461,6 +2665,19 @@ dependencies = [ "url", ] +[[package]] +name = "symposium-benchsuite" +version = "0.1.0" +dependencies = [ + "anyhow", + "criterion", + "indoc", + "serde_json", + "symposium", + "tempfile", + "tokio", +] + [[package]] name = "symposium-install" version = "0.1.0" @@ -2666,6 +2883,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.11.0" @@ -3192,6 +3419,22 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -3201,6 +3444,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" diff --git a/Cargo.toml b/Cargo.toml index 8e080599..f3bc8c16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,8 @@ license = "MIT OR Apache-2.0" description = "AI the Rust Way" repository = "https://github.com/symposium-dev/symposium" readme = "README.md" +autobenches = false +exclude = ["/benches"] [lib] name = "symposium" @@ -63,4 +65,4 @@ indoc = "2.0.7" symposium-testlib = { path = "symposium-testlib" } [workspace] -members = [".", "symposium-testlib", "symposium-install", "symposium-sdk", "xtask"] +members = [".", "symposium-testlib", "symposium-install", "symposium-sdk", "xtask", "benches/benchsuite"] diff --git a/benches/README.md b/benches/README.md new file mode 100644 index 00000000..89248454 --- /dev/null +++ b/benches/README.md @@ -0,0 +1,53 @@ +# Symposium benchmarks + +This directory contains Symposium's focused performance benchmarks, checked-in workloads, and shared benchmark support. The suite is developed incrementally: every target should be independently runnable and have a clearly documented interpretation. + +See the [benchmarking design](../md/design/benchmarking.md) for the suite architecture, measurement policy, CI strategy, and lifecycle criteria. + +## Layout + +- `benchsuite/` is the non-publishable workspace package containing benchmark targets and shared support code. +- `fixtures/` contains the composable deterministic workloads described in its own [README](fixtures/README.md). + +Shared support code handles fixtures and sandbox mechanics. Each benchmark target is responsible for defining its own scenarios and timed operations. + +## Current targets + +| Target | Cases | Lifecycle | +| --- | --- | --- | +| [`workspace_deps`](benchsuite/benches/workspace_deps.rs) | `symposium_cache_miss`, `new_resolver_disk_cache_hit` | Experimental | +| [`hook_dispatch`](benchsuite/benches/hook_dispatch.rs) | `pre_tool_use_minimal_config`, `pre_tool_use_local_registry` | Experimental | + +`workspace_deps` compares dependency resolution with an empty Symposium +workspace cache against a new resolver reusing a valid disk cache. The miss is +not a fully cold machine load: Cargo and operating-system caches may already be +warm. The target's source contains the complete measurement contracts. + +`hook_dispatch` measures the in-process `PreToolUse` path in an unchanged +workspace. Its minimal case disables all plugin registries to establish the +fixed pipeline and Cargo workspace-lookup floor. Its local-registry case adds +the checked-in three-plugin registry as a representative end-to-end workload. +The two Cargo workspace lookups currently dominate both cases, so the latter +is not an isolated measurement of registry or predicate processing. + +## Commands + +Run commands from the repository root: + +```text +cargo check -p symposium-benchsuite --all-targets +cargo test -p symposium-benchsuite --lib +cargo test -p symposium-benchsuite --benches +cargo bench -p symposium-benchsuite --bench workspace_deps +cargo bench -p symposium-benchsuite --bench hook_dispatch +``` + +Pass a case name after `--` to run only that case. + +## Benchmark contracts + +Every benchmark target documents its measurement contract in the crate-level doc comment next to the implementation. This README acts as an index and does not duplicate those contracts. + +## Lifecycle + +New benchmarks begin as `experimental`. Measurements remain informational until sufficient history demonstrates that a benchmark is stable enough to become `observed` or `gated`. diff --git a/benches/benchsuite/Cargo.toml b/benches/benchsuite/Cargo.toml new file mode 100644 index 00000000..cb0e3e8e --- /dev/null +++ b/benches/benchsuite/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "symposium-benchsuite" +version = "0.1.0" +edition = "2024" +publish = false +autobenches = false + +[dependencies] +anyhow = "1.0.104" +indoc = "2.0.7" +symposium = { version = "0.4.0", path = "../.." } +tempfile = "3.27.0" + +[dev-dependencies] +criterion = "0.8.2" +serde_json = "1" +tokio = { version = "1", features = ["rt-multi-thread"] } + +[[bench]] +name = "workspace_deps" +harness = false + +[[bench]] +name = "hook_dispatch" +harness = false diff --git a/benches/benchsuite/benches/hook_dispatch.rs b/benches/benchsuite/benches/hook_dispatch.rs new file mode 100644 index 00000000..937b93af --- /dev/null +++ b/benches/benchsuite/benches/hook_dispatch.rs @@ -0,0 +1,373 @@ +//! Unchanged-workspace hook dispatch benchmarks. +//! +//! `PreToolUse` fires once per agent tool call, so its latency is the cost a +//! user feels most often. Preparation asserts every property a measurement +//! depends on, so +//! `cargo test -p symposium-benchsuite --bench hook_dispatch` is a correctness +//! preflight for the dispatch path. +//! +//! # Contract +//! +//! Both cases share these fields; the table records what each one adds. +//! +//! - **Claim:** End-to-end wall-clock latency of the in-process `PreToolUse` +//! pipeline in an unchanged Cargo workspace, at its floor and with a +//! representative local registry. Neither case isolates registry processing. +//! - **Workload:** A staged copy of the reference project, plus the three-entry +//! local registry for that case, with default auto-sync enabled, both builtin +//! registries disabled, fresh workspace state, and a valid `WorkspaceDeps` +//! disk cache. +//! - **Timed operation:** `execute_hook`: input parsing, the auto-sync +//! freshness decision, builtin dispatch, workspace-cache reuse, plugin +//! activation, and output serialization. +//! - **Excluded setup:** Everything `HookDispatchWorkload::prepare` does, +//! including fixture staging, `Symposium` and Tokio runtime construction, +//! cache population, and the invariant checks. +//! - **Invariants:** Auto-sync is enabled; the loaded registries and plugins are +//! exactly those the scenario names; workspace state and the dependency cache +//! are valid; `cargo metadata` is not attempted; no external plugin process +//! runs; and a preflight produces the expected no-op output. +//! - **Metric:** Wall-clock time per in-process `PreToolUse` dispatch. +//! - **Noise:** Two Cargo workspace-lookup subprocesses dominate both results +//! and can mask changes in registry loading and predicate evaluation. +//! Filesystem and operating-system caches, process scheduling, shared-runner +//! hardware, and developer-level Cargo configuration also vary. +//! - **Lifecycle:** Experimental. +//! +//! | Case | Configuration | Adds to the timed operation | +//! | --- | --- | --- | +//! | `pre_tool_use_minimal_config` | no registries | nothing; the pipeline and subprocess floor | +//! | `pre_tool_use_local_registry` | one path registry, three plugins | registry loading, hook selection, and predicate evaluation, with the sentinel absent so the gated hook never spawns | + +use std::{ + hint::black_box, + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::{Context, Result, ensure}; +use criterion::{Criterion, SamplingMode, criterion_group, criterion_main}; +use serde_json::{Value, json}; +use tokio::runtime::{Builder, Runtime}; + +use symposium::{ + config::Symposium, + hook::{self, HookAgent, HookEvent}, + plugins, + workspace_state::WorkspaceState, +}; +use symposium_benchsuite::{Fixture, MetadataRejectingCargo, Sandbox, StagedFixture}; + +const LOCAL_REGISTRY_PLUGINS: &[&str] = &["always-active", "dormant", "predicate-gated"]; +const PREDICATE_SENTINEL: &str = ".symposium-benchmark-never-present"; + +/// The checked-in configuration and fixtures for one dispatch measurement. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum HookDispatchScenario { + Minimal, + LocalRegistry, +} + +impl HookDispatchScenario { + const fn config(self) -> &'static str { + match self { + Self::Minimal => include_str!("../../fixtures/config/minimal.toml"), + Self::LocalRegistry => { + include_str!("../../fixtures/config/local-registry.toml") + } + } + } + + const fn registry_names(self) -> &'static [&'static str] { + match self { + Self::Minimal => &[], + Self::LocalRegistry => &["benchmark-local"], + } + } + + const fn plugin_names(self) -> &'static [&'static str] { + match self { + Self::Minimal => &[], + Self::LocalRegistry => LOCAL_REGISTRY_PLUGINS, + } + } + + fn stage_supporting_fixtures(self, sandbox: &Sandbox) -> Result<()> { + match self { + Self::Minimal => Ok(()), + Self::LocalRegistry => { + sandbox.stage(Fixture::LocalRegistry)?; + Ok(()) + } + } + } + + fn verify_process_state(self) -> Result<()> { + match self { + Self::Minimal => Ok(()), + Self::LocalRegistry => { + let current_dir = std::env::current_dir() + .context("reading the benchmark process working directory")?; + let sentinel = current_dir.join(PREDICATE_SENTINEL); + + ensure!( + !sentinel.try_exists().with_context(|| format!( + "checking for predicate sentinel `{}`", + sentinel.display() + ))?, + "predicate sentinel unexpectedly exists: {}", + sentinel.display() + ); + + Ok(()) + } + } + } +} + +/// Construction checks every property a measurement depends on, so a broken +/// setup fails the run rather than shortening a sample. +struct HookDispatchWorkload { + sandbox: Sandbox, + symposium: Symposium, + runtime: Runtime, + input: String, +} + +impl HookDispatchWorkload { + fn prepare(scenario: HookDispatchScenario) -> Result { + let sandbox = Sandbox::new()?; + let project = sandbox.stage(Fixture::ReferenceProject)?; + scenario.stage_supporting_fixtures(&sandbox)?; + sandbox.write_config(scenario.config())?; + + // `from_dir` reads `config.toml` eagerly, so it has to exist by now. + let symposium = Symposium::from_dir(sandbox.config_dir()); + // The pipeline awaits subprocesses; a worker pool would only add noise. + let runtime = Builder::new_current_thread() + .enable_all() + .build() + .context("building the benchmark Tokio runtime")?; + + let workspace_root = warm_workspace_cache(&symposium, &project)?; + mark_workspace_synced(&symposium, &workspace_root)?; + + let input = pre_tool_use_payload(project.path())?; + let workload = Self { + sandbox, + symposium, + runtime, + input, + }; + + workload.verify_configuration(scenario, &project)?; + workload.verify_dispatch()?; + + Ok(workload) + } + + /// Run the operation measured by each dispatch case. + fn dispatch(&self) -> Result> { + self.dispatch_with(&self.symposium) + } + + /// Run one dispatch through a supplied context so the preflight can + /// substitute a guarded Cargo. + fn dispatch_with(&self, symposium: &Symposium) -> Result> { + self.runtime.block_on(async { + hook::execute_hook( + symposium, + HookAgent::Claude, + HookEvent::PreToolUse, + &self.input, + ) + .await + .context("dispatching the PreToolUse hook") + }) + } + + /// Prove the configuration this case describes is the one in effect. + /// + /// Each check covers a way the workload could silently measure less: + /// auto-sync off returns before workspace lookup, the wrong configuration + /// changes the registry set, and missing or malformed entries reduce the + /// plugins loaded from the fixture. + fn verify_configuration( + &self, + scenario: HookDispatchScenario, + project: &StagedFixture, + ) -> Result<()> { + ensure!( + self.symposium.config.auto_sync, + "the hook-dispatch workload requires auto-sync to be enabled" + ); + + let registries = self.symposium.registry_instances(); + check_names( + "registry instances", + scenario.registry_names(), + registries.iter().map(|registry| registry.name.as_str()), + )?; + + let resolver = self.symposium.workspace_deps(project.path()); + let workspace = resolver + .load() + .context("loading the prepared workspace disk cache")?; + let registry = self.runtime.block_on(plugins::load_registry_with_workspace( + &self.symposium, + Some(workspace), + )); + + check_names( + "loaded plugins", + scenario.plugin_names(), + registry + .plugins + .iter() + .map(|parsed| parsed.plugin.name.as_str()), + )?; + ensure!( + registry.warnings.is_empty(), + "the hook-dispatch configuration produced {} plugin load warning(s)", + registry.warnings.len() + ); + scenario.verify_process_state()?; + + Ok(()) + } + + /// Dispatch once through a Cargo that refuses `metadata`. + /// + /// Refusal alone proves nothing: a failed `metadata` becomes "no workspace", + /// which dispatch accepts and still returns `{}` for. The marker is what + /// separates reading the disk cache from re-resolving and discarding. + fn verify_dispatch(&self) -> Result<()> { + let guard = MetadataRejectingCargo::create_in(self.sandbox.root())?; + let mut guarded = Symposium::from_dir(self.sandbox.config_dir()); + guarded.set_cargo_override(guard.executable().to_path_buf()); + + let output = self.dispatch_with(&guarded)?; + + ensure!( + !guard.saw_metadata()?, + "the unchanged path ran `cargo metadata` instead of reading the \ + workspace disk cache" + ); + + let output: Value = + serde_json::from_slice(&output).context("parsing the hook output as JSON")?; + ensure!( + output == json!({}), + "expected a no-op hook output, found `{output}`" + ); + + Ok(()) + } +} + +fn check_names<'a>( + kind: &str, + expected: &[&str], + actual: impl IntoIterator, +) -> Result<()> { + let mut expected = expected.to_vec(); + let mut actual: Vec<_> = actual.into_iter().collect(); + expected.sort_unstable(); + actual.sort_unstable(); + + ensure!( + actual == expected, + "unexpected {kind}: expected [{}], found [{}]", + expected.join(", "), + actual.join(", ") + ); + + Ok(()) +} + +/// Validate the resolved graph and leave a warm disk cache behind. +fn warm_workspace_cache(symposium: &Symposium, project: &StagedFixture) -> Result { + let resolver = symposium.workspace_deps(project.path()); + let workspace = resolver.load().with_context(|| { + format!( + "resolving the staged benchmark workspace `{}`", + project.path().display() + ) + })?; + + project.check_workspace(workspace)?; + + Ok(workspace.root.clone()) +} + +/// `run_auto_sync` skips its work only when recorded state says the workspace +/// is unchanged; without this the dispatch measures a full sync instead. The +/// recorded root mirrors what a real sync writes. +fn mark_workspace_synced(symposium: &Symposium, workspace_root: &Path) -> Result<()> { + let mut state = WorkspaceState::load(symposium, workspace_root); + state.record_sync(workspace_root); + state.workspace_root = Some(workspace_root.to_path_buf()); + state.save(symposium, workspace_root); + + ensure!( + WorkspaceState::load(symposium, workspace_root).sync_is_fresh(workspace_root), + "recorded workspace state did not reload as fresh for `{}`", + workspace_root.display() + ); + + Ok(()) +} + +/// `cwd` is load-bearing: `execute_hook` falls back to the working directory of +/// the process, which for a bench binary is the Symposium workspace itself. +fn pre_tool_use_payload(project: &Path) -> Result { + let project = project + .to_str() + .with_context(|| format!("staged project path is not UTF-8: {}", project.display()))?; + let payload = json!({ + "hook_event_name": "PreToolUse", + "tool_name": "Bash", + "cwd": project, + "session_id": "benchmark", + "tool_input": { "command": "true" }, + }); + + serde_json::to_string(&payload).context("serializing the PreToolUse payload") +} + +fn benchmark_hook_dispatch(criterion: &mut Criterion) { + let minimal = HookDispatchWorkload::prepare(HookDispatchScenario::Minimal) + .expect("preparing the minimal hook dispatch workload"); + let local_registry = HookDispatchWorkload::prepare(HookDispatchScenario::LocalRegistry) + .expect("preparing the local-registry hook dispatch workload"); + let mut group = criterion.benchmark_group("hook_dispatch"); + + // Hook dispatch is subprocess-bound, so use Criterion's minimum sample + // count and collect stability data across runs in the pinned environment. + // Flat sampling is explicit because Auto can switch modes as warm-up + // latency changes, making otherwise comparable runs use different + // statistics and allowing linear sampling to overshoot the time target. + group.sample_size(10); + group.measurement_time(Duration::from_secs(15)); + group.sampling_mode(SamplingMode::Flat); + group.bench_function("pre_tool_use_minimal_config", |bencher| { + bencher.iter(|| { + let output = black_box(&minimal) + .dispatch() + .expect("the timed minimal hook dispatch failed"); + black_box(output); + }); + }); + group.bench_function("pre_tool_use_local_registry", |bencher| { + bencher.iter(|| { + let output = black_box(&local_registry) + .dispatch() + .expect("the timed local-registry hook dispatch failed"); + black_box(output); + }); + }); + group.finish(); +} + +criterion_group!(benches, benchmark_hook_dispatch); +criterion_main!(benches); diff --git a/benches/benchsuite/benches/workspace_deps.rs b/benches/benchsuite/benches/workspace_deps.rs new file mode 100644 index 00000000..8af29301 --- /dev/null +++ b/benches/benchsuite/benches/workspace_deps.rs @@ -0,0 +1,216 @@ +//! Workspace dependency resolution benchmarks. +//! +//! # `symposium_cache_miss` contract +//! +//! - **Claim:** A Symposium workspace-cache miss measures the complete work +//! needed to resolve and persist dependency metadata for the reference +//! project. Most of the measured time belongs to Cargo, so this quantifies +//! the work avoided by a valid Symposium cache rather than Symposium's own +//! overhead. +//! - **Workload:** A staged copy of the checked-in reference project, resolved +//! through isolated Symposium configuration and cache directories. +//! - **Timed operation:** `WorkspaceDeps::load()`, including workspace lookup, +//! Cargo metadata, result construction, and cache write-through. +//! - **Excluded setup:** Fixture staging, sandbox construction, workspace graph +//! validation, workspace-cache removal, and resolver construction. +//! - **Invariants:** The staged graph has exactly the promised members and path +//! dependencies; per-iteration setup establishes an empty workspace cache +//! before every sample; loading must succeed rather than becoming a false +//! fast sample. +//! - **Metric:** Wall-clock time per `WorkspaceDeps::load()` call. +//! - **Noise:** Cargo subprocess startup, operating-system filesystem caches, +//! process scheduling, shared-runner hardware, and developer-level Cargo +//! configuration during local runs. This is a Symposium cache miss, not a +//! fully cold machine load. +//! - **Lifecycle:** Experimental. +//! +//! # `new_resolver_disk_cache_hit` contract +//! +//! - **Claim:** A new resolver with a valid Symposium disk cache measures the +//! recurring component cost paid when dependency metadata can be reused. The +//! result is normally dominated by `cargo locate-project`; it is not a direct +//! benchmark of JSON deserialization. +//! - **Workload:** A staged copy of the checked-in reference project with a +//! populated and validated Symposium workspace cache. +//! - **Timed operation:** `WorkspaceDeps::load()`, including workspace lookup, +//! cache validation, file reading, deserialization, and memoization. +//! - **Excluded setup:** Fixture staging, sandbox construction, initial Cargo +//! metadata resolution, workspace validation, cache-hit preflight, and new +//! resolver construction. +//! - **Invariants:** Every sample receives a new resolver with an empty +//! in-memory cache; an untimed metadata-rejecting Cargo preflight proves the +//! disk cache is used; loading must succeed; the fixture lockfile is unchanged. +//! - **Metric:** Wall-clock time per `WorkspaceDeps::load()` call. +//! - **Noise:** Cargo subprocess startup, filesystem and operating-system +//! caches, process scheduling, shared-runner hardware, and developer-level +//! Cargo configuration during local runs. +//! - **Lifecycle:** Experimental. + +use std::{hint::black_box, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use criterion::{BatchSize, Criterion, SamplingMode, criterion_group, criterion_main}; + +use symposium::{dirs::SymposiumDirs, pm::WorkspaceDeps}; +use symposium_benchsuite::{Fixture, MetadataRejectingCargo, Sandbox, StagedFixture}; + +struct WorkspaceDepsWorkload { + sandbox: Sandbox, + project: StagedFixture, + dirs: SymposiumDirs, + guarded_cargo: MetadataRejectingCargo, +} + +impl WorkspaceDepsWorkload { + fn prepare() -> Result { + let sandbox = Sandbox::new()?; + let project = sandbox.stage(Fixture::ReferenceProject)?; + let dirs = SymposiumDirs::new( + sandbox.config_dir().to_path_buf(), + sandbox.cache_dir().to_path_buf(), + None, + ); + let guarded_cargo = MetadataRejectingCargo::create_in(sandbox.root())?; + let workload = Self { + sandbox, + project, + dirs, + guarded_cargo, + }; + + workload.resolve_and_check_workspace()?; + workload.verify_cache_reset()?; + // Cache-reset verification deliberately leaves the cache empty. Rebuild + // it so the cache-hit case starts from a validated, populated state. + workload.resolve_and_check_workspace()?; + + Ok(workload) + } + + fn resolve_and_check_workspace(&self) -> Result<()> { + let resolver = self.dirs.workspace_deps(self.project.path()); + let workspace = resolver.load().with_context(|| { + format!( + "resolving staged benchmark workspace `{}`", + self.project.path().display() + ) + })?; + + self.project.check_workspace(workspace) + } + + /// Prove that the sandbox clears the cache location `WorkspaceDeps` uses. + /// + /// The directory name is duplicated across the two crates. If Symposium + /// changes it without updating the benchsuite, cache clearing would become + /// a no-op and silently turn iterations after the first into cache hits. + fn verify_cache_reset(&self) -> Result<()> { + ensure!( + self.cache_contains_entries()?, + "the validating load wrote no cache entry under `{}`", + self.sandbox.cache_dir().display() + ); + + self.sandbox.clear_workspace_cache()?; + + ensure!( + !self.cache_contains_entries()?, + "workspace cache reset left entries under `{}`", + self.sandbox.cache_dir().display() + ); + + Ok(()) + } + + fn cache_contains_entries(&self) -> Result { + let cache_dir = self.sandbox.cache_dir(); + let first_entry = cache_dir + .read_dir() + .with_context(|| format!("reading benchmark cache `{}`", cache_dir.display()))? + .next() + .transpose() + .with_context(|| format!("reading an entry in `{}`", cache_dir.display()))?; + + Ok(first_entry.is_some()) + } + + fn verify_disk_cache_hit(&self) -> Result<()> { + let guarded_dirs = SymposiumDirs::new( + self.sandbox.config_dir().to_path_buf(), + self.sandbox.cache_dir().to_path_buf(), + Some(self.guarded_cargo.executable().to_path_buf()), + ); + let resolver = guarded_dirs.workspace_deps(self.project.path()); + let workspace = resolver.load().context( + "loading the prepared disk cache through metadata-rejecting Cargo; \ + the cache lookup or workspace lookup missed", + )?; + + self.project.check_workspace(workspace) + } + + fn cache_miss_resolver(&self) -> Result { + self.sandbox.clear_workspace_cache()?; + Ok(self.dirs.workspace_deps(self.project.path())) + } + + fn disk_cache_hit_resolver(&self) -> WorkspaceDeps { + self.dirs.workspace_deps(self.project.path()) + } +} + +fn load_for_measurement(resolver: WorkspaceDeps) { + let resolver = black_box(resolver); + let workspace = resolver + .load() + .expect("workspace resolution failed during measurement"); + black_box(workspace); +} + +fn benchmark_workspace_deps(criterion: &mut Criterion) { + let workload = WorkspaceDepsWorkload::prepare() + .expect("preparing the workspace dependency benchmark workload"); + let mut group = criterion.benchmark_group("workspace_deps"); + + // These subprocess-bound cases can take more than a second per iteration. + // Criterion's minimum sample count keeps each run bounded; stability comes + // from repeated runs in the pinned benchmark environment. + group.sample_size(10); + group.measurement_time(Duration::from_secs(15)); + // Cache misses are long-running, so flat sampling bounds the work. Making + // the mode explicit also prevents latency drift from changing the model. + group.sampling_mode(SamplingMode::Flat); + group.bench_function("symposium_cache_miss", |bencher| { + bencher.iter_batched( + || { + workload + .cache_miss_resolver() + .expect("preparing a Symposium workspace-cache miss") + }, + load_for_measurement, + // PerIteration is load-bearing, not a memory choice: setup clears the + // workspace cache, and a larger batch runs every setup before timing + // the batch, so only the first iteration would be a cache miss. + BatchSize::PerIteration, + ); + }); + workload + .verify_disk_cache_hit() + .expect("the disk cache must be valid before measuring cache hits"); + // Cache hits are short enough for linear sampling, which retains + // Criterion's per-iteration regression model at an affordable cost. + group.sampling_mode(SamplingMode::Linear); + group.bench_function("new_resolver_disk_cache_hit", |bencher| { + bencher.iter_batched( + || workload.disk_cache_hit_resolver(), + load_for_measurement, + // Setup only creates independent resolvers; every timed load reads + // the same immutable disk cache, so batching cannot change the path. + BatchSize::SmallInput, + ); + }); + group.finish(); +} + +criterion_group!(benches, benchmark_workspace_deps); +criterion_main!(benches); diff --git a/benches/benchsuite/src/cargo.rs b/benches/benchsuite/src/cargo.rs new file mode 100644 index 00000000..06c08942 --- /dev/null +++ b/benches/benchsuite/src/cargo.rs @@ -0,0 +1,146 @@ +//! Cargo process guards used by benchmark preflights. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result}; +#[cfg(not(windows))] +use indoc::indoc; + +/// Marker written beside the guard the first time `metadata` is refused. +const MARKER_FILE: &str = "metadata-attempted"; + +#[cfg(not(windows))] +const METADATA_REJECTING_SCRIPT: &str = indoc! {r#" + #!/bin/sh + if [ "$1" = "metadata" ]; then + : > "${0%/*}/metadata-attempted" + exit 1 + fi + exec cargo "$@" +"#}; + +// A `goto` rather than a parenthesised block: batch parses blocks eagerly. +#[cfg(windows)] +const METADATA_REJECTING_SCRIPT: &str = concat!( + "@echo off\r\n", + "if not \"%~1\"==\"metadata\" goto forward\r\n", + "type nul > \"%~dp0metadata-attempted\"\r\n", + "exit /b 1\r\n", + ":forward\r\n", + "cargo %*\r\n", +); + +/// A Cargo executable that forwards every command except `metadata`, which it +/// refuses while recording the attempt. +/// +/// Refusing is not itself an assertion: `WorkspaceDeps` turns a failed +/// `metadata` into "no workspace", which callers accept. Preflights have to +/// check [`saw_metadata`](Self::saw_metadata). +#[derive(Debug)] +pub struct MetadataRejectingCargo { + executable: PathBuf, + marker: PathBuf, +} + +impl MetadataRejectingCargo { + /// Create the guard under `parent`, which must not already contain one. + pub fn create_in(parent: &Path) -> Result { + let directory = parent.join("metadata-rejecting-cargo"); + fs::create_dir(&directory).with_context(|| { + format!( + "creating metadata-rejecting Cargo directory `{}`", + directory.display() + ) + })?; + let executable = write_executable(&directory)?; + + Ok(Self { + executable, + marker: directory.join(MARKER_FILE), + }) + } + + pub fn executable(&self) -> &Path { + &self.executable + } + + /// Whether the guard has been asked to run `cargo metadata`. + pub fn saw_metadata(&self) -> Result { + self.marker.try_exists().with_context(|| { + format!( + "checking the Cargo guard marker `{}`", + self.marker.display() + ) + }) + } +} + +#[cfg(not(windows))] +fn write_executable(directory: &Path) -> Result { + use std::os::unix::fs::PermissionsExt; + + let executable = directory.join("cargo"); + fs::write(&executable, METADATA_REJECTING_SCRIPT) + .with_context(|| format!("writing Cargo guard `{}`", executable.display()))?; + fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)) + .with_context(|| format!("making Cargo guard executable `{}`", executable.display()))?; + + Ok(executable) +} + +#[cfg(windows)] +fn write_executable(directory: &Path) -> Result { + let executable = directory.join("cargo.cmd"); + fs::write(&executable, METADATA_REJECTING_SCRIPT) + .with_context(|| format!("writing Cargo guard shim `{}`", executable.display()))?; + + Ok(executable) +} + +#[cfg(test)] +mod tests { + use std::process::Command; + + use anyhow::ensure; + use tempfile::tempdir; + + use super::*; + + #[test] + fn forwards_other_commands_and_rejects_metadata() -> Result<()> { + let temporary_directory = tempdir()?; + let cargo = MetadataRejectingCargo::create_in(temporary_directory.path())?; + + let forwarded = Command::new(cargo.executable()) + .arg("--version") + .output() + .context("running a forwarded Cargo command")?; + ensure!( + forwarded.status.success(), + "Cargo guard did not forward `--version`: {}", + String::from_utf8_lossy(&forwarded.stderr) + ); + ensure!( + !cargo.saw_metadata()?, + "forwarding a command must not record a metadata attempt" + ); + + let rejected = Command::new(cargo.executable()) + .arg("metadata") + .status() + .context("running rejected Cargo metadata")?; + ensure!( + !rejected.success(), + "Cargo guard unexpectedly allowed `metadata`" + ); + ensure!( + cargo.saw_metadata()?, + "refusing `metadata` must record the attempt" + ); + + Ok(()) + } +} diff --git a/benches/benchsuite/src/fixture.rs b/benches/benchsuite/src/fixture.rs new file mode 100644 index 00000000..ce80b254 --- /dev/null +++ b/benches/benchsuite/src/fixture.rs @@ -0,0 +1,502 @@ +//! Checked-in benchmark fixtures and their validation. + +use anyhow::{Context, Result, bail, ensure}; +use std::{ + ffi::OsStr, + fs, + path::{Path, PathBuf}, + sync::LazyLock, +}; +use symposium::pm::{LoadedWorkspace, WorkspaceCrate}; + +static FIXTURES_ROOT: LazyLock = LazyLock::new(|| { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("benchsuite must live inside the benches directory") + .join("fixtures") +}); + +#[derive(Debug, Clone, Copy)] +struct WorkspaceShape { + members: &'static [&'static str], + path_dependencies: &'static [&'static str], +} + +impl WorkspaceShape { + fn check_members(&self, root: &Path, members: &[PathBuf]) -> Result<()> { + let names = members + .iter() + .map(|member| leaf_name(member)) + .collect::>>()?; + check_names("workspace members", self.members, &names)?; + + for (member, name) in members.iter().zip(names) { + let expected = canonicalize(&root.join(name))?; + + ensure!( + canonicalize(member)? == expected, + "workspace member `{name}` resolved outside the fixture: expected `{}`, found `{}`", + expected.display(), + member.display() + ); + } + + Ok(()) + } + + fn check_dependencies(&self, root: &Path, dependencies: &[WorkspaceCrate]) -> Result<()> { + let names: Vec<_> = dependencies + .iter() + .map(|dependency| dependency.name.as_str()) + .collect(); + check_names("direct dependencies", self.path_dependencies, &names)?; + + for dependency in dependencies { + check_path_dependency(root, dependency)?; + } + + Ok(()) + } +} + +#[derive(Debug)] +struct FixtureSpec { + directory_name: &'static str, + required_files: &'static [&'static str], + workspace_shape: Option, +} + +const REFERENCE_PROJECT_SPEC: FixtureSpec = FixtureSpec { + directory_name: "reference-project", + required_files: &["Cargo.toml", "Cargo.lock", ".cargo/config.toml"], + workspace_shape: Some(WorkspaceShape { + members: &["cli", "server"], + path_dependencies: &["domain", "storage", "terminal"], + }), +}; + +const LOCAL_REGISTRY_SPEC: FixtureSpec = FixtureSpec { + directory_name: "local-registry", + required_files: &[ + "always-active/SYMPOSIUM.toml", + "predicate-gated/SYMPOSIUM.toml", + "predicate-gated/unexpected-hook.sh", + "dormant/SYMPOSIUM.toml", + ], + workspace_shape: None, +}; + +/// A checked-in benchmark workload under `benches/fixtures`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Fixture { + ReferenceProject, + LocalRegistry, +} + +impl Fixture { + const fn spec(self) -> &'static FixtureSpec { + match self { + Self::ReferenceProject => &REFERENCE_PROJECT_SPEC, + Self::LocalRegistry => &LOCAL_REGISTRY_SPEC, + } + } + + pub(crate) const fn directory_name(self) -> &'static str { + self.spec().directory_name + } + + fn source_dir(self) -> Result { + let path = FIXTURES_ROOT.join(self.directory_name()); + + ensure!( + path.is_dir(), + "benchmark fixture `{}` is missing: {}", + self.directory_name(), + path.display() + ); + + Ok(path) + } + + /// Copy the fixture into `destination`, which must not already exist. + fn copy_to(self, destination: impl AsRef) -> Result<()> { + copy_directory(&self.source_dir()?, destination.as_ref()) + } +} + +/// A fixture copied into a [`crate::Sandbox`], with its layout validated, so a +/// benchmark cannot time a workload that is missing checked-in files. +#[derive(Debug)] +pub struct StagedFixture { + fixture: Fixture, + path: PathBuf, +} + +impl StagedFixture { + pub(crate) fn stage(fixture: Fixture, path: PathBuf) -> Result { + fixture.copy_to(&path)?; + let staged = Self { fixture, path }; + staged.check_layout()?; + Ok(staged) + } + + pub fn fixture(&self) -> Fixture { + self.fixture + } + + pub fn path(&self) -> &Path { + &self.path + } + + fn check_layout(&self) -> Result<()> { + for required in self.fixture.spec().required_files { + let path = required + .split('/') + .fold(self.path.clone(), |path, part| path.join(part)); + + ensure!( + path.is_file(), + "staged fixture `{}` is missing `{required}`: {}", + self.fixture.directory_name(), + path.display() + ); + } + + Ok(()) + } + + /// Check a resolved Cargo graph against the shape this fixture promises. + pub fn check_workspace(&self, workspace: &LoadedWorkspace) -> Result<()> { + let Some(shape) = self.fixture.spec().workspace_shape else { + bail!( + "fixture `{}` is not a Cargo project", + self.fixture.directory_name() + ); + }; + + let root = canonicalize(&self.path)?; + ensure!( + canonicalize(&workspace.root)? == root, + "workspace root mismatch: expected `{}`, found `{}`", + root.display(), + workspace.root.display() + ); + + shape.check_members(&root, &workspace.members)?; + shape.check_dependencies(&root, &workspace.crates) + } +} + +fn check_path_dependency(root: &Path, dependency: &WorkspaceCrate) -> Result<()> { + // The fixture is hermetic only while every dependency resolves inside it. + let Some(path) = dependency.path.as_deref() else { + bail!( + "dependency `{}` is not a local path dependency", + dependency.name + ); + }; + let expected = canonicalize(&root.join(&dependency.name))?; + + ensure!( + canonicalize(path)? == expected, + "dependency `{}` resolved outside the fixture: expected `{}`, found `{}`", + dependency.name, + expected.display(), + path.display() + ); + + let Some(source_dir) = dependency.source_dir.as_deref() else { + bail!("dependency `{}` has no source directory", dependency.name); + }; + + ensure!( + canonicalize(source_dir)? == expected, + "dependency `{}` source directory resolved outside the fixture: expected `{}`, found `{}`", + dependency.name, + expected.display(), + source_dir.display() + ); + + Ok(()) +} + +/// Compare two name lists order-insensitively. Sorted vectors rather than sets, +/// so a duplicated name fails instead of being absorbed. +fn check_names(kind: &str, expected: &[&str], actual: &[&str]) -> Result<()> { + let mut expected = expected.to_vec(); + let mut actual = actual.to_vec(); + expected.sort_unstable(); + actual.sort_unstable(); + + ensure!( + expected == actual, + "{kind} mismatch: expected [{}], found [{}]", + expected.join(", "), + actual.join(", ") + ); + + Ok(()) +} + +fn leaf_name(path: &Path) -> Result<&str> { + let Some(name) = path.file_name().and_then(OsStr::to_str) else { + bail!("path has no usable directory name: {}", path.display()); + }; + + Ok(name) +} + +fn canonicalize(path: &Path) -> Result { + fs::canonicalize(path).with_context(|| format!("canonicalizing `{}`", path.display())) +} + +fn copy_directory(source: &Path, destination: &Path) -> Result<()> { + fs::create_dir(destination).with_context(|| { + format!( + "creating fixture destination directory `{}`", + destination.display() + ) + })?; + + for entry in source + .read_dir() + .with_context(|| format!("reading fixture directory `{}`", source.display()))? + { + let entry = entry.with_context(|| format!("reading an entry in `{}`", source.display()))?; + copy_entry(&entry, destination)?; + } + + Ok(()) +} + +fn copy_entry(entry: &fs::DirEntry, destination_directory: &Path) -> Result<()> { + let source = entry.path(); + let destination = destination_directory.join(entry.file_name()); + let file_type = entry + .file_type() + .with_context(|| format!("reading file type for `{}`", source.display()))?; + + if file_type.is_dir() { + copy_directory(&source, &destination) + } else if file_type.is_file() { + fs::copy(&source, &destination).with_context(|| { + format!( + "copying fixture file `{}` to `{}`", + source.display(), + destination.display() + ) + })?; + Ok(()) + } else { + bail!( + "fixture contains an unsupported filesystem entry: {}", + source.display() + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Sandbox; + use symposium::dirs::SymposiumDirs; + use tempfile::tempdir; + + fn resolve_reference_project() -> Result<(Sandbox, StagedFixture, LoadedWorkspace)> { + let sandbox = Sandbox::new()?; + let project = sandbox.stage(Fixture::ReferenceProject)?; + let dirs = SymposiumDirs::new( + sandbox.config_dir().to_path_buf(), + sandbox.cache_dir().to_path_buf(), + None, + ); + let workspace = dirs + .workspace_deps(project.path()) + .load() + .context("resolving the staged reference project")? + .as_ref() + .clone(); + + Ok((sandbox, project, workspace)) + } + + #[test] + fn finds_reference_project_fixture() -> Result<()> { + let source_dir = Fixture::ReferenceProject.source_dir()?; + + assert!(source_dir.join("Cargo.toml").is_file()); + + Ok(()) + } + + #[test] + fn finds_local_registry_fixture() -> Result<()> { + let source_dir = Fixture::LocalRegistry.source_dir()?; + + assert!( + source_dir + .join("always-active") + .join("SYMPOSIUM.toml") + .is_file() + ); + + Ok(()) + } + + #[test] + fn copies_reference_project_fixture() -> Result<()> { + let temporary_directory = tempdir()?; + let destination = temporary_directory.path().join("reference-project"); + + Fixture::ReferenceProject.copy_to(&destination)?; + + assert!(destination.join("Cargo.toml").is_file()); + assert!(destination.join("domain/src/lib.rs").is_file()); + + Ok(()) + } + + #[test] + fn refuses_to_merge_into_an_existing_destination() -> Result<()> { + let temporary_directory = tempdir()?; + let destination = temporary_directory.path().join("reference-project"); + let sentinel = destination.join("sentinel"); + + fs::create_dir(&destination)?; + fs::write(&sentinel, "leave me untouched")?; + + let error = Fixture::ReferenceProject + .copy_to(&destination) + .expect_err("copying into an existing destination must fail"); + + assert!( + error + .to_string() + .contains(&destination.display().to_string()), + "error does not name destination `{}`: {error:#}", + destination.display() + ); + assert_eq!(fs::read_to_string(sentinel)?, "leave me untouched"); + assert!(!destination.join("Cargo.toml").try_exists()?); + + Ok(()) + } + + #[test] + fn staging_checks_the_promised_layout() -> Result<()> { + let sandbox = Sandbox::new()?; + let project = sandbox.stage(Fixture::ReferenceProject)?; + + project.check_layout()?; + fs::remove_file(project.path().join("Cargo.lock"))?; + + let error = project + .check_layout() + .expect_err("a missing required file must fail validation"); + assert!(error.to_string().contains("Cargo.lock")); + + Ok(()) + } + + #[test] + fn accepts_the_resolved_reference_project() -> Result<()> { + let (_sandbox, project, workspace) = resolve_reference_project()?; + + project.check_workspace(&workspace) + } + + #[test] + fn rejects_a_missing_workspace_member() -> Result<()> { + let (_sandbox, project, mut workspace) = resolve_reference_project()?; + + workspace.members.pop(); + + let error = project + .check_workspace(&workspace) + .expect_err("a missing workspace member must fail validation"); + assert!(error.to_string().contains("workspace members mismatch")); + + Ok(()) + } + + #[test] + fn rejects_a_workspace_member_outside_the_fixture() -> Result<()> { + let (sandbox, project, mut workspace) = resolve_reference_project()?; + let member_name = workspace.members[0] + .file_name() + .context("fixture member must have a directory name")?; + let outside_member = sandbox.config_dir().join(member_name); + + fs::create_dir(&outside_member)?; + workspace.members[0] = outside_member; + + let error = project + .check_workspace(&workspace) + .expect_err("a workspace member outside the fixture must fail validation"); + assert!(error.to_string().contains("resolved outside the fixture")); + + Ok(()) + } + + #[test] + fn rejects_a_missing_dependency() -> Result<()> { + let (_sandbox, project, mut workspace) = resolve_reference_project()?; + + workspace.crates.pop(); + + let error = project + .check_workspace(&workspace) + .expect_err("a missing direct dependency must fail validation"); + assert!(error.to_string().contains("direct dependencies mismatch")); + + Ok(()) + } + + #[test] + fn rejects_a_non_path_dependency() -> Result<()> { + let (_sandbox, project, mut workspace) = resolve_reference_project()?; + + workspace.crates[0].path = None; + + let error = project + .check_workspace(&workspace) + .expect_err("a registry dependency must fail validation"); + assert!(error.to_string().contains("not a local path dependency")); + + Ok(()) + } + + #[test] + fn rejects_a_dependency_source_outside_the_fixture() -> Result<()> { + let (sandbox, project, mut workspace) = resolve_reference_project()?; + let outside_source = sandbox.config_dir().join(&workspace.crates[0].name); + + fs::create_dir(&outside_source)?; + workspace.crates[0].source_dir = Some(outside_source); + + let error = project + .check_workspace(&workspace) + .expect_err("a dependency source outside the fixture must fail validation"); + assert!( + error + .to_string() + .contains("source directory resolved outside the fixture") + ); + + Ok(()) + } + + #[test] + fn rejects_a_workspace_check_on_a_registry_fixture() -> Result<()> { + let (_sandbox, _project, workspace) = resolve_reference_project()?; + let registry_sandbox = Sandbox::new()?; + let registry = registry_sandbox.stage(Fixture::LocalRegistry)?; + + let error = registry + .check_workspace(&workspace) + .expect_err("the registry fixture is not a Cargo project"); + assert!(error.to_string().contains("not a Cargo project")); + + Ok(()) + } +} diff --git a/benches/benchsuite/src/lib.rs b/benches/benchsuite/src/lib.rs new file mode 100644 index 00000000..841ba0ad --- /dev/null +++ b/benches/benchsuite/src/lib.rs @@ -0,0 +1,9 @@ +//! Shared fixture and sandbox support for Symposium benchmarks. + +mod cargo; +mod fixture; +mod sandbox; + +pub use cargo::MetadataRejectingCargo; +pub use fixture::{Fixture, StagedFixture}; +pub use sandbox::Sandbox; diff --git a/benches/benchsuite/src/sandbox.rs b/benches/benchsuite/src/sandbox.rs new file mode 100644 index 00000000..a0a6eb5b --- /dev/null +++ b/benches/benchsuite/src/sandbox.rs @@ -0,0 +1,200 @@ +//! Isolated filesystem state for benchmark workloads. + +use std::{ + fs::{self, File}, + io::Write, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result}; +use tempfile::{Builder, TempDir}; + +use crate::fixture::{Fixture, StagedFixture}; + +#[derive(Debug)] +pub struct Sandbox { + root: TempDir, + config_dir: PathBuf, + cache_dir: PathBuf, +} + +impl Sandbox { + pub fn new() -> Result { + let root = Builder::new() + .prefix("symposium-benchmark-") + .tempdir() + .context("creating benchmark sandbox")?; + let config_dir = root.path().join("symposium-home"); + let cache_dir = config_dir.join("cache"); + + fs::create_dir_all(&cache_dir).with_context(|| { + format!( + "creating benchmark sandbox directories under `{}`", + root.path().display() + ) + })?; + + Ok(Self { + root, + config_dir, + cache_dir, + }) + } + + /// Copy `fixture` into the sandbox and validate its layout. + pub fn stage(&self, fixture: Fixture) -> Result { + StagedFixture::stage(fixture, self.root().join(fixture.directory_name())) + } + + /// Write the sandbox configuration, refusing to replace an existing file. + pub fn write_config(&self, contents: &str) -> Result<()> { + let path = self.config_dir.join("config.toml"); + let mut file = File::create_new(&path) + .with_context(|| format!("creating benchmark configuration `{}`", path.display()))?; + + file.write_all(contents.as_bytes()) + .with_context(|| format!("writing benchmark configuration `{}`", path.display())) + } + + /// Remove the sandbox's workspace dependency caches. + pub fn clear_workspace_cache(&self) -> Result<()> { + let workspace_cache = self.cache_dir.join("workspaces"); + + match fs::remove_dir_all(&workspace_cache) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| { + format!( + "removing benchmark workspace cache `{}`", + workspace_cache.display() + ) + }), + } + } + + pub fn root(&self) -> &Path { + self.root.path() + } + + pub fn config_dir(&self) -> &Path { + &self.config_dir + } + + pub fn cache_dir(&self) -> &Path { + &self.cache_dir + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn creates_isolated_sandbox_directories() -> Result<()> { + let sandbox = Sandbox::new()?; + + assert!(sandbox.root().is_dir()); + assert!(sandbox.config_dir().is_dir()); + assert!(sandbox.cache_dir().is_dir()); + assert_eq!(sandbox.config_dir().parent(), Some(sandbox.root())); + assert_eq!(sandbox.cache_dir().parent(), Some(sandbox.config_dir())); + + Ok(()) + } + + #[test] + fn stages_only_the_requested_fixture() -> Result<()> { + let sandbox = Sandbox::new()?; + + let project = sandbox.stage(Fixture::ReferenceProject)?; + + assert_eq!(project.path(), sandbox.root().join("reference-project")); + assert_eq!(project.fixture(), Fixture::ReferenceProject); + assert!(!sandbox.root().join("local-registry").try_exists()?); + + Ok(()) + } + + #[test] + fn refuses_to_stage_the_same_fixture_twice() -> Result<()> { + let sandbox = Sandbox::new()?; + + sandbox.stage(Fixture::LocalRegistry)?; + sandbox + .stage(Fixture::LocalRegistry) + .expect_err("staging the same fixture twice must fail"); + + Ok(()) + } + + #[test] + fn writes_configuration_contents() -> Result<()> { + let sandbox = Sandbox::new()?; + + sandbox.write_config("benchmark configuration")?; + + assert_eq!( + fs::read_to_string(sandbox.config_dir().join("config.toml"))?, + "benchmark configuration" + ); + + Ok(()) + } + + #[test] + fn refuses_to_overwrite_configuration() -> Result<()> { + let sandbox = Sandbox::new()?; + let config_file = sandbox.config_dir().join("config.toml"); + sandbox.write_config("original configuration")?; + + let error = sandbox + .write_config("replacement configuration") + .expect_err("writing configuration twice must fail"); + + assert!( + error + .to_string() + .contains(&config_file.display().to_string()), + "error should identify the existing configuration: {error:#}" + ); + assert_eq!(fs::read_to_string(config_file)?, "original configuration"); + + Ok(()) + } + + #[test] + fn clears_only_the_workspace_cache() -> Result<()> { + let sandbox = Sandbox::new()?; + let project = sandbox.stage(Fixture::ReferenceProject)?; + let config_file = sandbox.config_dir().join("config.toml"); + let workspace_cache = sandbox + .cache_dir() + .join("workspaces") + .join("reference-project"); + let binary_cache = sandbox + .cache_dir() + .join("binaries") + .join("example") + .join("1.0.0"); + + fs::write(&config_file, "benchmark configuration")?; + fs::create_dir_all(&workspace_cache)?; + fs::write(workspace_cache.join("workspace-deps.json"), "cached data")?; + fs::create_dir_all(&binary_cache)?; + fs::write(binary_cache.join("example"), "cached binary")?; + + sandbox.clear_workspace_cache()?; + sandbox.clear_workspace_cache()?; + + assert!(sandbox.cache_dir().is_dir()); + assert!(!workspace_cache.try_exists()?); + assert_eq!( + fs::read_to_string(binary_cache.join("example"))?, + "cached binary" + ); + assert!(project.path().join("Cargo.toml").is_file()); + assert_eq!(fs::read_to_string(config_file)?, "benchmark configuration"); + + Ok(()) + } +} diff --git a/benches/fixtures/README.md b/benches/fixtures/README.md new file mode 100644 index 00000000..7d93116b --- /dev/null +++ b/benches/fixtures/README.md @@ -0,0 +1,38 @@ +# Benchmark fixtures + +These checked-in fixtures are deterministic workloads composed by the benchmark targets. Support code copies them into an isolated sandbox and validates their invariants before starting a timed operation. + +## `config` + +`config` contains production-valid Symposium configurations selected by the +hook-dispatch scenarios. Both configurations disable the builtin registries. +The local-registry configuration adds the staged `local-registry` fixture by a +path relative to the sandbox's `symposium-home` configuration directory, where +the harness writes the selected file. + +## `reference-project` + +`reference-project` is a virtual Cargo workspace used by the workspace-dependency and hook-dispatch benchmarks. It has these invariants: + +- its workspace members are exactly `cli` and `server`; +- its local path dependencies are exactly `domain`, `terminal`, and `storage`; +- `domain` is a direct dependency of both members, `terminal` belongs only to `cli`, and `storage` belongs only to `server`; +- `Cargo.lock` is committed and `.cargo/config.toml` enables offline mode; +- neither the workspace root nor a member defines a workspace plugin through `SYMPOSIUM.toml`, `skills/`, or `.agents/skills/`. + +The three dependency packages contain empty `[workspace]` tables so Cargo does not associate them with Symposium's outer workspace. + +## `local-registry` + +`local-registry` is a path registry used by the representative hook-dispatch benchmark. It has exactly three manifest-backed entries: + +- `always-active` uses `depends-on = ["*"]`; +- `predicate-gated` is active but its `PreToolUse` hook is disabled by `path_exists(./.symposium-benchmark-never-present)`; +- `dormant` has no activation gate. + +Every entry contains `SYMPOSIUM.toml`. The predicate-gated entry's +`unexpected-hook.sh` is required fixture data, so the disabled hook always has +a valid command behind it. The registry contains no bare `SKILL.md` entry. This +keeps `src/skills.rs` outside the measured hook path. + +Before measuring, the harness must verify the project graph, assert that `.symposium-benchmark-never-present` is absent from the benchmark process's current working directory, and require the loaded plugin names to be exactly `always-active`, `predicate-gated`, and `dormant`. Cargo sets that working directory to the `benches/benchsuite` package root. A missing or malformed entry is a setup failure, never a faster sample. diff --git a/benches/fixtures/config/local-registry.toml b/benches/fixtures/config/local-registry.toml new file mode 100644 index 00000000..0be57a9a --- /dev/null +++ b/benches/fixtures/config/local-registry.toml @@ -0,0 +1,7 @@ +[defaults] +symposium-recommendations = false +user-plugins = false + +[[registry]] +name = "benchmark-local" +path = "../local-registry" diff --git a/benches/fixtures/config/minimal.toml b/benches/fixtures/config/minimal.toml new file mode 100644 index 00000000..a0baead7 --- /dev/null +++ b/benches/fixtures/config/minimal.toml @@ -0,0 +1,3 @@ +[defaults] +symposium-recommendations = false +user-plugins = false diff --git a/benches/fixtures/local-registry/always-active/SYMPOSIUM.toml b/benches/fixtures/local-registry/always-active/SYMPOSIUM.toml new file mode 100644 index 00000000..5cf3038f --- /dev/null +++ b/benches/fixtures/local-registry/always-active/SYMPOSIUM.toml @@ -0,0 +1,2 @@ +name = "always-active" +depends-on = ["*"] diff --git a/benches/fixtures/local-registry/dormant/SYMPOSIUM.toml b/benches/fixtures/local-registry/dormant/SYMPOSIUM.toml new file mode 100644 index 00000000..a3100dd1 --- /dev/null +++ b/benches/fixtures/local-registry/dormant/SYMPOSIUM.toml @@ -0,0 +1 @@ +name = "dormant" diff --git a/benches/fixtures/local-registry/predicate-gated/SYMPOSIUM.toml b/benches/fixtures/local-registry/predicate-gated/SYMPOSIUM.toml new file mode 100644 index 00000000..75287e9f --- /dev/null +++ b/benches/fixtures/local-registry/predicate-gated/SYMPOSIUM.toml @@ -0,0 +1,8 @@ +name = "predicate-gated" +depends-on = ["*"] + +[[hooks]] +name = "never-runs" +event = "PreToolUse" +command = { script = "unexpected-hook.sh" } +predicates = ["path_exists(./.symposium-benchmark-never-present)"] diff --git a/benches/fixtures/local-registry/predicate-gated/unexpected-hook.sh b/benches/fixtures/local-registry/predicate-gated/unexpected-hook.sh new file mode 100644 index 00000000..be26d821 --- /dev/null +++ b/benches/fixtures/local-registry/predicate-gated/unexpected-hook.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +printf '%s\n' '{"PreToolUse":{"additionalContext":"unexpected benchmark hook execution"}}' diff --git a/benches/fixtures/reference-project/.cargo/config.toml b/benches/fixtures/reference-project/.cargo/config.toml new file mode 100644 index 00000000..d52f0a8c --- /dev/null +++ b/benches/fixtures/reference-project/.cargo/config.toml @@ -0,0 +1,2 @@ +[net] +offline = true diff --git a/benches/fixtures/reference-project/Cargo.lock b/benches/fixtures/reference-project/Cargo.lock new file mode 100644 index 00000000..9cfb2c7c --- /dev/null +++ b/benches/fixtures/reference-project/Cargo.lock @@ -0,0 +1,31 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "cli" +version = "0.1.0" +dependencies = [ + "domain", + "terminal", +] + +[[package]] +name = "domain" +version = "0.1.0" + +[[package]] +name = "server" +version = "0.1.0" +dependencies = [ + "domain", + "storage", +] + +[[package]] +name = "storage" +version = "0.1.0" + +[[package]] +name = "terminal" +version = "0.1.0" diff --git a/benches/fixtures/reference-project/Cargo.toml b/benches/fixtures/reference-project/Cargo.toml new file mode 100644 index 00000000..ca5e3505 --- /dev/null +++ b/benches/fixtures/reference-project/Cargo.toml @@ -0,0 +1,4 @@ +[workspace] +members = ["cli", "server"] +exclude = ["domain", "terminal", "storage"] +resolver = "3" diff --git a/benches/fixtures/reference-project/cli/Cargo.toml b/benches/fixtures/reference-project/cli/Cargo.toml new file mode 100644 index 00000000..d7d60e0e --- /dev/null +++ b/benches/fixtures/reference-project/cli/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "cli" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +domain = { version = "0.1.0", path = "../domain" } +terminal = { version = "0.1.0", path = "../terminal" } diff --git a/benches/fixtures/reference-project/cli/src/main.rs b/benches/fixtures/reference-project/cli/src/main.rs new file mode 100644 index 00000000..f328e4d9 --- /dev/null +++ b/benches/fixtures/reference-project/cli/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/benches/fixtures/reference-project/domain/Cargo.toml b/benches/fixtures/reference-project/domain/Cargo.toml new file mode 100644 index 00000000..77b41dc9 --- /dev/null +++ b/benches/fixtures/reference-project/domain/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "domain" +version = "0.1.0" +edition = "2024" +publish = false + +[workspace] diff --git a/benches/fixtures/reference-project/domain/src/lib.rs b/benches/fixtures/reference-project/domain/src/lib.rs new file mode 100644 index 00000000..2975d495 --- /dev/null +++ b/benches/fixtures/reference-project/domain/src/lib.rs @@ -0,0 +1 @@ +//! Shared domain types for the reference project fixture. diff --git a/benches/fixtures/reference-project/server/Cargo.toml b/benches/fixtures/reference-project/server/Cargo.toml new file mode 100644 index 00000000..384d5e25 --- /dev/null +++ b/benches/fixtures/reference-project/server/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "server" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +domain = { version = "0.1.0", path = "../domain" } +storage = { version = "0.1.0", path = "../storage" } diff --git a/benches/fixtures/reference-project/server/src/main.rs b/benches/fixtures/reference-project/server/src/main.rs new file mode 100644 index 00000000..f328e4d9 --- /dev/null +++ b/benches/fixtures/reference-project/server/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/benches/fixtures/reference-project/storage/Cargo.toml b/benches/fixtures/reference-project/storage/Cargo.toml new file mode 100644 index 00000000..fcfb3ce4 --- /dev/null +++ b/benches/fixtures/reference-project/storage/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "storage" +version = "0.1.0" +edition = "2024" +publish = false + +[workspace] diff --git a/benches/fixtures/reference-project/storage/src/lib.rs b/benches/fixtures/reference-project/storage/src/lib.rs new file mode 100644 index 00000000..a4d5079d --- /dev/null +++ b/benches/fixtures/reference-project/storage/src/lib.rs @@ -0,0 +1 @@ +//! Storage support for the Atlas server fixture. diff --git a/benches/fixtures/reference-project/terminal/Cargo.toml b/benches/fixtures/reference-project/terminal/Cargo.toml new file mode 100644 index 00000000..b6efa14d --- /dev/null +++ b/benches/fixtures/reference-project/terminal/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "terminal" +version = "0.1.0" +edition = "2024" +publish = false + +[workspace] diff --git a/benches/fixtures/reference-project/terminal/src/lib.rs b/benches/fixtures/reference-project/terminal/src/lib.rs new file mode 100644 index 00000000..65f9c706 --- /dev/null +++ b/benches/fixtures/reference-project/terminal/src/lib.rs @@ -0,0 +1 @@ +//! Terminal support for the Atlas CLI fixture. diff --git a/md/SUMMARY.md b/md/SUMMARY.md index 0ed860b4..ad3baf74 100644 --- a/md/SUMMARY.md +++ b/md/SUMMARY.md @@ -75,6 +75,7 @@ - [`hook`](./design/hook-flow.md) - [Running tests](./design/running-tests.md) - [Writing tests](./design/testing-guidelines.md) + - [Benchmarking](./design/benchmarking.md) - [Governance](./design/governance.md) - [Common issues](./design/common-issues.md) - [Agent details](./design/agent-details/README.md) diff --git a/md/design/benchmarking.md b/md/design/benchmarking.md new file mode 100644 index 00000000..9f139008 --- /dev/null +++ b/md/design/benchmarking.md @@ -0,0 +1,273 @@ +# Benchmarking + +The performance question for the initial benchmark suite is: + +> How much wall-clock time does Symposium's in-process `PreToolUse` pipeline take in an unchanged workspace, with minimal and representative local configurations? + +The first performance story answers that question with two in-process hook cases and two `WorkspaceDeps` component measurements. The component numbers make the hook results interpretable: they show the difference between resolving metadata and reusing Symposium's disk cache. + +The suite begins with this bounded story rather than attempting to benchmark every performance-sensitive path in the same change. + +## Goals + +The benchmark suite is organized so that: + +- each benchmark addition can be built, run, and understood independently; +- the primary measurements represent the in-process portion of a user-visible operation; +- component measurements explain important contributors to that operation; +- fixtures and environment setup can be shared without hiding the operation being measured; +- workloads are deterministic and cannot access the network; +- benchmark names state the cache or workload conditions they control; +- normal pull requests compile the suite, while measurements run separately; +- performance gating is introduced only after a benchmark has a stable and useful history. + +The first story does not measure workspace-size scaling, `SessionStart`, real plugin hook subprocesses, remote registry refresh, networking, or every performance-sensitive module. Those require later, separately justified benchmark additions. + +## Organization + +The suite uses this layout: + +```text +benches/ +|-- README.md +|-- benchsuite/ +| |-- Cargo.toml +| |-- src/ +| | |-- cargo.rs +| | |-- fixture.rs +| | |-- lib.rs +| | `-- sandbox.rs +| `-- benches/ +| |-- hook_dispatch.rs +| `-- workspace_deps.rs +`-- fixtures/ + |-- README.md + |-- reference-project/ + | |-- .cargo/ + | | `-- config.toml + | |-- Cargo.toml + | |-- Cargo.lock + | |-- cli/ + | |-- server/ + | |-- domain/ + | |-- terminal/ + | `-- storage/ + `-- local-registry/ + |-- always-active/ + | `-- SYMPOSIUM.toml + |-- predicate-gated/ + | |-- SYMPOSIUM.toml + | `-- unexpected-hook.sh + `-- dormant/ + `-- SYMPOSIUM.toml +``` + +`benchsuite` is a non-publishable package (`publish = false`) listed explicitly in the root workspace's `members`. Its library owns reusable mechanics: locating and copying checked-in fixtures, creating isolated configuration and cache directories, validating prepared workloads, and constructing the metadata-rejecting Cargo guard used by untimed cache-hit preflights. Fixture metadata is centralized in private typed specifications so its directory, required files, and expected workspace shape cannot drift across separate declarations. The library exports only the fixture, staged-fixture, sandbox, and Cargo-guard capabilities needed by benchmark targets. Individual targets retain semantic ownership of their scenarios and timed operations. + +Each Criterion target is declared explicitly in the benchsuite manifest with `harness = false`. Shared support code does not wrap Criterion or define a universal benchmark framework. A target exposes Criterion's concepts directly so its measurement choices remain visible. + +`benches/fixtures` is separate from the runner package so future benchmark targets and performance tools can reuse its workloads. + +The root package sets `autobenches = false`, preventing Cargo from interpreting future paths under the top-level `benches` directory as benchmark targets of the `symposium` package. It also excludes `/benches` from the published package. +`cargo package --list` verifies that benchmark-only files are absent from the crate archive. + +The fixture manifest is a virtual workspace containing `cli` and `server`. The `domain`, `terminal`, and `storage` path dependencies are excluded from that workspace so Cargo metadata sees them as dependencies rather than members. Each dependency manifest contains an empty `[workspace]` marker, preventing Cargo from associating it with Symposium's outer workspace. The fixture therefore does not require entries in the root workspace's `exclude` list. + +## Benchmark contract + +Every benchmark target begins with a doc comment containing these fields: + + +| Field | Meaning | +| --------------- | ---------------------------------------------------------------- | +| Claim | The performance property the benchmark is intended to represent. | +| Workload | The fixture and inputs supplied to the code. | +| Timed operation | The exact operation included in the measurement. | +| Excluded setup | Preparation deliberately kept outside the timer. | +| Invariants | Conditions checked to ensure the intended path is exercised. | +| Metric | The quantity reported and its unit. | +| Noise | Known uncontrolled effects and interpretation limits. | +| Lifecycle | `experimental`, `observed`, or `gated`. | + + +The target's doc comment is the single source of truth because it is next to the code that can invalidate the contract. `benches/README.md` is an index of targets, commands, lifecycle states, and links to those contracts; it does not duplicate all eight fields. + +## Framework + +The initial suite uses [Criterion.rs](https://criterion-rs.github.io/book/). The performance story includes filesystem access and Cargo subprocesses, so wall-clock measurement and statistical sampling are appropriate. A callgrind-based instruction counter would not represent the latency of those external operations. It may still be useful for a later CPU-bound benchmark. + +The implementation uses `std::hint::black_box` for both values passed from Criterion setup into a timed closure and results returned by the timed operation. Fixture preparation, cache-state construction, runtime construction, and correctness assertions remain outside the timer. + +The initial `WorkspaceDeps` cases use Criterion's minimum of ten samples and a 15-second measurement target. These subprocess-bound cases can take more than a second per iteration and vary substantially across machines, so the suite bounds individual runs instead of continually increasing the target time. Lifecycle stability is evaluated from repeated runs in the pinned benchmark environment rather than from a larger local sample count. + +## Hook cases: unchanged workspace + +The hook target contains two cases: + + +| Case | Configuration | Interpretation | +| ------------------------------------------- | ------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | +| `hook_dispatch/pre_tool_use_minimal_config` | Both builtin registries disabled and no configured plugins | The fixed in-process pipeline and Cargo-subprocess floor. | +| `hook_dispatch/pre_tool_use_local_registry` | Builtin registries disabled and one small local path registry configured | The headline end-to-end case for a representative local registry. It includes registry loading, activation gating, hook selection, and predicate evaluation, but does not isolate their cost from the Cargo-subprocess floor. | + + +Their shared contract is: + + +| Field | Definition | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Claim | Wall-clock latency of Symposium's in-process `PreToolUse` pipeline in an unchanged Cargo workspace, at its minimal floor and with a representative local registry. | +| Workload | A simulated agent event using the checked-in fixtures, with default auto-sync enabled, fresh workspace state, and a valid `WorkspaceDeps` disk cache. The local-registry case loads the registry fixture's three plugins. | +| Timed operation | Input parsing, auto-sync freshness decision, built-in dispatch, registry and workspace-plugin loading, plugin activation, hook selection, predicate evaluation, and output serialization. | +| Excluded setup | Fixture copy, `Symposium` construction, Tokio runtime construction, initial cache population, workspace-state preparation, and invariant checks. CLI startup, configuration parsing, registry refresh, stdin/stdout, and terminal I/O are not measured. | +| Invariants | The workspace state and dependency cache are valid; metadata and network access are not attempted; the loaded plugin names are exactly the three fixture entries; no external plugin process runs; the expected successful hook output is produced. | +| Metric | Wall-clock time per in-process hook dispatch. | +| Noise | The two Cargo workspace-lookup subprocesses currently dominate the result and can mask changes in the in-process registry and predicate work. Filesystem and operating-system caches, process scheduling, shared-runner hardware, and developer-level Cargo configuration also vary. | +| Lifecycle | `experimental`. | + + +The local registry has three fixed entries: + +- `always-active` uses the explicit `depends-on = ["*"]` gate and exercises the active-plugin path; +- `predicate-gated` has a `PreToolUse` hook gated by `path_exists(./.symposium-benchmark-never-present)`; setup asserts that path is absent from the benchmark process's current working directory, so hook selection and predicate evaluation run without spawning its otherwise-valid command; +- `dormant` has no inferred or explicit activation gate and therefore exercises the dormant-plugin path. + +The benchmark package calls the public `symposium::hook::execute_hook` API directly rather than depending on `symposium-testlib`. This follows the same simulation seam as the test harness while keeping the benchmark package's support code focused. + +The predicate's relative path resolves from the benchmark process's current working directory, not from the copied project fixture. Cargo sets that directory to the `benches/benchsuite` package root. Setup reads the actual current directory and verifies the sentinel path is absent there before measurement. + +In the current implementation, the unchanged-workspace path executes `cargo locate-project` once during the auto-sync freshness check and again when the new `WorkspaceDeps` resolves its disk cache. These are identical subprocesses with identical arguments and working directory. The cases make that floor visible and will register a change if the flow later reuses the workspace root or otherwise removes one lookup. That optimization follows the benchmark addition rather than being bundled into it. + +The local-registry case therefore reports representative end-to-end dispatch +latency, not an isolated registry-processing measurement. The subprocess floor +can hide changes in the smaller in-process portion; a focused component case is +needed later if those operations require their own regression sentinel. + +## Component cases: `WorkspaceDeps` + +The initial component target has two cases: + + +| Case | Prepared state | Timed operation | Interpretation | +| -------------------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `workspace_deps/symposium_cache_miss` | Empty Symposium workspace cache and a new resolver | `WorkspaceDeps::load()`, including workspace lookup, Cargo metadata, and cache write-through | Quantifies the work avoided by a valid Symposium cache. Most time belongs to Cargo rather than Symposium. | +| `workspace_deps/new_resolver_disk_cache_hit` | Valid disk cache and a new resolver | `WorkspaceDeps::load()`, including workspace lookup, cache validation, file read, and deserialization | Quantifies the component cost paid by each new resolver when the cache is valid. It is normally dominated by the `cargo locate-project` subprocess. | + + +The first case is called a *Symposium cache miss*, not a *cold load*. Criterion repeats work on one machine, so the operating system's filesystem cache and the Cargo executable may already be warm. The harness controls Symposium's cache state, not the complete machine state. + +The cases may use different Criterion measurement settings. The subprocess case needs fewer, longer samples than an in-process CPU benchmark. + +There is no `memory_cache_hit` performance case. Once initialized, `WorkspaceDeps::load()` primarily measures `OnceLock` and `Option::as_ref`, not a meaningful user-visible Symposium operation. The memoization invariant is protected by a correctness test instead. + +There is also no direct `try_disk_cache` benchmark in the initial story. That function is private, and exposing an implementation detail solely to measure a microsecond-scale parse is not justified while workspace lookup dominates the user-visible disk-hit path. A direct parsing benchmark can be added if cache size or profiling later shows serialization to be material. + +### Fixture + +The checked-in `reference-project` fixture contains a virtual Cargo workspace with two members, `cli` and `server`, and three local path dependencies: `domain`, `terminal`, and `storage`. `domain` is shared by both members. The dependency packages are excluded from fixture workspace membership, and a `Cargo.lock` is committed. This produces a small but nontrivial direct dependency graph. + +The separate `local-registry` fixture contains the three manifest-backed entries used by the representative hook case. Keeping the registry outside the Cargo project models a separately configured registry and prevents workspace-plugin discovery from observing registry content. The fixtures README records the graph and registry invariants that support code validates before measurement. + +The `reference-project` fixture contains `.cargo/config.toml` with Cargo offline mode enabled. Path dependencies and a committed lockfile avoid registry resolution; the Cargo configuration enforces the no-network invariant rather than relying on that layout by convention. + +The local-registry configuration points at the copied `local-registry` fixture with a sandbox-relative path. The minimal configuration leaves that directory unconfigured. Both disable the builtin recommendations and user-plugin registries, so neither case depends on mutable user or remote content. + +The fixtures represent one small project and one registry, not a workspace-size or plugin-count scaling curve. Larger or generated fixtures require a separate benchmark claim. + +Each benchmark run copies the required fixtures into an isolated sandbox. The sandbox also contains dedicated Symposium configuration and cache directories, so a run cannot read or modify the developer's normal Symposium state. + +The initial harness does not change the benchmark process's `CARGO_HOME`. Process-wide environment mutation is unsafe once other threads may exist, and the production resolver has no per-command Cargo-environment seam. CI provides an ephemeral Cargo home; local runs may still be influenced by user-level Cargo configuration. The fixture-local offline setting enforces the important no-network property, and the remaining local configuration is recorded as noise rather than expanding production APIs solely for the benchmark. + +### Setup and data flow + +For `symposium_cache_miss`, per-iteration setup removes only the sandbox's workspace cache and constructs a new resolver. Criterion's per-iteration setup runs outside the timer. The timed load recreates the cache. + +For `new_resolver_disk_cache_hit`, setup loads the workspace once and verifies that the cache file exists. Each measured iteration receives a new resolver pointed at that cache, so its in-memory `OnceLock` is empty while the disk cache is valid. + +Before measurement, an untimed preflight uses a mock Cargo executable that forwards `locate-project` but rejects `metadata`. A successful load therefore proves that the disk cache was used. Timed samples switch back to the real Cargo executable so the wrapper does not add another shell process to the result. + +The harness also validates the workspace root, the expected two members, the expected three dependencies, and the required cache state. + +`WorkspaceDeps` records `Cargo.lock` modification times with whole-second granularity. The fixture lockfile is immutable during a benchmark run. Future benchmarks that modify it must account for that granularity rather than assume an immediate timestamp change will invalidate the cache. + +## Failures and correctness checks + +Shared fixture helpers return errors with the fixture, workspace, or cache path needed to diagnose the failure. The benchmark executable reports the error and stops. A setup failure or `WorkspaceDeps::load()` returning `None` must never be converted into a timing sample. + +The benchsuite library has unit tests for fixture discovery, copying, sandbox preparation, and its benchmark-specific mock-Cargo preflight helper. Cache behavior belongs to the main crate and is tested in `tests/workspace_cache.rs` with `symposium-testlib`'s existing cross-platform mock-Cargo support: + +1. The first cache-miss load invokes `locate-project` and `metadata` once each; repeated loads through that resolver invoke neither again. +2. A new resolver with a valid disk cache can invoke `locate-project` but must not invoke `metadata`. +3. Advancing `Cargo.lock`'s modification time by one full second invalidates the disk cache and makes a new resolver invoke `metadata` again. The test sets the timestamp explicitly rather than sleeping or relying on filesystem timing. + +The wrapper is never part of a timed sample. + +Criterion targets support a fast smoke run through: + +```text +cargo test -p symposium-benchsuite --benches +``` + +Smoke runs execute workloads without collecting full measurements. Normal pull request CI compiles benchmark targets and runs the small support-library and cache-invariant tests. Full smoke and measurement runs belong to the benchmark workflow. + +## Commands + +The operator guide records the authoritative commands. The initial interface is: + +```text +cargo check -p symposium-benchsuite --all-targets +cargo test -p symposium-benchsuite --lib +cargo test -p symposium-benchsuite --benches +cargo bench -p symposium-benchsuite --bench workspace_deps +cargo bench -p symposium-benchsuite --bench hook_dispatch +``` + +Criterion filters allow an individual group or case to run without executing unrelated benchmark additions. + +## CI and result lifecycle + +Normal pull request CI runs `cargo check -p symposium-benchsuite --all-targets` on native Linux, macOS, and Windows jobs. The musl cross-compilation job is not part of the initial benchmark check. Support-library and cache-invariant tests run as ordinary correctness tests. + +A separate measurement workflow runs: + +- on manual dispatch for a chosen ref; +- on pull requests that change benchmark code or a path participating in the measured flows, including the benchmark workflow file itself. + +The pull-request path filter follows package and configuration ownership boundaries rather than listing individual source modules. It covers `.cargo/**`, `benches/**`, `src/**`, `symposium-install/**`, `symposium-sdk/**`, the root Cargo manifests, and the benchmark workflow itself. This deliberately accepts some extra runs: a module-by-module list can miss a transitive dependency or become incomplete when code moves, silently leaving measured behavior uncovered. + +There is no weekly schedule initially. A scheduled job is added only when its results have a durable consumer or a named maintainer responsible for reviewing them. Until then, a recurring artifact would be write-only storage. + +The measurement workflow uses `ubuntu-24.04` and names an exact Rust toolchain version rather than the moving `stable` alias. Changing either is an explicit benchmark-environment change and resets historical comparability. It uses Node 24-compatible releases of the official GitHub checkout, cache, and artifact actions so the measurement job does not depend on a deprecated runner runtime. Each run records the commit SHA, Rust and Cargo versions, operating-system details, and available CPU information. + +Before measuring, the workflow runs every Criterion target once in test mode. This separates workload correctness from statistical measurement and fails early when fixture preparation, invariants, or timed operations are broken. + +Headline estimates are written to the Actions job summary so the person who triggered a run can read them without downloading an archive. The summary labels the measurements as experimental and informational. For the `WorkspaceDeps` pair, it includes both medians and the derived cache-miss-to-hit speedup ratio. Criterion's full result directory is uploaded as an expiring, run-attempt-specific artifact only for post-hoc inspection. The attempt identifier prevents an immutable artifact from colliding with one produced by an earlier rerun. General build caches must not implicitly supply an unnamed Criterion baseline; otherwise the displayed comparison can refer to an unrelated run. + +The initial workflow does not fail because of a measured slowdown and is not a required merge gate. Compilation failures, setup failures, and benchmark crashes remain visible failures rather than being hidden with `continue-on-error`. + +Benchmarks move through three lifecycle states: + +```text +experimental -> observed -> gated +``` + +- **Experimental to observed:** the workload contract is unchanged across six consecutive successful runs using the same runner image and exact toolchain; a maintainer reviews the results; and `(maximum median - minimum median) / median of the six medians` is at most 10%. +- **Observed to gated:** a named owner agrees to triage failures; at least 20 paired no-change base/head comparisons run in the intended comparison environment; the 95th percentile absolute paired difference is at most 3%; and the regression threshold is no smaller than twice that measured noise. + +A benchmark that does not meet these conditions remains in its current state. The numbers are initial operating criteria and can be revised explicitly when collected data demonstrates that a different definition is more useful. + +All initial benchmarks start as experimental. Shared GitHub-hosted hardware may never be stable enough for gating even with a fixed image and toolchain; gating can therefore require paired execution on a controlled runner or dedicated hardware. + +## Incremental delivery + +The first pull request is built as independently working additions: + +1. root-package safeguards, the benchsuite workspace package, and the operator README; +2. shared fixture and sandbox support with unit tests; +3. cache-behavior correctness tests; +4. the Symposium-cache-miss component case; +5. the new-resolver disk-cache-hit component case; +6. the minimal and local-registry unchanged-workspace `PreToolUse` cases; +7. CI workflows and final documentation updates. + +Each addition compiles and has one stated purpose before the next is introduced. Implementation findings can revise this design when the code exposes a misleading workload or unnecessary abstraction. diff --git a/symposium-testlib/src/lib.rs b/symposium-testlib/src/lib.rs index 4d46f513..44b50241 100644 --- a/symposium-testlib/src/lib.rs +++ b/symposium-testlib/src/lib.rs @@ -477,31 +477,34 @@ impl TestContext { /// through the command interpreter, and production spawns the cargo /// override directly, so this needs no production change. It does require /// `sh` on PATH (the documented Windows dev/CI requirement). - pub fn set_mock_cargo(&mut self, script: &str) { - let cargo_override = self.write_mock_cargo(script); + /// Returns the generated `sh` script path so tests can locate files the + /// script writes relative to `$0` without depending on its directory. + pub fn set_mock_cargo(&mut self, script: &str) -> PathBuf { + let (cargo_override, script_path) = self.write_mock_cargo(script); self.sym.set_cargo_override(cargo_override); + script_path } - /// Write the mock cargo as a directly-spawnable program; return its path. + /// Write the mock cargo and return its executable and `sh` script paths. #[cfg(not(windows))] - fn write_mock_cargo(&self, script: &str) -> PathBuf { + fn write_mock_cargo(&self, script: &str) -> (PathBuf, PathBuf) { use std::os::unix::fs::PermissionsExt; let path = self.tempdir.join("mock-cargo"); std::fs::write(&path, script).unwrap(); std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); - path + (path.clone(), path) } /// Windows can't exec a shebang script, so run it through `sh` via a /// one-line `.cmd` shim (Rust spawns `.cmd` through the command interpreter). #[cfg(windows)] - fn write_mock_cargo(&self, script: &str) -> PathBuf { + fn write_mock_cargo(&self, script: &str) -> (PathBuf, PathBuf) { let sh_script = self.tempdir.join("mock-cargo.sh"); std::fs::write(&sh_script, script).unwrap(); let sh_script_fwd = sh_script.to_string_lossy().replace('\\', "/"); let cmd_shim = self.tempdir.join("mock-cargo.cmd"); std::fs::write(&cmd_shim, format!("@sh \"{sh_script_fwd}\" %*\r\n")).unwrap(); - cmd_shim + (cmd_shim, sh_script) } /// Replace variable content with stable placeholders for snapshot tests. diff --git a/tests/fixtures/workspace-cache0/.cargo/config.toml b/tests/fixtures/workspace-cache0/.cargo/config.toml new file mode 100644 index 00000000..d52f0a8c --- /dev/null +++ b/tests/fixtures/workspace-cache0/.cargo/config.toml @@ -0,0 +1,2 @@ +[net] +offline = true diff --git a/tests/fixtures/workspace-cache0/Cargo.lock b/tests/fixtures/workspace-cache0/Cargo.lock new file mode 100644 index 00000000..c5d55dde --- /dev/null +++ b/tests/fixtures/workspace-cache0/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "workspace-cache-fixture" +version = "0.0.0" diff --git a/tests/fixtures/workspace-cache0/Cargo.toml b/tests/fixtures/workspace-cache0/Cargo.toml new file mode 100644 index 00000000..43d56ef8 --- /dev/null +++ b/tests/fixtures/workspace-cache0/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "workspace-cache-fixture" +version = "0.0.0" +edition = "2024" +publish = false + +[workspace] diff --git a/tests/fixtures/workspace-cache0/src/lib.rs b/tests/fixtures/workspace-cache0/src/lib.rs new file mode 100644 index 00000000..1ad8fdbd --- /dev/null +++ b/tests/fixtures/workspace-cache0/src/lib.rs @@ -0,0 +1 @@ +//! Minimal Cargo workspace used to verify Symposium's workspace cache. diff --git a/tests/workspace_cache.rs b/tests/workspace_cache.rs new file mode 100644 index 00000000..c8059b6f --- /dev/null +++ b/tests/workspace_cache.rs @@ -0,0 +1,180 @@ +use anyhow::{Context, Result}; +use std::{ + fs::{self, File, FileTimes}, + path::Path, + sync::Arc, + time::Duration, +}; +use symposium::pm::{LoadedWorkspace, WorkspaceDeps}; +use symposium_testlib::{TestMode, with_fixture}; + +const CARGO_CALL_LOG: &str = ".symposium-cargo-calls"; +const WORKSPACE_FIXTURE: &[&str] = &["workspace-cache0"]; +const RECORDING_CARGO: &str = indoc::indoc! {r#" + #!/bin/sh + printf '%s\n' "$1" >> "${0%/*}/.symposium-cargo-calls" + exec cargo "$@" +"#}; +const METADATA_REJECTING_CARGO: &str = indoc::indoc! {r#" + #!/bin/sh + printf '%s\n' "$1" >> "${0%/*}/.symposium-cargo-calls" + if [ "$1" = "metadata" ]; then + exit 1 + fi + exec cargo "$@" +"#}; + +fn read_cargo_calls(call_log: &Path) -> Result { + fs::read_to_string(call_log) + .with_context(|| format!("reading Cargo call log `{}`", call_log.display())) +} + +fn load_or_panic<'a>( + resolver: &'a WorkspaceDeps, + call_log: &Path, + failure: &str, +) -> &'a Arc { + resolver.load().unwrap_or_else(|| { + panic!( + "{failure}; Cargo calls:\n{}", + read_cargo_calls(call_log).unwrap_or_else(|error| format!("<{error:#}>")) + ) + }) +} + +fn advance_modified_time(path: &Path) -> Result<()> { + let modified = fs::metadata(path) + .and_then(|metadata| metadata.modified()) + .with_context(|| format!("reading modification time for `{}`", path.display()))?; + let file = File::options().write(true).open(path).with_context(|| { + format!( + "opening `{}` to advance its modification time", + path.display() + ) + })?; + let times = FileTimes::new().set_modified(modified + Duration::from_secs(1)); + file.set_times(times) + .with_context(|| format!("advancing modification time for `{}`", path.display())) +} + +#[tokio::test] +async fn repeated_loads_run_cache_miss_commands_once() -> Result<()> { + with_fixture( + TestMode::SimulationOnly, + WORKSPACE_FIXTURE, + async |mut context| { + let call_log = context + .set_mock_cargo(RECORDING_CARGO) + .with_file_name(CARGO_CALL_LOG); + let workspace = context + .workspace_root + .as_deref() + .context("workspace-cache0 must provide a workspace root")?; + let resolver = context.sym.workspace_deps(workspace); + + let first_load = load_or_panic( + &resolver, + &call_log, + "initial workspace dependency load failed", + ); + let second_load = load_or_panic( + &resolver, + &call_log, + "memoized workspace dependency load failed", + ); + assert!( + Arc::ptr_eq(first_load, second_load), + "repeated loads should return the memoized workspace" + ); + + let calls = read_cargo_calls(&call_log)?; + assert_eq!(calls, "locate-project\nmetadata\n"); + + Ok(()) + }, + ) + .await +} + +#[tokio::test] +async fn new_resolver_uses_the_disk_cache_without_metadata() -> Result<()> { + with_fixture( + TestMode::SimulationOnly, + WORKSPACE_FIXTURE, + async |mut context| { + let call_log = context + .set_mock_cargo(RECORDING_CARGO) + .with_file_name(CARGO_CALL_LOG); + // Own the path because replacing mock Cargo later mutably borrows `context`. + let workspace = context + .workspace_root + .clone() + .context("workspace-cache0 must provide a workspace root")?; + let first_resolver = context.sym.workspace_deps(&workspace); + + load_or_panic( + &first_resolver, + &call_log, + "cache-populating workspace dependency load failed", + ); + + fs::remove_file(&call_log).context("clearing Cargo call log after cache population")?; + let call_log = context + .set_mock_cargo(METADATA_REJECTING_CARGO) + .with_file_name(CARGO_CALL_LOG); + + let second_resolver = context.sym.workspace_deps(&workspace); + load_or_panic( + &second_resolver, + &call_log, + "new resolver did not use the disk cache", + ); + + let calls = read_cargo_calls(&call_log)?; + assert_eq!(calls, "locate-project\n"); + + Ok(()) + }, + ) + .await +} + +#[tokio::test] +async fn changed_cargo_lock_invalidates_disk_cache() -> Result<()> { + with_fixture( + TestMode::SimulationOnly, + WORKSPACE_FIXTURE, + async |mut context| { + let call_log = context + .set_mock_cargo(RECORDING_CARGO) + .with_file_name(CARGO_CALL_LOG); + let workspace = context + .workspace_root + .as_deref() + .context("workspace-cache0 must provide a workspace root")?; + let first_resolver = context.sym.workspace_deps(workspace); + + load_or_panic( + &first_resolver, + &call_log, + "cache-populating workspace dependency load failed", + ); + + fs::remove_file(&call_log).context("clearing Cargo call log after cache population")?; + advance_modified_time(&workspace.join("Cargo.lock"))?; + + let second_resolver = context.sym.workspace_deps(workspace); + load_or_panic( + &second_resolver, + &call_log, + "workspace dependency reload failed after Cargo.lock changed", + ); + + let calls = read_cargo_calls(&call_log)?; + assert_eq!(calls, "locate-project\nmetadata\n"); + + Ok(()) + }, + ) + .await +}