diff --git a/.github/actions/setup-validation/action.yml b/.github/actions/setup-validation/action.yml index ef185f4f..4cfda85e 100644 --- a/.github/actions/setup-validation/action.yml +++ b/.github/actions/setup-validation/action.yml @@ -10,6 +10,9 @@ inputs: java: description: Install Java 21 for bounded symbolic model checking. default: "false" + rust: + description: Install the pinned Rust toolchain (rust/rust-toolchain.toml) with clippy and rustfmt. + default: "false" runs: using: composite steps: @@ -35,3 +38,14 @@ runs: with: distribution: temurin java-version: "21" + # The toolchain pin matches rust/rust-toolchain.toml and the cargo 1.98.1 + # prerequisite probe in formal/validation.mjs. + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master 2026-09-12 + if: inputs.rust == 'true' + with: + toolchain: 1.98.1 + components: clippy, rustfmt + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + if: inputs.rust == 'true' + with: + workspaces: rust diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53a9fcfb..c1c52ef2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,17 +67,32 @@ jobs: retention-days: 14 if-no-files-found: warn - # The existing required `test` status now requires both native language jobs. + rust: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: ./.github/actions/setup-validation + with: + rust: "true" + - name: Check Rust formatting, clippy and native tests including the smoke conformance run + run: make check-rust + - name: Check real Redis, Valkey and Cluster integrations and invalidation vectors + run: make integration-rust + + # The existing required `test` status now requires every native language job. test: - needs: [typescript, go] + needs: [typescript, go, rust] if: always() runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Require successful TypeScript and Go checks + - name: Require successful TypeScript, Go and Rust checks env: TYPESCRIPT_RESULT: ${{ needs.typescript.result }} GO_RESULT: ${{ needs.go.result }} + RUST_RESULT: ${{ needs.rust.result }} run: | test "$TYPESCRIPT_RESULT" = success test "$GO_RESULT" = success + test "$RUST_RESULT" = success diff --git a/.github/workflows/formal-full.yaml b/.github/workflows/formal-full.yaml index fab8a2d3..6aa4b9c5 100644 --- a/.github/workflows/formal-full.yaml +++ b/.github/workflows/formal-full.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: exploration: - description: Explore a fresh seed in both ports in addition to pinned acceptance. + description: Explore a fresh seed in every port in addition to pinned acceptance. type: boolean default: false schedule: @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest # Run 35568655690 exhausted the old hour inside the expanded challenge # campaign, before generation or native replay. Allow the model job's - # budget below plus time for the fresh corpus and both ports; individual + # budget below plus time for the fresh corpus and every port; individual # evaluator and replay commands retain their own hang bounds. timeout-minutes: 180 steps: @@ -32,7 +32,8 @@ jobs: with: quint: "true" go: "true" - - name: Explore a fresh recorded seed and replay both ports + rust: "true" + - name: Explore a fresh recorded seed and replay every port run: make explore - name: Preserve exploratory sources, seed and counterexamples if: always() @@ -77,8 +78,8 @@ jobs: retention-days: 14 # The single producer for every downstream lane: corpus generation, fixture - # recomputation and the shared witness evidence. Both port replays and both - # mutation measurements depend only on this job's artifact and run in parallel. + # recomputation and the shared witness evidence. Every port replay and mutation + # measurement depend only on this job's artifact and run in parallel. generate: # Scheduled workflows run the default branch; keep the intended main scope explicit. if: github.event_name != 'schedule' || github.ref == 'refs/heads/main' @@ -256,6 +257,37 @@ jobs: if-no-files-found: warn retention-days: 14 + rust-parity: + needs: generate + runs-on: ubuntu-latest + # The Rust replay compiles the crate in release mode before replaying the + # same corpus Go completes in about 4.5 minutes; the budget mirrors go-parity. + timeout-minutes: 40 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: ./.github/actions/setup-validation + with: + rust: "true" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: formal-traces + path: .formal-traces + - name: Require Rust replay of the complete corpus + run: make formal-rust + - name: Preserve Rust parity completion evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: rust-parity-evidence + path: | + .formal-traces/rust-replay.jsonl + .formal-traces/rust-replay-summary.json + .formal-traces/rust-context.json + .formal-traces/rust-completion.json + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + # The Go lane bounds the whole workflow: every mutant replays the generated # cohort, about 1.6 minutes per mutant on a fast runner and about 3 on the # slow class (13 mutants took 25 minutes in run 34660598461 and 47-48 in @@ -324,11 +356,72 @@ jobs: if-no-files-found: warn retention-days: 14 + # The Rust lane compiles the crate and its test binaries in release mode + # once per mutant (dependencies are cached across mutants) and replays the + # generated cohort in about a minute; the budget mirrors go-mutations. + rust-mutations: + needs: generate + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + MUTATION_SHARD: ${{ matrix.shard }}/3 + timeout-minutes: 45 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: ./.github/actions/setup-validation + with: + rust: "true" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: formal-traces + path: .formal-traces + - name: Measure one shard of Rust assertion-based fault detection + run: make mutations-rust + - name: Preserve this shard's Rust mutation evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: rust-semantic-shard-${{ matrix.shard }} + path: .formal-traces/rust-semantic/shards/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + + rust-mutations-merge: + needs: rust-mutations + # Run after the shards regardless of their result so a missing shard is + # diagnosed by the merge; skip only when the matrix itself never ran. + if: always() && needs.rust-mutations.result != 'skipped' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: ./.github/actions/setup-validation + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: rust-semantic-shard-* + path: .formal-traces/rust-semantic/shards + merge-multiple: true + - name: Require every Rust shard and merge the complete report + run: make mutations-merge-rust + - name: Preserve Rust mutation evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: rust-semantic-evidence + path: .formal-traces/rust-semantic/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + # A failed or skipped dependency cannot turn smoke evidence into full acceptance. # The mutation lanes count through their merge jobs: a merge succeeds only # when every shard's report is present, consistent and free of lost detections. formal-full: - needs: [check-models, generate, typescript-parity, symbolic, typescript-mutations-merge, go-parity, go-mutations-merge, exploration] + needs: [check-models, generate, typescript-parity, symbolic, typescript-mutations-merge, go-parity, go-mutations-merge, rust-parity, rust-mutations-merge, exploration] if: always() runs-on: ubuntu-latest timeout-minutes: 5 @@ -351,6 +444,13 @@ jobs: with: name: go-parity-evidence path: formal-summary/go + - name: Collect Rust completion evidence + if: always() + continue-on-error: true + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: rust-parity-evidence + path: formal-summary/rust - name: Collect model check report if: always() continue-on-error: true @@ -383,13 +483,16 @@ jobs: formal-summary/go/go-completion.json formal-summary/go/go-context.json formal-summary/go/go-replay-summary.json + formal-summary/rust/rust-completion.json + formal-summary/rust/rust-context.json + formal-summary/rust/rust-replay-summary.json formal-summary/model-check/model-properties/report.json formal-summary/symbolic/report.json formal-summary/exploration/*/report.json include-hidden-files: true if-no-files-found: warn retention-days: 90 - - name: Require the model check, the generated corpus, both port replays and both merged mutation measurements + - name: Require the model check, the generated corpus, every port replay and every merged mutation measurement env: CHECK_MODELS_RESULT: ${{ needs.check-models.result }} GENERATE_RESULT: ${{ needs.generate.result }} @@ -397,6 +500,8 @@ jobs: TYPESCRIPT_MUTATIONS_MERGE_RESULT: ${{ needs.typescript-mutations-merge.result }} GO_RESULT: ${{ needs.go-parity.result }} GO_MUTATIONS_MERGE_RESULT: ${{ needs.go-mutations-merge.result }} + RUST_MUTATIONS_MERGE_RESULT: ${{ needs.rust-mutations-merge.result }} + RUST_RESULT: ${{ needs.rust-parity.result }} SYMBOLIC_RESULT: ${{ needs.symbolic.result }} EXPLORATION_RESULT: ${{ needs.exploration.result }} EXPLORATION_REQUIRED: ${{ github.event_name == 'schedule' || inputs.exploration == true }} @@ -407,6 +512,8 @@ jobs: test "$TYPESCRIPT_MUTATIONS_MERGE_RESULT" = success test "$GO_RESULT" = success test "$GO_MUTATIONS_MERGE_RESULT" = success + test "$RUST_MUTATIONS_MERGE_RESULT" = success + test "$RUST_RESULT" = success test "$SYMBOLIC_RESULT" = success if [ "$EXPLORATION_REQUIRED" = true ]; then test "$EXPLORATION_RESULT" = success diff --git a/.github/workflows/formal.yaml b/.github/workflows/formal.yaml index 3636230e..cbd0633b 100644 --- a/.github/workflows/formal.yaml +++ b/.github/workflows/formal.yaml @@ -24,7 +24,8 @@ jobs: - uses: ./.github/actions/setup-validation with: go: "true" - - name: Audit contracts and replay committed Quint smoke in both ports + rust: "true" + - name: Audit contracts and replay committed Quint smoke in every port run: make audit smoke - name: Detect changes requiring fresh Quint artifact generation id: fixture-scope diff --git a/.gitignore b/.gitignore index 76a65fcb..5877d6cb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ docs/.vitepress/cache/ .idea/ .vscode/ *.log +rust/target/ diff --git a/AGENTS.md b/AGENTS.md index 5ca0da6b..6e29d6e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project overview -DialCache has TypeScript and Go implementations with explicit request-scoped enablement, local and Redis layers, runtime rollout controls, request coalescing, targeted invalidation, and adapter-based observability. +DialCache has TypeScript, Go and Rust implementations with explicit request-scoped enablement, local and Redis layers, runtime rollout controls, request coalescing, targeted invalidation, and adapter-based observability. ## Structure @@ -27,6 +27,7 @@ src/ internal/ # Cache layers, runtime config, payload compression, and invalidation Lua script test/ # Unit and Redis integration tests go/ # Go module, public cache and adapters, shared-corpus replay +rust/ # Rust crate, public cache and adapters, shared-corpus replay (tests/conformance.rs) formal/ # Quint behavioral source of truth, contracts and portable vectors ``` @@ -64,8 +65,8 @@ formal/ # Quint behavioral source of truth, contracts and portab readable as behavior definitions, share helpers with identical meaning, retain independent property checks, and register executable evidence in the catalogs. - Define portable behavior in Quint first. Require consequential generated - witnesses and replay the same histories in TypeScript and Go; keep native - API, wire and integration tests for their explicit boundaries. + witnesses and replay the same histories in TypeScript, Go and Rust; keep + native API, wire and integration tests for their explicit boundaries. ## Validation @@ -75,8 +76,10 @@ make check make integration ``` -Use `make formal` for complete Quint model checks, corpus generation and both -ports' full replay, then `make mutations` for assertion-strength checks. +`make check-rust` runs the Rust crate's fmt, clippy, unit, vector, scenario and +smoke checks; `make formal-rust` completes its replay of the generated corpus. +Use `make formal` for complete Quint model checks, corpus generation and every +port's full replay, then `make mutations` for assertion-strength checks. `make ci` runs all validation in the required order. `make help` lists targets and prerequisites; `formal/README.md` documents the fast PR and full-validation workflows. Full behavior/model/replay changes require full validation before diff --git a/Makefile b/Makefile index 47360da9..76097672 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ NODE ?= node -.PHONY: help check check-ts check-go docs audit smoke formal formal-check formal-generate formal-ts formal-go fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go package-floor ci explore model-check +.PHONY: help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go mutations-rust mutations-merge-rust integration integration-ts integration-go integration-rust package-floor ci explore model-check -help check check-ts check-go docs audit smoke formal formal-check formal-generate formal-ts formal-go fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go package-floor ci explore model-check: +help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go mutations-rust mutations-merge-rust integration integration-ts integration-go integration-rust package-floor ci explore model-check: $(NODE) formal/validation.mjs $@ diff --git a/README.md b/README.md index 2e14012e..68b83bd4 100644 --- a/README.md +++ b/README.md @@ -132,10 +132,10 @@ for sampling and comparison behavior. | Recovery from selected source failures | [Stale-on-error](https://lan17.github.io/DialCache/stale-on-error.html) | | Shared execution and deadlines | [Coalescing and liveness](https://lan17.github.io/DialCache/coalescing.html) | | Methods, options, and exports | [API reference](https://lan17.github.io/DialCache/api.html) | -| Go implementation and shared behavior contracts | [Go guide](go/README.md) · [Quint specification](formal/README.md) · [Worked walkthrough](formal/WALKTHROUGH.md) | +| Go and Rust implementations and shared behavior contracts | [Go guide](go/README.md) · [Rust guide](rust/README.md) · [Quint specification](formal/README.md) · [Worked walkthrough](formal/WALKTHROUGH.md) | -The Go port and the TypeScript library replay the same Quint-generated -histories. Whether those histories reach every required boundary is decided by +The Go port, the Rust port and the TypeScript library replay the same +Quint-generated histories. Whether those histories reach every required boundary is decided by one language-neutral evaluator, `node formal/witnesses.mjs evaluate`, that any port runs over the same corpus; no port depends on another port's test suite for its completion evidence. diff --git a/formal/PORTING.md b/formal/PORTING.md index f3b2bce3..858a683b 100644 --- a/formal/PORTING.md +++ b/formal/PORTING.md @@ -33,7 +33,7 @@ API names, threading model, internal storage, or scheduling implementation. | External effects | Hold/release actual policy, Redis read/write, serialization, and decoding operations by their invocation IDs. A held operation cannot finish until its external release or specified failure. | | Clock control | Control wall time and elapsed time separately. Advance or shift only the specified clock; follow the profile's timer-delivery rule. Preserve fractional units in the local-clock profile. | | Storage inputs | Seed the specified bytes/value/TTL or execute invalidation. The adapter must expose the requested atomic primary snapshot and maintenance result. | -| Settle | After every command, bring the implementation to quiescence under the `causally-ready-v1` contract: every task spawned by the implementation or the driver has finished or is blocked on a driver-owned gate, a driver-owned timer that is not yet due, or a driver-owned scope gate, and nothing is runnable. This obligation falls on the library as well as the test: it must expose its detached scheduling to the test executor (Go's `Defer` hook drained under `synctest.Wait`, the Node fake-timer microtask queue drained by `advanceTimersByTimeAsync(0)`) so the driver reaches quiescence without guessing turn counts or advancing deadline time. The coordinator checks this contract by name on every behavior `observe` through the driver's settlement receipt, rules R1 to R5 in the [trace and observation contract](#trace-and-observation-contract); a failed rule is a `Settlement violation`, an infrastructure error that earns no comparison credit. Both ports include a no-settle control: [formal-settlement-control.test.ts](../test/formal-settlement-control.test.ts) skips the TypeScript drain, and [settlement_control_replay_test.go](../go/settlement_control_replay_test.go) reports the Go observation held before the drain; each must fail with a settlement violation, never an observation mismatch, on the smoke history of every behavior-driver profile. A third port carries an equivalent control against its own driver. | +| Settle | After every command, bring the implementation to quiescence under the `causally-ready-v1` contract: every task spawned by the implementation or the driver has finished or is blocked on a driver-owned gate, a driver-owned timer that is not yet due, or a driver-owned scope gate, and nothing is runnable. This obligation falls on the library as well as the test: it must expose its detached scheduling to the test executor (Go's `Defer` hook drained under `synctest.Wait`, the Node fake-timer microtask queue drained by `advanceTimersByTimeAsync(0)`) so the driver reaches quiescence without guessing turn counts or advancing deadline time. The coordinator checks this contract by name on every behavior `observe` through the driver's settlement receipt, rules R1 to R5 in the [trace and observation contract](#trace-and-observation-contract); a failed rule is a `Settlement violation`, an infrastructure error that earns no comparison credit. Both ports include a no-settle control: [formal-settlement-control.test.ts](../test/formal-settlement-control.test.ts) skips the TypeScript drain, and [settlement_control_replay_test.go](../go/settlement_control_replay_test.go) reports the Go observation held before the drain; each must fail with a settlement violation, never an observation mismatch, on the smoke history of every behavior-driver profile. A further port carries an equivalent control against its own driver. | | Observe | Read actual caller outcomes, source/effect counts, event order, timestamps, values and error categories after the step. Collect observations independently of expected model state. | | Cleanup | Drain or release test-owned work, restore clock/fault hooks, and isolate the next history. Unfinished work must not silently leak into another history. | @@ -333,9 +333,9 @@ from the repository root: ```sh make check # Fast native checks and committed smoke; not full acceptance. -make formal # Rust model/corpus checks, then prepared TS and Go replay. +make formal # Quint model/corpus checks, then prepared TS, Go and Rust replay. make model-check # Separate finite symbolic checks; requires Java 21 and tar. -make mutations # Measures both fault catalogs over the generated corpus. +make mutations # Measures every fault catalog over the generated corpus. make integration # Real-server interoperability; requires Docker. ``` @@ -344,8 +344,8 @@ including symbolic checks and the exact Node 22.15.0 package floor. `make formal-check` runs the Quint evidence lane (every scheduled model, its public regressions and the model mutation challenges); the aggregate requires it, but the port lanes do not wait for it. `make formal-generate` produces the -full corpus and the shared witness evidence; `make formal-ts` and -`make formal-go` each prepare and complete one port's run against them. The +full corpus and the shared witness evidence; `make formal-ts`, `make formal-go` +and `make formal-rust` each prepare and complete one port's run against them. The parity and mutation lanes depend only on the generated corpus and shared witness evidence and run in parallel in hosted CI, whose aggregate requires all of them. These are the same entry points used by hosted CI. @@ -358,7 +358,7 @@ complete inventory. A green smoke lane supplies no full-parity claim. Print the current required IDs with `node formal/conformance.mjs inventory`. The commands below describe the lower-level completion API for implementers of another port; the Make -targets already orchestrate it for TS and Go. +targets already orchestrate it for TypeScript, Go and Rust. The shared inventory contains stable language-neutral IDs: @@ -383,35 +383,40 @@ Prepare each port immediately before its native tests. For TS, the low-level command is `node formal/conformance.mjs prepare typescript .formal-traces/ts-context.json`. Run its complete native suite, evaluate the shared witness evidence, and validate its completion before using -`node formal/conformance.mjs prepare go .formal-traces/go-context.json`. -Go preparation binds the witness evidence the shared evaluator just produced. -Do not prepare both contexts consecutively before running either suite. +`node formal/conformance.mjs prepare go .formal-traces/go-context.json` or +`node formal/conformance.mjs prepare rust .formal-traces/rust-context.json`. +Go and Rust preparation bind the witness evidence the shared evaluator just +produced. Do not prepare two contexts consecutively before running either suite. -For another language, supply a JSON array containing every repository-relative +TypeScript, Go and Rust have built-in default source inventories. For another +language, supply a JSON array containing every repository-relative implementation, driver, adapter, dependency-lock and test-configuration file that affects execution: ```sh -node formal/conformance.mjs prepare rust .formal-traces/rust-context.json rust/conformance-sources.json +node formal/conformance.mjs prepare zig .formal-traces/zig-context.json zig/conformance-sources.json ``` These commands do not run the implementation. They record a unique run ID, preparation time, specification/source fingerprints, the exact corpus bytes, and the required case inventory. A port's source manifest is a reviewed input declaration; the checker cannot discover an omitted native dependency itself. -Go's default inputs also include the shared source/fixture definitions and the -evaluated witness JSON under `.formal-traces/go-parity-witnesses/`. Another port -that consumes auxiliary evidence must include those files in its input manifest. +The Go and Rust default inputs also include the shared source/fixture +definitions and the evaluated witness JSON under +`.formal-traces/go-parity-witnesses/`. Another port that consumes auxiliary +evidence must include those files in its input manifest. Run native tests with complete trace directory selectors. Preserve the original assertion report and its timestamps. The supplied report adapters accept -Vitest JSON and `go test -json`, respectively: +Vitest JSON, `go test -json` and the Rust harness's JSON-lines report: ```sh node formal/conformance-adapters.mjs typescript .formal-traces/ts-replay.json .formal-traces/ts-context.json > .formal-traces/ts-completion.json node formal/conformance-adapters.mjs go .formal-traces/go-replay.jsonl .formal-traces/go-context.json > .formal-traces/go-completion.json +node formal/conformance-adapters.mjs rust .formal-traces/rust-replay.jsonl .formal-traces/rust-context.json > .formal-traces/rust-completion.json node formal/conformance.mjs check .formal-traces/ts-completion.json .formal-traces/ts-context.json node formal/conformance.mjs check .formal-traces/go-completion.json .formal-traces/go-context.json +node formal/conformance.mjs check .formal-traces/rust-completion.json .formal-traces/rust-context.json ``` Another language emits the same JSON completion schema: `schemaVersion: 1`, @@ -427,7 +432,7 @@ failed and incomplete results are rejected. So are changed source/corpus bytes and native runs predating preparation. Keep contexts, corpus, original reports and completion JSON together under `.formal-traces/`. Mutation targets validate the relevant full reports against current source and corpus fingerprints before -starting: TS mutations need TS completion, and Go mutations need both ports. +starting: TS mutations need TS completion, and Go mutations need TS and Go completion. Supplied adapters verify actual native assertion records before producing completion results. A completion document is test evidence, not cryptographic attestation that an untrusted driver behaved @@ -524,10 +529,10 @@ replay does. The TypeScript suite runs the same evaluator over the histories it parsed for replay and writes no evidence. The Go replay also checks that every required label names at least one history of the bound corpus. -## Current limitations for a third port +## Current limitations for a further port The coordinator checks the `causally-ready-v1` settlement contract through the -settlement receipt on every behavior observe, so a third port's driver is held +settlement receipt on every behavior observe, so a further port's driver is held to rules R1 to R5 from its first replay. What the coordinator still cannot check: `runnable` is the driver's own attestation, verified by one zero-time drain rather than by inspecting the executor; timer delivery (R6) is checked @@ -538,7 +543,7 @@ zero-delay timer or immediate the library schedules while a fake-timer tick is in progress becomes due one millisecond later, so such work stays parked until the next advance and `runnable` truthfully reads 0; the cross-port replay shows no history lands observable work there. Each port also keeps a no-settle -control against its own driver, so a third port writes one the same way. +control against its own driver, so a further port writes one the same way. Node 24 is required as test tooling: the coordinator and the witness evaluator are Node scripts that a port's test run spawns, and the completion checker and diff --git a/formal/README.md b/formal/README.md index cd32f98b..1d030a4b 100644 --- a/formal/README.md +++ b/formal/README.md @@ -1,7 +1,7 @@ # Executable DialCache specification -Quint defines the portable contracts that TypeScript, Go and future ports must -preserve. TypeScript is the executable reference those contracts formalize; a +Quint defines the portable contracts that TypeScript, Go, Rust and future ports +must preserve. TypeScript is the executable reference those contracts formalize; a disagreement between the two is settled by a distinguishing regression and a recorded decision, not by editing the easier side. Native drivers execute external commands against the real libraries; generated expectations stay in @@ -62,7 +62,7 @@ The verification models emphasize individual ownership or safety boundaries: | [dialcache-stale-recovery.qnt](./dialcache-stale-recovery.qnt) | Retained bytes, age checks and recovery authority | | [dialcache-redis-protocol.qnt](./dialcache-redis-protocol.qnt) | Frame/fence validation order | -Conformance profiles expose external commands that both language drivers replay: +Conformance profiles expose external commands that every language driver replays: | Profile | Behavior and interactions | | --- | --- | @@ -122,12 +122,12 @@ downloads its pinned solver archive. ```sh make help # Targets and prerequisites. make check # Native checks, package, docs and inventories. -make smoke # Committed Quint-derived histories in both ports. -make formal # Rust model checks, full corpus and both-port completion. +make smoke # Committed Quint-derived histories in every port. +make formal # Quint model checks, full corpus and every port's completion. make differential # Replay composed profiles' reference corpus through the working tree. make model-check # Separate finite symbolic checks; Java 21 and tar required. make mutations # Challenge assertions after full replay has passed. -make integration # Real Redis/Valkey/Cluster and interoperability. +make integration # Real Redis/Valkey/Cluster in every port and interoperability. make explore # Fresh recorded seed in an isolated source snapshot. make ci NODE22_BIN=/absolute/path/to/node22/bin/node ``` @@ -137,14 +137,15 @@ scheduled model with the Rust evaluator, the public regressions and the model mutation challenges. Its first command, `node formal/run-models.mjs check`, runs only the unmodified model checks and regressions; the next step runs the complete pinned fault campaign. `make formal` and `make ci` require both steps. -`make explore` retains the model checks, generation and both port replays but +`make explore` retains the model checks, generation and all port replays but omits that identical pinned campaign; its result remains non-acceptance evidence. `make formal-generate` runs generation, fixture -recomputation and the shared witness evaluation; `make formal-ts` and -`make formal-go` then complete each port's replay against that exact corpus. -`make mutations-ts` and `make mutations-go` split the fault campaigns. The -parity and mutation lanes depend only on the generated corpus and shared witness -evidence, so hosted CI runs all four in parallel and none of them waits for the +recomputation and the shared witness evaluation; `make formal-ts`, +`make formal-go` and `make formal-rust` then complete each port's replay against +that exact corpus. `make mutations-ts`, `make mutations-go` and +`make mutations-rust` split the fault campaigns. The parity and mutation lanes +depend only on the generated corpus and shared witness evidence, so hosted CI +runs all six in parallel and none of them waits for the model check, which runs beside generation; the aggregate requires every lane. `make fixtures-check` recomputes committed artifacts; after an intentional model edit, update them with `node formal/generate-artifacts.mjs --write` first. @@ -160,7 +161,7 @@ keeps a separate source snapshot, seed, corpus and diagnostic replay evidence. S [VALIDATION.md](./VALIDATION.md) for CI policy and report interpretation. Scheduled named public-action Quint regressions exercise their declared -boundaries independently of sampling. Both ports replay those histories and the +boundaries independently of sampling. Every port replays those histories and the complete sampled corpus; required witness coverage is checked across their union. A model regression reaches implementations only when registered for replay in `execution.json`, and the manifest validator requires every diff --git a/formal/SEMANTIC-COVERAGE.md b/formal/SEMANTIC-COVERAGE.md index 8286c4b5..0d8a1cbc 100644 --- a/formal/SEMANTIC-COVERAGE.md +++ b/formal/SEMANTIC-COVERAGE.md @@ -161,24 +161,39 @@ results; fresh measurements belong with their exact source and corpus artifacts. Use the [shared Make targets and pinned prerequisites](./README.md#generating-and-replaying-behavior): ```sh -make formal # Regenerate and complete prepared TS and Go replay. -make mutations # Measure both fault catalogs over the generated corpus. +make formal # Regenerate and complete prepared TS, Go and Rust replay. +make mutations # Measure every fault catalog over the generated corpus. ``` -`make mutations-ts` and `make mutations-go` depend only on the corpus produced -by `make formal-generate` and its shared witness evidence, not on either port's -completion report. Hosted CI runs both mutation lanes in parallel with both -parity lanes off one generation job; the aggregate requires all of them. Each -report records the source, corpus and witness fingerprints it measured. Raw -measurement programs remain `measure-semantics.mjs` and -`measure-go-semantics.mjs`; the Make targets supply the pinned prerequisite +`make mutations-ts`, `make mutations-go` and `make mutations-rust` depend only +on the corpus produced by `make formal-generate` and its shared witness +evidence, not on any port's completion report. Hosted CI runs the three +mutation lanes in parallel with the three parity lanes off one generation job; +the aggregate requires all of them. Each report records the source, corpus and +witness fingerprints it measured. Raw measurement programs remain +`measure-semantics.mjs`, `measure-go-semantics.mjs` and +`measure-rust-semantics.mjs`; the Make targets supply the pinned prerequisite checks used by hosted full validation. +Rust has a separate catalog, [`rust-mutations.json`](./rust-mutations.json). Its +ordinary cohort runs crate unit tests and native integration tests, excluding +harness controls, protocol-vector suites and real-server tests. Its generated +and fixed cohorts select the corresponding conformance harness suites. The +runner requires complete reports and assertion evidence; an infrastructure +failure is a failed measurement. Rust results establish detection for this +catalog only; the TypeScript/Go model-to-mutant boundary mappings in +`mutations.json` do not confer Rust coverage. Each mutant builds in release +mode in an isolated crate copy. Cargo build directories are isolated by shard +selection automatically; partial runs use a separate directory. A direct runner +invocation can override the directory with `DIALCACHE_RUST_TARGET_DIR`, but +concurrent shards must use distinct override paths to avoid sharing mutant binaries. + Hosted runs shard each lane with `MUTATION_SHARD=/`, matching the workflow matrix; `test/formal-validation.test.ts` pins how many mutants a shard may hold within its timeout, so catalog growth fails the pull request until the matrix grows. A shard reruns the compile check, every unmodified baseline and the witness evaluation before its contiguous slice of the catalog, and writes an incomplete report under `shards/-of-/`. `make mutations-merge-ts` and -`make mutations-merge-go` (`merge-mutation-reports.mjs`) assemble the complete +`make mutations-merge-go` and `make mutations-merge-rust` +(`merge-mutation-reports.mjs`) assemble the complete report from those shards and refuse any inconsistency: a missing or duplicated shard, a shard that failed or claims completion, differing fingerprints or baseline results, or mutations that do not cover the catalog exactly once in diff --git a/formal/VALIDATION.md b/formal/VALIDATION.md index 84ada5d7..7c63a04c 100644 --- a/formal/VALIDATION.md +++ b/formal/VALIDATION.md @@ -9,11 +9,11 @@ lists tool prerequisites and focused reproduction commands. | Task | Command | Evidence | | --- | --- | --- | | Routine implementation checks | `make check` | Native tests, coverage, package, docs and source audits | -| Full portable acceptance | `make formal` | Rust model checks, generated histories, TS replay, shared witness evaluation, Go replay, exact completion inventories; no Java | +| Full portable acceptance | `make formal` | Quint model checks with the Rust evaluator, generated histories, shared witness evaluation, TypeScript/Go/Rust replay, exact completion inventories; no Java | | Finite symbolic rules | `make model-check` | Scheduled bounded checks with checksummed standalone Apalache; requires Java 21, `tar` and pinned Quint | -| Challenge implementation assertions | `make mutations` | Compiling semantic faults tested against both completed ports | +| Challenge implementation assertions | `make mutations` | Compiling semantic faults tested against every completed port | | Real server behavior | `make integration` | Redis, Valkey, Cluster and cross-language interoperability | -| Explore another schedule sample | `make explore` | Separate source snapshot, recorded random seed, both-port replay | +| Explore another schedule sample | `make explore` | Separate source snapshot, recorded random seed, every port's replay | | Check composed profiles against their previous text | `make differential` | Lint baseline, then both-direction replay against the merge base with `origin/main`; fails on any disagreement or trace growth above the model's bound | | All required local lanes | `make ci NODE22_BIN=/absolute/path/to/node22/bin/node` | Native, formal, separate symbolic, integration and mutation runs | @@ -22,10 +22,11 @@ every scheduled model, the public regressions and the model mutation challenges. It produces nothing the port lanes consume, so the hosted workflow runs it as a `check-models` job beside generation; only the aggregate waits for it. `make formal-generate` runs `node formal/witnesses.mjs evaluate --profile all` -immediately after generation, before either port replays. That shared, +immediately after generation, before any port replays. That shared, language-neutral step is the sole producer of `.formal-traces/go-parity-witnesses/`; the TypeScript suite only checks the same gate. `make formal-ts`, `make formal-go`, -`make mutations-ts` and `make mutations-go` depend only on the generated corpus +`make formal-rust`, `make mutations-ts`, `make mutations-go` and +`make mutations-rust` depend only on the generated corpus and that witness evidence, so the hosted workflow runs them in parallel and the aggregate requires all of them. @@ -76,7 +77,7 @@ identify the revision and completed local or hosted validation. ## Reading a completion report Evidence lives under `.formal-traces/` and in the workflow's uploaded artifacts. -The prepared `ts-context.json` and `go-context.json` bind the specification, +The prepared `ts-context.json`, `go-context.json` and `rust-context.json` bind the specification, implementation, harness, corpus and required case inventory. Their matching completion reports require every scheduled case to pass. A changed input, missing result, skipped case, duplicate result or stale report fails acceptance. @@ -84,6 +85,7 @@ missing result, skipped case, duplicate result or stale report fails acceptance. ```sh node formal/conformance.mjs check .formal-traces/ts-completion.json .formal-traces/ts-context.json node formal/conformance.mjs check .formal-traces/go-completion.json .formal-traces/go-context.json +node formal/conformance.mjs check .formal-traces/rust-completion.json .formal-traces/rust-context.json ``` Keep reports with their source/corpus fingerprints and native assertion output. @@ -93,7 +95,7 @@ and environmental assumptions are defined once in ## Mutation evidence -Each native mutation report includes a `boundary` entry for every mapped +Each TypeScript or Go mutation report includes a `boundary` entry for every mapped challenge. A separate coordinator replay continues after observation mismatches and records the differing fields at every step; it preserves the normal driver and settlement checks. `confirmed` means the intended checkpoint differs on a @@ -126,7 +128,7 @@ states remain readable but do not satisfy current vector declarations. Model mutations challenge the specification's independent properties. Implementation mutations challenge the assertions that connect generated -histories to real TS/Go behavior. Report these measurements separately, including +histories to real TypeScript, Go and Rust behavior. Report these measurements separately, including survivors. Structural invariants and semantic obligations are also different units; a total invariant count is not a measure of specification strength. @@ -171,20 +173,23 @@ job per language reads the shard reports and writes the complete report. It refuses a missing, duplicated or failed shard, shards whose source, catalog, corpus or witness fingerprints or baseline results differ, and coverage that is not the catalog exactly once in order. Only the merged report is complete -evidence; a shard report is never `complete`. Each shard's budget is 40 -minutes: the baselines plus its slice of the catalog at the slow runner's +evidence; a shard report is never `complete`. Each TypeScript/Go shard's budget +is 40 minutes: the baselines plus its slice of the catalog at the slow runner's per-mutant cost, plus one hung cohort's bound. The Go mutation runner still bounds each `go test` invocation at 8 minutes to catch a hung mutant, not to pace a slow runner. Locally, `MUTATION_SHARD=/ make mutations-ts` for every index of the workflow matrix reproduces the shards under `.formal-traces/semantic/shards/-of-/`, and `make mutations-merge-ts` assembles the report that an unsharded -`make mutations-ts` writes; the Go targets mirror this. +`make mutations-ts` writes; the Go and Rust targets mirror this. The Rust +lane compiles in release mode and bounds each cargo invocation at 25 minutes. +Its separate catalog and evidence scope are described in +[SEMANTIC-COVERAGE.md](./SEMANTIC-COVERAGE.md#reproduction-and-ci). `MUTATION_ONLY=M18 make mutations-ts` measures one mutant into a partial report for authoring; it is never complete evidence. The workflow's `formal-full` aggregate job retains a small `formal-summary` artifact for 90 -days: both completion and context reports, the Go replay summary, the model +days: every port's completion and context reports, the Go replay summary, the model properties `report.json` from the `check-models` job, the symbolic `report.json` and, on scheduled or exploration runs, each exploration `report.json`. Trace corpora, model check counterexamples and mutation evidence keep the 14-day @@ -206,7 +211,7 @@ is never reported as a failure. `make explore` selects and records a fresh seed, copies current tracked and new source files, and runs all unmodified Rust model checks and regressions, -generation, witness checks and both native replays in that isolated snapshot. +generation, witness checks and every native replay in that isolated snapshot. It omits the identical pinned model-fault campaign, which remains mandatory in `make formal-check`, `make formal` and `make ci`. It does not run the separate symbolic lane or require Java. @@ -218,14 +223,14 @@ A witness-check failure can mean an unreached boundary or invalid witness evidence; inspect the native report before attributing it to sampling. Go replay still runs after a TypeScript witness-check failure. The exploration report keeps that seed's witness report under `witnesses`. The witness step itself is -tolerated so both ports replay, but its baseline gate decides the outcome -afterwards: when every required label is present and both ports pass yet a +tolerated so every port replays, but its baseline gate decides the outcome +afterwards: when every required label is present and all ports pass yet a gated label's sampled hits collapsed against the recorded baseline (below the tolerance and more than `freshSeedSigma` Poisson deviations below the recorded count, or to zero), the report ends with `coverage-gate-failure` and the lane fails. That is a statement about exploration quality on that seed, not about native behavior. Exploration also refuses to pass without a completed witness -report for its own seed: when both ports passed but the tolerated evaluator +report for its own seed: when all ports passed but the tolerated evaluator step wrote no report, or an unreadable or incomplete one, or one judged under another seed or covering fewer profiles than the snapshot's own manifest schedules (a saved run is judged against the inventory it was saved with), the diff --git a/formal/check-rust-replay.mjs b/formal/check-rust-replay.mjs new file mode 100644 index 00000000..64c00365 --- /dev/null +++ b/formal/check-rust-replay.mjs @@ -0,0 +1,83 @@ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { root } from './execution.mjs'; + +import { conformanceInventory } from './conformance.mjs'; +import { nativeBinding } from './conformance-bindings.mjs'; + +// The Rust harness reports one JSON object per line: a start record, one +// record per executed case named by its shared inventory id, and a finish +// record with totals. Case ids are the inventory ids themselves (the Rust +// binding is the identity), so this gate reads the report against the common +// inventory directly, with no second list of required behavior. +const inventoryRoots = new Set(['sampled', 'regression', 'scenario', 'protocol', 'witness']); +const epochMs = value => Number.isSafeInteger(value) && value > 0; + +/** Pure completed-report gate. Counts only exact passed case records, never totals the harness printed. */ +export function checkRustReplay(report, inventory = conformanceInventory()) { + if (typeof report !== 'string' || !report.trim()) throw new Error('Empty Rust replay report'); + if (!Array.isArray(inventory) || !inventory.length) throw new Error('Empty Rust replay inventory'); + const required = new Map(inventory.map(entry => [nativeBinding(entry, 'rust'), entry])); + if (required.size !== inventory.length) throw new Error('Duplicate Rust case binding'); + const cases = new Map(); + let started, finish; + for (const [index, line] of report.trim().split('\n').entries()) { + let event; + try { event = JSON.parse(line); } catch { throw new Error(`Invalid Rust JSON record at line ${index + 1}`); } + if (!event || typeof event !== 'object' || Array.isArray(event) || typeof event.kind !== 'string') throw new Error(`Invalid Rust replay record at line ${index + 1}`); + if (finish) throw new Error('Rust report continues after the finish record'); + if (event.kind === 'start') { + if (started || index !== 0) throw new Error('Duplicate or misplaced Rust start record'); + if (event.schemaVersion !== 1 || event.implementation !== 'rust' || !epochMs(event.startedAt)) throw new Error('Unsupported Rust start record'); + started = event; + continue; + } + if (!started) throw new Error('Rust replay record precedes the start record'); + if (event.kind === 'case') { + const { id } = event; + if (typeof id !== 'string' || !id) throw new Error(`Invalid Rust case id at line ${index + 1}`); + if (event.status !== 'passed' && event.status !== 'failed') throw new Error(`Invalid Rust case status: ${id}`); + if (!epochMs(event.startedAt) || !epochMs(event.finishedAt) || event.finishedAt < event.startedAt || event.startedAt < started.startedAt) throw new Error(`Invalid Rust case timing: ${id}`); + if (event.message !== undefined && typeof event.message !== 'string') throw new Error(`Invalid Rust case message: ${id}`); + if (event.status === 'failed') throw new Error(`Rust replay failed: ${id}${event.message ? ` (${event.message})` : ''}`); + if (cases.has(id)) throw new Error(`Duplicate Rust replay case: ${id}`); + if (!required.has(id)) { + const segment = id.split('/')[0]; + if (segment === 'smoke') throw new Error(`Smoke history in a full Rust replay: ${id}`); + if (inventoryRoots.has(segment)) throw new Error(`Unexpected Rust replay case (inventory drift): ${id}`); + throw new Error(`Unknown Rust replay case: ${id}`); + } + cases.set(id, event); + } else if (event.kind === 'finish') { + if (event.status !== 'passed' && event.status !== 'failed') throw new Error('Invalid Rust finish status'); + if (event.status !== 'passed') throw new Error(`Rust replay finished with status ${event.status}`); + if (!epochMs(event.finishedAt) || event.finishedAt < started.startedAt) throw new Error('Invalid Rust finish timing'); + if (event.cases !== cases.size || event.failed !== 0) throw new Error(`Rust finish totals disagree with the case records: ${event.cases} reported, ${cases.size} recorded, ${event.failed} failed`); + finish = event; + } else { + throw new Error(`Unsupported Rust replay record kind: ${event.kind}`); + } + } + if (!finish) throw new Error('Rust replay is incomplete: missing finish record'); + for (const id of required.keys()) if (!cases.has(id)) throw new Error(`Missing passed Rust replay case: ${id}`); + const profiles = [...new Set(inventory.filter(entry => entry.profile).map(entry => entry.profile))]; + const count = (category, profile) => inventory.filter(entry => entry.category === category && (profile === undefined || entry.profile === profile)).length; + const generated = Object.fromEntries(profiles.map(profile => [profile, count('sampled', profile)])); + const quintRegressions = Object.fromEntries(profiles.map(profile => [profile, count('regression', profile)])); + return { schemaVersion: 1, implementation: 'rust', status: 'pass', + generated, generatedTraces: Object.values(generated).reduce((sum, n) => sum + n, 0), + quintRegressions, quintRegressionTraces: Object.values(quintRegressions).reduce((sum, n) => sum + n, 0), + fixedScenarios: count('scenario'), protocolVectors: count('protocol'), + witnessProfiles: inventory.filter(entry => entry.category === 'witness').map(entry => entry.profile).sort(), + executedCases: cases.size }; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + if (process.argv.length > 3) throw new Error('Usage: node formal/check-rust-replay.mjs [rust-replay.jsonl]'); + const path = process.argv[2] ? resolve(process.argv[2]) : root + '.formal-traces/rust-replay.jsonl'; + const report = readFileSync(path, 'utf8'); + const result = checkRustReplay(report, conformanceInventory()); + console.log(JSON.stringify({ ...result, reportSha256: createHash('sha256').update(report).digest('hex') }, null, 2)); +} diff --git a/formal/conformance-adapters.mjs b/formal/conformance-adapters.mjs index f6c86376..03f8d9f7 100644 --- a/formal/conformance-adapters.mjs +++ b/formal/conformance-adapters.mjs @@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs'; import { basename, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { checkGoReplay, loadGoReplayInventory } from './check-go-replay.mjs'; +import { checkRustReplay } from './check-rust-replay.mjs'; import { root } from './execution.mjs'; import { readJSON, digest, fingerprint, validateContext, checkCompletion } from './conformance.mjs'; @@ -51,11 +52,22 @@ export function parseGoReport(text, inventory) { }) }; } +// The Rust harness names cases by inventory id, so the strict report gate is +// also the binding check; only the execution window is read here. +export function parseRustReport(text, inventory) { + checkRustReplay(text, inventory); + const events = text.trim().split('\n').map(line => JSON.parse(line)); + const startedAt = events[0].startedAt, finishedAt = events.at(-1).finishedAt; + if (!Number.isFinite(startedAt) || !Number.isFinite(finishedAt)) throw new Error('Rust report is missing execution timestamps'); + return { startedAt, finishedAt, results: inventory.map(entry => ({ id: entry.id, status: 'passed' })) }; +} + export function adaptReport(language, text, context) { validateContext(context); if (context.language !== language) throw new Error('Wrong port context'); const parsed = language === 'typescript' ? parseTypeScriptReport(text, context.inventory) : language === 'go' - ? parseGoReport(text, context.inventory) : (() => { throw new Error('Unsupported native report adapter'); })(); + ? parseGoReport(text, context.inventory) : language === 'rust' ? parseRustReport(text, context.inventory) + : (() => { throw new Error('Unsupported native report adapter'); })(); const report = { schemaVersion: 1, language, runId: context.runId, contextSha256: fingerprint(context), ...parsed, status: 'passed', nativeReportSha256: digest(text) }; checkCompletion(report, context); @@ -63,6 +75,6 @@ export function adaptReport(language, text, context) { } if (process.argv[1] === fileURLToPath(import.meta.url)) { const [language, nativePath, contextPath, ...extra] = process.argv.slice(2); - if (extra.length || !language || !nativePath || !contextPath) throw new Error('Usage: node formal/conformance-adapters.mjs '); + if (extra.length || !language || !nativePath || !contextPath) throw new Error('Usage: node formal/conformance-adapters.mjs '); console.log(JSON.stringify(adaptReport(language, readFileSync(resolve(root, nativePath), 'utf8'), readJSON(contextPath)), null, 2)); } diff --git a/formal/conformance-bindings.mjs b/formal/conformance-bindings.mjs index 3af69129..9efe96f8 100644 --- a/formal/conformance-bindings.mjs +++ b/formal/conformance-bindings.mjs @@ -37,7 +37,10 @@ export function nativeBinding(entry, language, workspace = root) { if (entry.category === 'protocol') return `${protocolGoRoots[entry.group] ?? 'TestProtocolRemainingVectors'}/${goName(entry.name)}`; return `TestGeneratedWitnessEvidence/${entry.profile}`; } - if (language !== 'typescript') throw new Error('Native report adapter is only supplied for TypeScript and Go'); + // The Rust harness names every case by its shared inventory id, so the + // binding is the identity: the report is read against the inventory directly. + if (language === 'rust') return entry.id; + if (language !== 'typescript') throw new Error('Native report adapter is only supplied for TypeScript, Go and Rust'); if (entry.category === 'sampled' || entry.category === 'regression') return [profileFile(entry.profile), `${profileSuite(entry.profile)} replays ${resolve(workspace, entry.path)}`]; if (entry.category === 'scenario') return ['formal-behavior.test.ts', `portable behavioral scenarios ${entry.feature}: ${entry.name}`]; diff --git a/formal/conformance.mjs b/formal/conformance.mjs index de39c39d..889e5f1c 100644 --- a/formal/conformance.mjs +++ b/formal/conformance.mjs @@ -53,6 +53,13 @@ export function defaultSources(language) { ...filesBelow('go').filter(path => /\.(go|ts)$/.test(path) || /\/go\.(mod|sum)$/.test(path)), ...readExecution().models.filter(model => model.profile && model.profile !== 'core') .map(model => `.formal-traces/go-parity-witnesses/${model.profile}.json`)]; + // Rust binds the same shared fixtures, its crate sources, manifests and + // lockfile, and the same witness evidence the Go replay consumes. The build + // directory is a cache, never an input. + if (language === 'rust') return [...shared, + ...filesBelow('rust').filter(path => !path.startsWith('rust/target/') && /\.(rs|toml|lock)$/.test(path)), + ...readExecution().models.filter(model => model.profile && model.profile !== 'core') + .map(model => `.formal-traces/go-parity-witnesses/${model.profile}.json`)]; fail('New languages must supply an explicit JSON list of implementation and harness source paths'); } function hashes(paths) { @@ -99,7 +106,7 @@ export function validateContext(context, { current = true } = {}) { if (!isDeepStrictEqual(context.implementation, hashes(Object.keys(context.implementation)))) fail('Implementation inputs changed during run'); // Default bindings must include new files too; custom port inventories are // an explicit, reviewable declaration of the complete execution inputs. - if (['typescript', 'go'].includes(context.language) && !isDeepStrictEqual(Object.keys(context.implementation).sort(), defaultSources(context.language).sort())) fail('Implementation source inventory changed during run'); + if (['typescript', 'go', 'rust'].includes(context.language) && !isDeepStrictEqual(Object.keys(context.implementation).sort(), defaultSources(context.language).sort())) fail('Implementation source inventory changed during run'); if (!isDeepStrictEqual(context.corpus, corpusInputs(context.inventory))) fail('Shared corpus changed during run'); } return context; diff --git a/formal/explore.mjs b/formal/explore.mjs index c1e099b4..f7f1e374 100644 --- a/formal/explore.mjs +++ b/formal/explore.mjs @@ -7,13 +7,14 @@ import { cleanEnvironment, executeSteps, validationPlan } from './validation.mjs import { nativeBinding } from './conformance-bindings.mjs'; import { parseTypeScriptReport } from './conformance-adapters.mjs'; import { checkGoReplay } from './check-go-replay.mjs'; +import { checkRustReplay } from './check-rust-replay.mjs'; import { canonicalSeed, reportFileName } from './witnesses.mjs'; const root = fileURLToPath(new URL('../', import.meta.url)); const hash = value => createHash('sha256').update(value).digest('hex'); const inside = (directory, path) => path.startsWith(directory + sep); -const reportPaths = { typescript: '.formal-traces/ts-replay.json', go: '.formal-traces/go-replay.jsonl' }; -const contextPaths = { typescript: '.formal-traces/ts-context.json', go: '.formal-traces/go-context.json' }; +const reportPaths = { typescript: '.formal-traces/ts-replay.json', go: '.formal-traces/go-replay.jsonl', rust: '.formal-traces/rust-replay.jsonl' }; +const contextPaths = { typescript: '.formal-traces/ts-context.json', go: '.formal-traces/go-context.json', rust: '.formal-traces/rust-context.json' }; export function explorationSeed(value = `0x${randomBytes(8).toString('hex')}`) { try { return canonicalSeed(value); } @@ -21,7 +22,7 @@ export function explorationSeed(value = `0x${randomBytes(8).toString('hex')}`) { } // Share model generation and native execution with acceptance. Exploration has -// its own report: a seed's missing witness must not prevent the other port from +// its own report: a seed's missing witness must not prevent the other ports from // executing the histories. No acceptance completion/adaptation step runs here. export function explorationPlan(directory, seed, options = {}) { const normalized = explorationSeed(seed); @@ -30,16 +31,16 @@ export function explorationPlan(directory, seed, options = {}) { // This campaign uses the manifest's pinned seed, not the exploration seed. // Full acceptance keeps it; exploration retains every unmodified model job. if (script === 'formal/check-model-properties.mjs') return []; - if (step.remove || ['formal/conformance-adapters.mjs', 'formal/check-go-replay.mjs'].includes(script) + if (step.remove || ['formal/conformance-adapters.mjs', 'formal/check-go-replay.mjs', 'formal/check-rust-replay.mjs'].includes(script) || script === 'formal/conformance.mjs' && step.args[1] === 'check') return []; if (script === 'formal/conformance.mjs' && step.args[1] === 'prepare') { return [{ label: `Prepare exploratory ${step.args[2]} context`, explorationContext: step.args[2] }]; } if (script === 'formal/run-models.mjs') return [{ ...step, env: { ...step.env, QUINT_SEED: normalized } }]; - if (step.env?.DIALCACHE_MBT_TRACE_DIR) return [{ ...step, nativeReport: step.command === 'go' ? 'go' : 'typescript' }]; - // A seed's missing witness is classified by both native reports. The shared - // evaluator runs before either replay and still writes evidence for complete - // profiles; its exit status must not stop either port from executing that + if (step.env?.DIALCACHE_MBT_TRACE_DIR) return [{ ...step, nativeReport: step.command === 'go' ? 'go' : step.command === 'cargo' ? 'rust' : 'typescript' }]; + // A seed's missing witness is classified by every native report. The shared + // evaluator runs before native replay and still writes evidence for complete + // profiles; its exit status must not stop any port from executing that // seed's histories. // The evaluator learns the corpus seed from the same variable run-models.mjs // reads, so its baseline gate applies the exploration rule to this seed. @@ -158,6 +159,22 @@ export function nativeExplorationResult(language, text, context, directory, pack required: inventory.map(entry => ({ name: nativeBinding(entry, language), category: entry.category })) }; checkGoReplay(events.map(event => JSON.stringify(event.Action === 'fail' ? { ...event, Action: 'pass' } : event)).join('\n'), native); startedAt = Date.parse(events[0].Time); finishedAt = Date.parse(events.at(-1).Time); + } else if (language === 'rust') { + // The Rust report names cases by inventory id; a failed case record is the + // native counterexample. The all-passed copy reuses the strict report gate. + const records = text.trim().split('\n').map(line => JSON.parse(line)); + const cases = new Map(inventory.map(entry => [nativeBinding(entry, language), entry])); + for (const record of records) { + if (record.kind !== 'case' || record.status !== 'failed') continue; + const entry = cases.get(record.id); + if (entry) failed.push(entry); else otherFailures.push(record.id); + } + const finish = records.at(-1); + if (finish?.kind !== 'finish') throw new Error('Rust report has no finish record: the harness crashed or timed out before completing.'); + if ((finish.status === 'failed') !== (failed.length + otherFailures.length > 0)) throw new Error('Rust report status disagrees with its case records.'); + checkRustReplay(records.map(record => JSON.stringify(record.kind === 'case' ? { ...record, status: 'passed', message: undefined } + : record.kind === 'finish' ? { ...record, status: 'passed', failed: 0 } : record)).join('\n'), inventory); + startedAt = records[0]?.startedAt; finishedAt = finish.finishedAt; } else throw new Error('Unsupported exploratory port.'); if (!Number.isFinite(startedAt) || !Number.isFinite(finishedAt) || startedAt < context.createdAt || finishedAt < startedAt || finishedAt > Date.now() + 60_000) throw new Error('Stale or invalid native report timestamps.'); @@ -327,9 +344,9 @@ async function executeExploration(seed, { directory = root, environment = proces }); verifyHashes(workspace, [report.sources]); report.sourcesUnchanged = true; - if (report.native.map(result => result.language).sort().join() !== 'go,typescript' + if (report.native.map(result => result.language).sort().join() !== 'go,rust,typescript' || report.native.some(result => !['passed', 'native-failure', 'witness-check-failure'].includes(result.status))) { - throw new Error('Exploration did not finish both native ports.'); + throw new Error('Exploration did not finish every native port.'); } // The witness step is tolerated so both ports replay, but its baseline // gate still decides the outcome afterwards: a fresh seed whose sampled diff --git a/formal/go-parity.json b/formal/go-parity.json index 7645cd7a..6dc88cdb 100644 --- a/formal/go-parity.json +++ b/formal/go-parity.json @@ -10,10 +10,10 @@ "inputs": { "semanticCasesSha256": "faa768fbf06eddc03dc4f26570094b1df066fb075f898043b40c6ce9d8c952a1", "executionSha256": "3df06c53eb69f02fb6eafabd51c6cc7a306c6739e04ab3fa521b53389da559f2", - "sourceAuditSha256": "2b749b2b0a17cc7134108fe939fdb7d7792c19883c591379af4b66f2135cd699", + "sourceAuditSha256": "1634e1c839d88babf762b44aaf2db9e0ea514409b2689581327fdf4a00fd40d9", "featureCoverageSha256": "91723cc5398c8fff102d2beb199ebcab82af137e11f1cb9771b8d16d0304a95c", "coverageWitnessesSha256": "2b2d5d57daacba3ecd3637f5a917d58b0b63fcc88893e651cd602cdba1ef4244", - "profilesSha256": "f8ec93fe5aa23faa69edd0a632a645874a6190a48ed613d2c65f82045219f49c" + "profilesSha256": "2ebb3ba9c9b71334bcbbaa1e2d9c1d539664ed69e826942ff829a2c31fe9434b" }, "inventory": { "behavioralCases": 244, diff --git a/formal/measure-rust-semantics.mjs b/formal/measure-rust-semantics.mjs new file mode 100644 index 00000000..8d8b3ab6 --- /dev/null +++ b/formal/measure-rust-semantics.mjs @@ -0,0 +1,293 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { resolve, relative } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { causalPropertyAssertion } from './measure-go-semantics.mjs'; +import { mutantCatalogPath, mutantsForPort, readMutantCatalog } from './execution.mjs'; +import { fingerprintFiles, gateDetections, languages, selectMutations, selectionDirectory, selectionFromArguments } from './mutation-reports.mjs'; + +// The Rust counterpart of measure-go-semantics.mjs: apply each catalogued +// fault to an isolated copy of the crate, require it to compile, and run three +// cohorts against it. Detection is an assertion failure; a compiler error, +// crash, timeout, missing report or incomplete run fails the measurement. +const root = fileURLToPath(new URL('../', import.meta.url)); +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const json = file => JSON.parse(readFileSync(file, 'utf8')); + +// Test binaries that are the harness itself, its negative control, harness +// infrastructure, the protocol vector suites (the harness replays those +// vectors in the generated and fixed cohorts) or real servers are not +// ordinary tests. +export const infrastructureTestFile = /^(?:conformance|settlement_control|harness_infra|redis_integration|protocol_\w+)\.rs$/; + +// libtest prints one `test ... ` line per test and one +// `test result:` line per binary; `cargo test --no-fail-fast` runs every +// selected binary even after one fails. Names are qualified by their binary +// because unit and integration tests may share module paths. +export function evaluateCargoTestOutput(output, exitCode, expectedBinaries) { + const tests = new Map(); + let binary = '', results = 0, passed = 0, failed = 0; + for (const line of output.split('\n')) { + const running = /^\s*Running (?:unittests )?(\S+)/.exec(line); + if (running) { binary = running[1]; continue; } + const test = /^test (\S+) \.\.\. (ok|FAILED|ignored)/.exec(line); + if (test) { + const name = `${binary}::${test[1]}`; + if (tests.has(name)) throw new Error(`duplicate test execution ${name}`); + tests.set(name, test[2]); + continue; + } + const result = /^test result: (ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored/.exec(line); + if (result) { results += 1; passed += Number(result[2]); failed += Number(result[3]); } + } + if (!results) throw new Error('no libtest result line: the ordinary cohort did not run'); + if (expectedBinaries !== undefined && results !== expectedBinaries) throw new Error(`expected ${expectedBinaries} test binaries to report, saw ${results}`); + const failingTests = [...tests].filter(([, outcome]) => outcome === 'FAILED').map(([name]) => name); + if (failed !== failingTests.length) throw new Error('libtest totals disagree with the listed outcomes'); + if (exitCode !== 0 && !failed) throw new Error(`cargo test exited ${exitCode} without a failing test: infrastructure failure`); + if (exitCode === 0 && failed) throw new Error('cargo test exited 0 with failing tests'); + return { state: failed ? 'detected' : 'survived', passed, failed, failingTests, + executedTests: [...tests].filter(([, outcome]) => outcome !== 'ignored').map(([name]) => name) }; +} + +// The harness writes one JSON object per line: a start header, one case per +// inventory id and a finish footer. A missing footer means the run crashed or +// timed out and is not evidence either way. +export function evaluateRustReport(text, exitCode, stderr = '') { + if (/^conformance harness failed:|^coverage:/m.test(stderr)) throw new Error('Rust harness infrastructure or coverage failure'); + const records = text.trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); + if (!records.length) throw new Error('empty Rust harness report'); + const [start] = records; + if (start.kind !== 'start' || start.schemaVersion !== 1 || start.implementation !== 'rust') throw new Error('Rust harness report lacks its start header'); + const finish = records.at(-1); + if (finish.kind !== 'finish') throw new Error('Rust harness report has no finish record: the run crashed or timed out'); + const cases = records.slice(1, -1); + const seen = new Set(); + const assertionKinds = {}, assertionEvidence = {}; + for (const record of cases) { + if (record.kind !== 'case' || typeof record.id !== 'string' || !['passed', 'failed'].includes(record.status)) throw new Error('malformed Rust harness case record'); + if (seen.has(record.id)) throw new Error(`duplicate case ${record.id}`); + seen.add(record.id); + const category = record.id.split('/')[0]; + if (!['sampled', 'regression', 'scenario', 'protocol', 'witness'].includes(category)) throw new Error(`unknown Rust case ${record.id}`); + if (record.status === 'failed') { + if (category === 'witness') throw new Error(`witness audit failure ${record.id}`); + const message = record.message; + if (typeof message !== 'string' || !message) throw new Error(`failure has no assertion evidence: ${record.id}`); + if (category === 'protocol') { + if (!/PROTOCOL_ASSERTION_FAILURE expected:[\s\S]*actual:/.test(message)) throw new Error(`protocol failure lacks assertion evidence: ${record.id}`); + assertionKinds[record.id] = 'protocol-assertion'; + } else if (/expected:[\s\S]*actual:/.test(message)) assertionKinds[record.id] = 'observation-mismatch'; + else if (causalPropertyAssertion(message)) assertionKinds[record.id] = 'causal-property'; + else throw new Error(`replay failure lacks observation or validated causal property evidence: ${record.id}`); + assertionEvidence[record.id] = message; + } + } + if (!cases.length) throw new Error('Rust harness report has no cases'); + const failingTests = cases.filter(record => record.status === 'failed').map(record => record.id); + if (finish.cases !== cases.length) throw new Error('finish totals disagree with the case records'); + if (finish.failed !== failingTests.length) throw new Error('finish totals disagree with the failed cases'); + if (finish.status !== (failingTests.length ? 'failed' : 'passed')) throw new Error('finish status disagrees with the case records'); + if (exitCode === 0 && failingTests.length) throw new Error('harness exited 0 with failed cases'); + if (exitCode !== 0 && !failingTests.length) throw new Error(`harness exited ${exitCode} without a failed case: coverage or infrastructure failure`); + return { state: failingTests.length ? 'detected' : 'survived', passed: cases.length - failingTests.length, failed: failingTests.length, failingTests, assertionKinds, assertionEvidence, + executedTests: cases.map(record => record.id) }; +} + +function union(generated, fixed) { + return { state: generated.failed + fixed.failed ? 'detected' : 'survived', passed: generated.passed + fixed.passed, + failed: generated.failed + fixed.failed, failingTests: [...generated.failingTests, ...fixed.failingTests], components: ['generated', 'fixed'] }; +} + +// This native catalog is a deliberately smaller set than the current shared +// TypeScript/Go catalog. Record the gap instead of claiming complete coverage +// of its model-challenge boundary recordings. +export function rustMutationScope(catalog, typescript) { + if (catalog.schemaVersion !== 1 || !Array.isArray(catalog.mutations) || catalog.mutations.length < 13) throw new Error('expected versioned Rust fault catalog with its 13 baseline faults'); + const ids = new Set(), counterparts = new Set(); + for (const mutation of catalog.mutations) { + if (!/^M\d+$/.test(mutation.id) || ids.has(mutation.id) || counterparts.has(mutation.typescriptMutation)) throw new Error('invalid/duplicate Rust mutation or counterpart ID'); + ids.add(mutation.id); counterparts.add(mutation.typescriptMutation); + const counterpart = typescript.find(item => item.id === mutation.typescriptMutation); + if (!counterpart || counterpart.case !== mutation.case) throw new Error(`invalid TypeScript counterpart ${mutation.id}`); + } + return { catalog: 'rust-native', modelBoundaryEvidence: false, + mappedMutations: [...counterparts], + unmappedSharedMutations: typescript.filter(item => !counterparts.has(item.id)).map(item => item.id) }; +} + +// Concurrent shards must never execute a sibling shard's mutated binaries. +// Keep the Cargo build cache separate by the same selection used for reports. +export function rustTargetDirectory(directory, selection) { + return selectionDirectory(resolve(directory, 'rust/target/semantic'), selection); +} + +export function measureRustSemantics({ shard = { index: 1, count: 1 }, only } = {}) { + const language = languages.rust; + const reportRoot = resolve(root, language.output); + const selection = { shard, only }; + const output = selectionDirectory(reportRoot, selection); + const started = Date.now(); + mkdirSync(reportRoot, { recursive: true }); + const report = { schemaVersion: 1, complete: false, ...(shard.count > 1 ? { shard: { index: shard.index, count: shard.count } } : {}), startedAt: new Date(started).toISOString(), baselines: {}, mutations: [] }; + const save = () => { report.elapsedSeconds = Math.round((Date.now() - started) / 1000); writeFileSync(resolve(output, 'report.json'), JSON.stringify(report, null, 2) + '\n'); }; + if (output !== reportRoot) { + if (only === undefined) writeFileSync(resolve(reportRoot, 'report.json'), JSON.stringify({ schemaVersion: 1, complete: false, startedAt: report.startedAt }, null, 2) + '\n'); + rmSync(output, { recursive: true, force: true }); + mkdirSync(output, { recursive: true }); + } + save(); + if (only === undefined) rmSync(resolve(reportRoot, 'report.md'), { force: true }); + const workspace = mkdtempSync(resolve(tmpdir(), 'dialcache-rust-semantic-')); + const cargo = process.env.CARGO_BIN ?? 'cargo'; + // Bounds a hung mutant, not a slow runner: one release build of the crate + // and its test binaries plus the 7,000-case replay take a few minutes on a + // hosted runner. One timeout aborts the whole measurement. + const timeout = 1_500_000; + try { + // Copies keep repo-relative paths while mutations stay outside the shared + // checkout; the crate's build output is not an input and is not copied. + // go/ is copied because a crate unit test compares the invalidation + // script with go/redis_adapter.go byte for byte. + for (const path of ['formal', 'test', 'src', 'go']) cpSync(resolve(root, path), resolve(workspace, path), { recursive: true }); + const target = resolve(root, 'rust/target'); + cpSync(resolve(root, 'rust'), resolve(workspace, 'rust'), { recursive: true, filter: source => source !== target && !source.startsWith(`${target}/`) }); + const crate = resolve(workspace, 'rust'); + const catalogPath = resolve(workspace, language.catalog); + const catalog = json(catalogPath); + const typescript = mutantsForPort(readMutantCatalog(path => readFileSync(resolve(workspace, path), 'utf8')), 'typescript'); + report.scope = rustMutationScope(catalog, typescript); + report.sharedCatalogSha256 = hash(readFileSync(resolve(workspace, mutantCatalogPath))); + const originals = new Map(), ids = new Set(); + for (const mutation of catalog.mutations) { + if (!/^M\d+$/.test(mutation.id) || ids.has(mutation.id)) throw new Error('invalid/duplicate mutation ID'); + ids.add(mutation.id); + if (!Array.isArray(mutation.edits) || !mutation.edits.length || !mutation.requiredDetections?.every(name => ['ordinary', 'generated', 'fixed', 'portable'].includes(name))) throw new Error(`invalid mutation ${mutation.id}`); + for (const edit of mutation.edits) { + if (!/^rust\/src\/[\w/-]+\.rs$/.test(edit.path) || !edit.before || edit.before === edit.after) throw new Error(`invalid production edit ${mutation.id}`); + const original = readFileSync(resolve(workspace, edit.path), 'utf8'); + if (original.split(edit.before).length !== 2) throw new Error(`${mutation.id}: anchor must occur exactly once; review source drift in ${edit.path}`); + originals.set(edit.path, original); + } + } + const selected = selectMutations(catalog.mutations, selection); + if (shard.count > 1) report.shard = { index: shard.index, count: shard.count, mutationIds: selected.map(m => m.id) }; + // Ordinary tests: the library's unit tests and every integration binary + // that is not harness infrastructure, discovered from the crate so a new + // native test file joins the cohort automatically. + const ordinaryFiles = readdirSync(resolve(crate, 'tests')).filter(file => file.endsWith('.rs') && !infrastructureTestFile.test(file)).sort(); + if (!ordinaryFiles.length) throw new Error('no ordinary Rust integration tests found'); + const ordinaryTargets = ['--lib', ...ordinaryFiles.flatMap(file => ['--test', file.replace(/\.rs$/, '')])]; + const witnessDirectory = resolve(process.env.DIALCACHE_WITNESS_EVIDENCE_DIR ?? resolve(root, '.formal-traces/go-parity-witnesses')); + const corpus = { + DIALCACHE_MBT_TRACE_DIR: resolve(root, '.formal-traces/conformance'), + DIALCACHE_EFFECTS_TRACE_DIR: resolve(root, '.formal-traces/effects'), + DIALCACHE_FEATURE_TRACE_DIR: resolve(root, '.formal-traces/features'), + DIALCACHE_WITNESS_EVIDENCE_DIR: witnessDirectory, + }; + const cohorts = { + ordinary: { targets: ordinaryTargets }, + generated: { suite: 'generated', protocolCorpus: 'generated', env: corpus }, + fixed: { suite: 'fixed', protocolCorpus: 'fixed', env: {} }, + }; + const env = { ...process.env }; + for (const name of Object.keys(env)) if (name.startsWith('DIALCACHE_')) delete env[name]; + // Dependencies build once and are shared by every mutant and run through + // the checkout's cache directory; the crate copy itself lives at another + // path, so cargo rebuilds exactly the crate and its tests per mutant. + env.CARGO_TARGET_DIR = process.env.DIALCACHE_RUST_TARGET_DIR ?? rustTargetDirectory(root, selection); + env.CARGO_TERM_COLOR = 'never'; + report.revision = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).stdout.trim(); + report.cargo = spawnSync(cargo, ['--version'], { cwd: crate, encoding: 'utf8' }).stdout?.trim(); + report.node = process.version; + report.catalogSha256 = hash(readFileSync(catalogPath)); + report.inputs = fingerprintFiles(workspace, language.inputs, { exclude: language.exclude }); + report.corpus = fingerprintFiles(root, ['.formal-traces/conformance', '.formal-traces/effects', '.formal-traces/features', '.formal-traces/regressions']); + report.witnesses = fingerprintFiles(witnessDirectory, ['.']); + report.sourceSha256 = Object.fromEntries([...originals].map(([path, text]) => [path, hash(text)])); + report.selections = { ordinary: ['lib', ...ordinaryFiles.map(file => file.replace(/\.rs$/, ''))], generated: 'conformance harness: DIALCACHE_RUST_SUITE=generated over the complete corpus, witness evidence and generated vectors', + fixed: 'conformance harness: DIALCACHE_RUST_SUITE=fixed over the fixed scenarios and checked-in vectors' }; + report.ordinaryFiles = ordinaryFiles; + const spawnCargo = (args, extraEnv) => spawnSync(cargo, args, { cwd: crate, env: { ...env, ...extraEnv }, encoding: 'utf8', timeout, maxBuffer: 256 * 1024 * 1024 }); + const compile = label => { + const result = spawnCargo(['test', '--release', '--all-features', '--no-run']); + writeFileSync(resolve(output, `${label}-compile.log`), (result.stdout ?? '') + (result.stderr ?? '')); + if (result.error || result.signal || result.status !== 0) throw new Error(`${label}: noncompiling mutant/baseline, not detection; see compile log`); + }; + const run = (label, cohort, baseline) => { + let parsed; + if (cohort === 'ordinary') { + const result = spawnCargo(['test', '--release', '--all-features', '--no-fail-fast', ...cohorts.ordinary.targets]); + writeFileSync(resolve(output, `${label}-${cohort}.log`), result.stdout ?? ''); + writeFileSync(resolve(output, `${label}-${cohort}.stderr.log`), result.stderr ?? ''); + if (result.error || result.signal) throw new Error(`${label}/${cohort}: runner infrastructure failed: ${result.error ?? result.signal}`); + parsed = evaluateCargoTestOutput(result.stdout ?? '', result.status, 1 + ordinaryFiles.length); + } else { + const reportPath = resolve(output, `${label}-${cohort}.jsonl`); + rmSync(reportPath, { force: true }); + const result = spawnCargo(['test', '--release', '--all-features', '--test', 'conformance'], + { ...cohorts[cohort].env, DIALCACHE_RUST_SUITE: cohorts[cohort].suite, DIALCACHE_PROTOCOL_CORPUS: cohorts[cohort].protocolCorpus, DIALCACHE_RUST_REPORT: reportPath }); + writeFileSync(resolve(output, `${label}-${cohort}.log`), result.stdout ?? ''); + writeFileSync(resolve(output, `${label}-${cohort}.stderr.log`), result.stderr ?? ''); + if (result.error || result.signal) throw new Error(`${label}/${cohort}: runner infrastructure failed: ${result.error ?? result.signal}`); + parsed = evaluateRustReport(existsSync(reportPath) ? readFileSync(reportPath, 'utf8') : '', result.status, result.stderr ?? ''); + } + if (baseline && parsed.failed) throw new Error(`${cohort}: unmodified baseline must pass; see baseline log`); + if (!baseline && JSON.stringify([...parsed.executedTests].sort()) !== JSON.stringify([...report.baselines[cohort].executedTests].sort())) throw new Error(`${label}/${cohort}: incomplete mutation run`); + writeFileSync(resolve(output, `${label}-${cohort}.json`), JSON.stringify(parsed, null, 2) + '\n'); + return parsed; + }; + // Every shard compiles and measures every baseline itself: its evidence + // stands on the environment it ran in, and the merge refuses shards whose + // baselines differ. + compile('baseline'); + for (const cohort of Object.keys(cohorts)) { + report.baselines[cohort] = run('baseline', cohort, true); + console.log(`baseline ${cohort}: ${report.baselines[cohort].passed} passing tests`); save(); + } + report.baselines.portable = union(report.baselines.generated, report.baselines.fixed); + for (const mutation of selected) { + const editedPaths = new Set(); + try { + for (const edit of mutation.edits) { + const path = resolve(workspace, edit.path), current = readFileSync(path, 'utf8'); + if (current.split(edit.before).length !== 2) throw new Error(`${mutation.id}: overlapping edits`); + writeFileSync(path, current.replace(edit.before, edit.after)); editedPaths.add(edit.path); + } + compile(mutation.id); + const result = { id: mutation.id, case: mutation.case, description: mutation.description, cohorts: {} }; + for (const cohort of Object.keys(cohorts)) result.cohorts[cohort] = run(mutation.id, cohort, false); + result.cohorts.portable = union(result.cohorts.generated, result.cohorts.fixed); + report.mutations.push(result); + console.log(`${mutation.id}: ${Object.entries(result.cohorts).map(([name, value]) => `${name}=${value.state}(${value.failed})`).join(', ')}`); save(); + } finally { for (const path of editedPaths) writeFileSync(resolve(workspace, path), originals.get(path)); } + } + if (only) { + // A filtered run is a local diagnostic, never evidence: it stays incomplete + // and is not gated. + report.partial = only; + save(); + console.log(`Measured ${selected.length} selected mutations without gating: ${relative(root, output)}/report.json`); + return report; + } + if (shard.count > 1) { + gateDetections(language, report, selected, { directory: root, summarize: false }); + save(); + console.log(`Shard ${shard.index}/${shard.count} measured ${selected.length} mutations: ${relative(root, output)}/report.json; merge with node formal/merge-mutation-reports.mjs rust`); + return report; + } + gateDetections(language, report, catalog.mutations, { directory: root }); + save(); + writeFileSync(resolve(output, 'report.md'), language.markdown(report)); + return report; + } catch (error) { report.error = String(error); save(); throw error; } + finally { rmSync(workspace, { recursive: true, force: true }); } +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + measureRustSemantics(selectionFromArguments(process.argv.slice(2))); + } catch (error) { console.error(error); process.exitCode = 1; } +} diff --git a/formal/merge-mutation-reports.mjs b/formal/merge-mutation-reports.mjs index f21704f0..c595cf14 100644 --- a/formal/merge-mutation-reports.mjs +++ b/formal/merge-mutation-reports.mjs @@ -135,7 +135,7 @@ export function readShardReports(directory) { // Any refusal leaves an incomplete report with the reason in its place. export function mergeMutationReports(name, { directory = root, shardsDirectory, outputDirectory } = {}) { const language = languages[name]; - if (!language) throw new Error(`Expected language ts or go; got ${name}`); + if (!language) throw new Error(`Expected language ts, go or rust; got ${name}`); const output = resolve(directory, outputDirectory ?? language.output); const shards = resolve(directory, shardsDirectory ?? resolve(output, 'shards')); const startedAt = new Date().toISOString(); @@ -147,9 +147,12 @@ export function mergeMutationReports(name, { directory = root, shardsDirectory, try { const catalogText = readFileSync(resolve(directory, language.catalog)); const catalog = JSON.parse(catalogText); - // The gate reads this port's section of each catalog entry through the one catalog reader. - const entries = mutantsForPort(readMutantCatalog(path => readFileSync(resolve(directory, path), 'utf8')), language.port); - merged = mergeShardReports(language, readShardReports(shards), { catalog, catalogSha256: sha256(catalogText), inputs: fingerprintFiles(directory, language.inputs) }); + // Rust keeps its native catalog; shared TypeScript/Go entries use the + // canonical port reader introduced on main. + const entries = language.port + ? mutantsForPort(readMutantCatalog(path => readFileSync(resolve(directory, path), 'utf8')), language.port) + : catalog.mutations; + merged = mergeShardReports(language, readShardReports(shards), { catalog, catalogSha256: sha256(catalogText), inputs: fingerprintFiles(directory, language.inputs, { exclude: language.exclude }) }); gateDetections(language, merged, entries, { directory }); } catch (error) { const failed = merged ?? { schemaVersion: 1, complete: false, startedAt }; @@ -166,7 +169,7 @@ export function mergeMutationReports(name, { directory = root, shardsDirectory, if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { const [name, shardsDirectory, ...extra] = process.argv.slice(2); if (!languages[name] || extra.length) { - console.error('Usage: node formal/merge-mutation-reports.mjs [shard-directory]'); + console.error('Usage: node formal/merge-mutation-reports.mjs [shard-directory]'); process.exitCode = 2; } else { try { diff --git a/formal/mutation-reports.mjs b/formal/mutation-reports.mjs index 8a8e67c6..b9e09add 100644 --- a/formal/mutation-reports.mjs +++ b/formal/mutation-reports.mjs @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { readFileSync, readdirSync } from 'node:fs'; +import { readFileSync, readdirSync, statSync } from 'node:fs'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { boundaryEvidence, challengesByMutant, mutantCatalogPath, mutantIdPattern, mutantPorts } from './execution.mjs'; @@ -115,12 +115,17 @@ export function selectionDirectory(output, { shard, only }) { // Hash the reviewed inputs, including uncommitted edits and exact file bytes. // Git revision alone cannot identify an exploratory run from a dirty worktree. -export function fingerprintFiles(directory, paths) { +// `exclude` names build output beneath an input (the Rust target directory) +// that is neither reviewed nor stable. +export function fingerprintFiles(directory, paths, { exclude = [] } = {}) { const files = []; + const skipped = new Set(exclude); const visit = path => { + if (skipped.has(path)) return; + if (statSync(resolve(directory, path)).isFile()) { files.push(path); return; } for (const entry of readdirSync(resolve(directory, path), { withFileTypes: true })) { if (entry.isDirectory()) visit(`${path}/${entry.name}`); - else if (entry.isFile()) files.push(`${path}/${entry.name}`); + else if (entry.isFile() && !skipped.has(`${path}/${entry.name}`)) files.push(`${path}/${entry.name}`); } }; for (const path of paths) visit(path); @@ -163,6 +168,7 @@ export function typescriptDetection(mutations, cases) { return { all: score(mutations), behavioral: score(mutations.filter(m => !protocol(m))), protocol: score(mutations.filter(protocol)) }; } +// The four-cohort score shared by the Go and Rust measurements. export function goDetection(mutations) { return Object.fromEntries(mutantPorts.go.cohorts.map(cohort => { const measured = mutations.filter(m => m.cohorts[cohort].state !== 'crashed'); @@ -261,6 +267,16 @@ function goMarkdown(report, directory = root) { 'Challenges are the model property challenges whose fault the mutant injects natively (nativeMutants in formal/execution.json). Full JSON records snapshot/corpus/witness fingerprints, selected tests, actual passing/failing leaf counts, and assertion diagnostics. Compilation errors, crashes, timeouts, missing witnesses, and skipped executions cannot count as detections.', ''].join('\n'); } +function rustMarkdown(report) { + return ['# Rust semantic mutation measurement', '', `Completed in ${report.elapsedSeconds}s. Counts measure this named fault catalog and exact corpus, not universal equivalence.`, '', + ...shardsMarkdown(report), + 'Scope: Rust native mutation catalog only. Shared TypeScript/Go model-challenge boundary coverage is not measured by this report.', '', + ...(report.scope ? [`Shared catalog mutants without a Rust binding: ${report.scope.unmappedSharedMutations.join(', ') || 'none'}.`, ''] : []), + '| Mutation | Contract case | Ordinary | Quint generated | Fixed supplement | Full portable |', '| --- | --- | --- | --- | --- | --- |', + ...report.mutations.map(m => `| ${m.id} | ${m.case} | ${m.cohorts.ordinary.state} | ${m.cohorts.generated.state} | ${m.cohorts.fixed.state} | ${m.cohorts.portable.state} |`), '', + 'Ordinary is the crate\'s unit and native tests; generated is the conformance harness over the complete corpus and witness evidence; fixed is the harness over the fixed scenarios and checked-in vectors. Full JSON records snapshot/corpus/witness fingerprints, cohort selections, actual passing/failing counts and assertion diagnostics. Compilation errors, crashes, timeouts, missing reports and incomplete runs cannot count as detections.', ''].join('\n'); +} + // Only a merged report carries `shards`; the single run's markdown is unchanged. function shardsMarkdown(report) { if (!report.shards) return []; @@ -277,6 +293,10 @@ export const languages = { markdown: typescriptMarkdown, recordsRegressions: false }, go: { name: 'Go', port: 'go', output: '.formal-traces/go-semantic', catalog: mutantCatalogPath, inputs: ['formal', 'go', 'test', 'src'], detection: mutations => goDetection(mutations), markdown: goMarkdown, recordsRegressions: true }, + // The crate's unit tests read go/redis_adapter.go (script byte equality), so + // the Go source is an input of the ordinary cohort as well. + rust: { name: 'Rust', boundaryEvidence: false, output: '.formal-traces/rust-semantic', catalog: 'formal/rust-mutations.json', inputs: ['formal', 'rust', 'test', 'src', 'go'], exclude: ['rust/target'], + detection: mutations => goDetection(mutations), markdown: rustMarkdown, recordsRegressions: true }, }; // Historical inspection may read a report anywhere. Gated inspection instead @@ -316,7 +336,8 @@ export function gateDetections(language, report, entries, { directory = root, su // Recompute verdicts from the current declarations and completed recordings. // An omitted mapping, a stale pin or a claimed confirmation without a clean // baseline must fail just as a measured non-detection does. - const regressions = requiredDetectionRegressions(entries, report.mutations, currentBoundaries(report, entries, directory)); + const boundaries = language.boundaryEvidence === false ? [] : currentBoundaries(report, entries, directory); + const regressions = requiredDetectionRegressions(entries, report.mutations, boundaries); if (summarize) report.detection = language.detection(report.mutations, directory); if (language.recordsRegressions) report.requiredDetectionRegressions = regressions; if (regressions.length) throw new Error(`Lost required detections: ${regressions.join(', ')}`); diff --git a/formal/profiles.json b/formal/profiles.json index 4a32f6bb..23bc2b3e 100644 --- a/formal/profiles.json +++ b/formal/profiles.json @@ -346,6 +346,29 @@ ], "definition": "go/README.md", "limits": "Same generated histories and portable scenarios as TypeScript, all protocol vectors, plus native Redis/Valkey/Cluster and cross-language integration. Claims require current corpus and witness fingerprints; finite evidence does not prove every possible schedule. Native API and value adaptations are documented in the Go README and parity ledger." + }, + { + "id": "rust", + "profiles": [ + "core", + "effects", + "scope", + "recovery", + "policy", + "shadow", + "admission", + "layers", + "independent", + "recovery-read", + "local-failure", + "runtime-boundaries", + "shadow-layers", + "local-clock", + "source-budgets", + "dark-layers", + "shadow-read-deadlines" + ], + "limits": "Same generated histories and portable scenarios as TypeScript and Go and all protocol vectors, replayed by the Rust conformance harness against the shared witness evidence. Claims require current corpus and witness fingerprints; finite evidence does not prove every possible schedule. Native Redis integration is not yet part of the claim." } ], "replaySources": [ diff --git a/formal/rust-mutations.json b/formal/rust-mutations.json new file mode 100644 index 00000000..89909ac0 --- /dev/null +++ b/formal/rust-mutations.json @@ -0,0 +1,302 @@ +{ + "schemaVersion": 1, + "mutations": [ + { + "id": "M01", + "case": "C45.maximum-age-exclusive", + "description": "Accept a retained candidate at the exact maximum age after decoding", + "typescriptMutation": "M01", + "edits": [ + { + "path": "rust/src/execution.rs", + "before": " if !valid || age as u64 >= max_age {\n self.recovery_event(RecoveryOutcome::Miss, None);\n return None;\n }\n self.recovery_event(RecoveryOutcome::Served, Some(age));", + "after": " if !valid || age as u64 > max_age {\n self.recovery_event(RecoveryOutcome::Miss, None);\n return None;\n }\n self.recovery_event(RecoveryOutcome::Served, Some(age));" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M02", + "case": "C45.recheck-after-decode", + "description": "Reuse the pre-decode age instead of observing age after asynchronous decoding", + "typescriptMutation": "M02", + "edits": [ + { + "path": "rust/src/execution.rs", + "before": " };\n let (age, valid) = self.frame_age(frame, Layer::Remote);", + "after": " };\n // Fault injection: retain the pre-decode age and validity." + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M03", + "case": "C25.late-source-rejected", + "description": "Accept an externally settled result after its elapsed deadline when the timer has not run", + "typescriptMutation": "M03", + "edits": [ + { + "path": "rust/src/deadline.rs", + "before": " Either::Left((value, _timer)) => {\n if since(clock, started) < budget {", + "after": " Either::Left((value, _timer)) => {\n if true {" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M04", + "case": "C34.second-fence", + "description": "Skip the tracked fence recheck after serialization", + "typescriptMutation": "M04", + "edits": [ + { + "path": "rust/src/execution.rs", + "before": " if let Some(fence) = fence {\n if stamp <= fence {", + "after": " if let Some(fence) = fence {\n if false && stamp <= fence {" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M05", + "case": "C03.detached-bypass", + "description": "Treat a detached context whose outer scope closed as still enabled", + "typescriptMutation": "M05", + "edits": [ + { + "path": "rust/src/scope.rs", + "before": " self.enabled && self.owner.as_ref().is_some_and(|owner| owner.is_live())", + "after": " self.enabled" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M06", + "case": "W04.strict-fence", + "description": "Accept a tracked frame timestamp equal to its watermark", + "typescriptMutation": "M06", + "edits": [ + { + "path": "rust/src/protocol/frame.rs", + "before": " if stamp <= fence {\n return miss(MissReason::WatermarkFenced);", + "after": " if stamp < fence {\n return miss(MissReason::WatermarkFenced);" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M07", + "case": "C51.changed-confirmation", + "description": "Confirm an unequal shadow result even when C1 contains different payload bytes", + "typescriptMutation": "M07", + "edits": [ + { + "path": "rust/src/shadow.rs", + "before": " ReadSnapshot::Hit(confirmed) if confirmed.payload.bytes == frame.payload.bytes => {}", + "after": " ReadSnapshot::Hit(_) => {}" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M08", + "case": "C60.invalid-logging-policy", + "description": "Enable mismatch logging after a malformed runtime logging flag", + "typescriptMutation": "M08", + "edits": [ + { + "path": "rust/src/policy.rs", + "before": " shadow.log_mismatches = false;\n shadow.logging_config_error = true;", + "after": " shadow.log_mismatches = true;\n shadow.logging_config_error = true;" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M09", + "case": "C60.logging-default-off", + "description": "Enable mismatch logging when its flag is omitted", + "typescriptMutation": "M09", + "edits": [ + { + "path": "rust/src/policy.rs", + "before": " let mut shadow = ResolvedShadow::default();", + "after": " let mut shadow = ResolvedShadow {\n log_mismatches: true,\n ..ResolvedShadow::default()\n };" + } + ], + "requiredDetections": [ + "generated", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M10", + "case": "C31.tracked-no-local-fill", + "description": "Publish an unvalidated tracked source value directly into local storage", + "typescriptMutation": "M10", + "edits": [ + { + "path": "rust/src/execution.rs", + "before": " if local_miss && !self.identity.tracked {", + "after": " if local_miss {" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M11", + "case": "C09.fixed-local-ttl", + "description": "Renew the local insertion TTL on every cache hit", + "typescriptMutation": "M11", + "edits": [ + { + "path": "rust/src/local.rs", + "before": " Ok(match self.entries.get(key) {\n Some(entry) => LocalRead::Live(entry.value.clone()),\n None => LocalRead::Absent,\n })", + "after": " Ok(match self.entries.get_mut(key) {\n Some(entry) => {\n entry.inserted_ms = now_ms;\n LocalRead::Live(entry.value.clone())\n }\n None => LocalRead::Absent,\n })" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M12", + "case": "W02.sorted-arguments", + "description": "Reverse the prescribed argument-name ordering", + "typescriptMutation": "M12", + "edits": [ + { + "path": "rust/src/identity.rs", + "before": " pairs.sort_by(|left, right| compare_utf16(&left.0, &right.0));", + "after": " pairs.sort_by(|left, right| compare_utf16(&right.0, &left.0));" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "generated", + "portable" + ] + }, + { + "id": "M13", + "case": "C54.expired-before-start", + "description": "Dispatch dark shadow reads after the job deadline already elapsed before deferred work starts", + "typescriptMutation": "M13", + "edits": [ + { + "path": "rust/src/shadow.rs", + "before": " if expired() {\n return Verdict::of(ShadowOutcome::Timeout);\n }\n let mut fill = false;", + "after": " if false && expired() {\n return Verdict::of(ShadowOutcome::Timeout);\n }\n let mut fill = false;" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + } + ] +} diff --git a/formal/source-audit.json b/formal/source-audit.json index 1b553d4b..18786505 100644 --- a/formal/source-audit.json +++ b/formal/source-audit.json @@ -3,7 +3,7 @@ "sources": [ { "path": "README.md", - "sha256": "17cd061cde252e466ba94fced8615d3bc7d2a6b88cced14fe62c3572dc86c173", + "sha256": "73fd3028635e25c046ff3af486f58ea18b2754038b60c58e12c7d9cc2624bfa5", "entries": [ {"line":1,"title":"DialCache","contracts":["B01"]}, {"line":24,"title":"Install","contracts":["B01"]}, @@ -1288,7 +1288,7 @@ }, { "path": "formal/PORTING.md", - "sha256": "9eed42f975c129b1b001b7189c3564dff79f25ae9a38f299853209a8f091e300", + "sha256": "e25af32085f0e845e98ec32c1f988accc451f0913050b483456069a086d2430f", "entries": [ {"line":1,"title":"Implementing another DialCache port"}, {"line":14,"title":"What a port supplies"}, @@ -1297,9 +1297,9 @@ {"line":196,"title":"Observation contract"}, {"line":294,"title":"Generated fixtures and fast local tests"}, {"line":329,"title":"Complete corpus and reusable acceptance checks"}, - {"line":437,"title":"Witness evidence"}, - {"line":527,"title":"Current limitations for a third port"}, - {"line":549,"title":"New-port acceptance"} + {"line":442,"title":"Witness evidence"}, + {"line":532,"title":"Current limitations for a further port"}, + {"line":554,"title":"New-port acceptance"} ], "review": { "kind": "tooling-guide", @@ -1347,15 +1347,15 @@ }, { "path": "formal/README.md", - "sha256": "acb61cc27f29cb9b3f8996fef1d72ae301ebff65ff0793515c62778cc657a6a0", + "sha256": "da2c12a96c6a63bf11098dd28a0512e334adf5362834828b35e846bc1dbd1a52", "entries": [ {"line":1,"title":"Executable DialCache specification"}, {"line":10,"title":"Start with your task"}, {"line":26,"title":"How the specification connects to code"}, {"line":46,"title":"Models and composition profiles"}, {"line":99,"title":"Generating and replaying behavior"}, - {"line":193,"title":"Shared verification and replay rules"}, - {"line":209,"title":"Evidence and scope"} + {"line":194,"title":"Shared verification and replay rules"}, + {"line":210,"title":"Evidence and scope"} ], "review": { "kind": "tooling-guide", @@ -1364,7 +1364,7 @@ }, { "path": "formal/SEMANTIC-COVERAGE.md", - "sha256": "788ccd40ccc5f3e3462ea25fab7ba577d84da0f52a0f6d5a1d1a63b7a7a18a7f", + "sha256": "cdaf28072d154b3503be5939ef7828073832ff8044546019fa4c35b988781810", "entries": [ {"line":1,"title":"Measuring semantic coverage"}, {"line":5,"title":"Evidence inventory"}, @@ -1373,7 +1373,7 @@ {"line":139,"title":"Behavioral mutation comparison"}, {"line":153,"title":"Recorded measurements"}, {"line":159,"title":"Reproduction and CI"}, - {"line":219,"title":"Model properties and cross-language execution"} + {"line":234,"title":"Model properties and cross-language execution"} ], "review": { "kind": "coverage-guide", @@ -1503,15 +1503,15 @@ }, { "path": "formal/VALIDATION.md", - "sha256": "7747e7e1668b8b9557306cf7724182c0995b84b0044c3434501ae0a9e37d3f21", + "sha256": "2feaf2ec97a22b0d4da5eb8c79302cbb4e12416e86762be90b5ab2ff632ee952", "entries": [ {"line":1,"title":"Validation and evidence"}, {"line":7,"title":"Choosing a run"}, - {"line":76,"title":"Reading a completion report"}, - {"line":94,"title":"Mutation evidence"}, - {"line":205,"title":"Exploratory runs"}, - {"line":256,"title":"A retained sampling regression"}, - {"line":293,"title":"Historical results"} + {"line":77,"title":"Reading a completion report"}, + {"line":96,"title":"Mutation evidence"}, + {"line":210,"title":"Exploratory runs"}, + {"line":261,"title":"A retained sampling regression"}, + {"line":298,"title":"Historical results"} ], "review": { "kind": "tooling-guide", @@ -1535,7 +1535,7 @@ }, { "path": "go/README.md", - "sha256": "6d6a8f5ac238772a100ac606199889c3a043562e5c718fbb0af85a4cc84166ad", + "sha256": "6f5e6f576990718ae212092a42dc1a3d3f62d2d88e8999ed06d21ac9a0dde98c", "entries": [ {"line":1,"title":"DialCache for Go"}, {"line":19,"title":"Use"}, diff --git a/formal/validation.mjs b/formal/validation.mjs index 12714603..ef466630 100644 --- a/formal/validation.mjs +++ b/formal/validation.mjs @@ -8,37 +8,42 @@ const root = fileURLToPath(new URL('../', import.meta.url)); const replayTests = ['test/formal-conformance.test.ts', 'test/formal-effects.test.ts', 'test/formal-features.test.ts', 'test/formal-local-clock.test.ts', 'test/formal-behavior.test.ts', 'test/formal-protocol-vectors.test.ts']; const aggregateTargets = { - check: ['check-ts', 'check-go', 'docs', 'audit'], - formal: ['formal-check', 'formal-generate', 'formal-ts', 'formal-go'], - mutations: ['mutations-ts', 'mutations-go'], - integration: ['integration-ts', 'integration-go'], + check: ['check-ts', 'check-go', 'check-rust', 'docs', 'audit'], + formal: ['formal-check', 'formal-generate', 'formal-ts', 'formal-go', 'formal-rust'], + mutations: ['mutations-ts', 'mutations-go', 'mutations-rust'], + integration: ['integration-ts', 'integration-go', 'integration-rust'], ci: ['check', 'package-floor', 'formal', 'model-check', 'integration', 'mutations'], }; export const targetDescriptions = { - check: 'TypeScript and Go checks, docs build and reviewed inventories; no Quint generation or Docker', + check: 'TypeScript, Go and Rust checks, docs build and reviewed inventories; no Quint generation or Docker', 'check-ts': 'Typecheck, unit coverage, build and packed-package checks on Node 24', 'check-go': 'Go vet, formatting check and default tests with race detection', + 'check-rust': 'Rust formatting check, clippy with warnings denied and default tests including the smoke conformance run', docs: 'Build the documentation site', audit: 'Check source, behavior, feature, Go and generated-fixture freshness inventories', - smoke: 'Replay committed Quint-derived fixtures in TypeScript and Go; no full completion claim', - formal: 'Check every scheduled Quint model, generate the complete corpus and shared witness evidence, then complete TypeScript and Go replay', + smoke: 'Replay committed Quint-derived fixtures in TypeScript, Go and Rust; no full completion claim', + formal: 'Check every scheduled Quint model, generate the complete corpus and shared witness evidence, then complete TypeScript, Go and Rust replay', 'formal-check': 'Typecheck and run every scheduled Quint model, its public regressions and the model mutation challenges', 'formal-generate': 'Generate/recompute artifacts and evaluate shared witness evidence over the complete corpus', 'formal-ts': 'Complete prepared TypeScript replay of the generated corpus', 'formal-go': 'Complete prepared Go replay of the generated corpus with race detection', + 'formal-rust': 'Complete prepared Rust replay of the generated corpus in release mode', 'fixtures-check': 'Recompute every committed model-derived artifact with pinned Quint', 'kernel-fixtures': 'Typecheck the kernel library fixtures (test/fixtures/kernel) and run every run they declare', differential: 'Check the composition lint baseline, then replay every composed profile against its reference corpus (merge base with DIFFERENTIAL_REFERENCE, default origin/main) in both directions (DIFFERENTIAL_SHARD=/ replays one shard balanced by estimated profile replay time, as the hosted lane does with four)', - explore: 'Explore a new recorded seed and replay both ports in an isolated source snapshot', + explore: 'Explore a new recorded seed and replay every port in an isolated source snapshot', 'model-check': 'Symbolically verify the scheduled finite rules with pinned Quint/Apalache (Java 21)', - mutations: 'Measure TypeScript and Go semantic mutations over generated histories, real Redis vectors and shared witnesses (Docker)', + mutations: 'Measure TypeScript, Go and Rust semantic mutations over generated histories, real Redis vectors and shared witnesses (Docker)', 'mutations-ts': 'Measure TypeScript semantic mutations over generated histories and real Redis vectors; requires Docker (MUTATION_SHARD=/ measures one shard; MUTATION_ONLY=, measures the named mutants into a partial report)', 'mutations-go': 'Measure Go semantic mutations over generated histories, real Redis vectors and shared witnesses; requires Docker (MUTATION_SHARD=/ measures one shard; MUTATION_ONLY=, measures the named mutants into a partial report)', + 'mutations-rust': 'Measure Rust semantic mutations over the generated corpus and shared witness evidence (MUTATION_SHARD=/ measures one shard; MUTATION_ONLY=, measures the named mutants into a partial report)', 'mutations-merge-ts': 'Merge TypeScript mutation shards into the complete report; refuses inconsistent or missing shards', 'mutations-merge-go': 'Merge Go mutation shards into the complete report; refuses inconsistent or missing shards', - integration: 'Run real TypeScript and Go Redis/Valkey/Cluster integration checks', + 'mutations-merge-rust': 'Merge Rust mutation shards into the complete report; refuses inconsistent or missing shards', + integration: 'Run real TypeScript, Go and Rust Redis/Valkey/Cluster integration checks', 'integration-ts': 'Run TypeScript real integration checks', 'integration-go': 'Run Go real integration and interoperability checks with race detection', + 'integration-rust': 'Run Rust real Redis/Valkey/Cluster integration checks and invalidation vector replay', 'package-floor': 'Check zstd and the packed package on exact Node 22.15.0 (NODE22_BIN)', ci: 'Run check, package-floor, formal, model-check, integration and mutations in dependency order', }; @@ -51,16 +56,16 @@ export function expandTargets(target) { // MUTATION_SHARD=/ narrows one measurement lane to a shard of // its catalog; the merge target later assembles the complete report. // MUTATION_ONLY=, measures only the named mutants into a partial -// report that is never evidence. Only the two leaf lanes accept them: an +// report that is never evidence. Only the three leaf lanes accept them: an // aggregate that silently ignored them would run the complete measurement the // caller did not ask for. -const shardedTargets = ['mutations-ts', 'mutations-go']; +const shardedTargets = ['mutations-ts', 'mutations-go', 'mutations-rust']; export function mutationSelectionArguments(target, environment = process.env) { const set = [['MUTATION_SHARD', environment.MUTATION_SHARD], ['MUTATION_ONLY', environment.MUTATION_ONLY]].filter(([, value]) => value !== undefined); if (!set.length) return []; const stated = set.map(([name, value]) => `${name}=${value}`).join(' '); if (!shardedTargets.includes(target)) { - if (expandTargets(target).some(name => shardedTargets.includes(name))) throw new Error(`${stated} applies only to make mutations-ts and make mutations-go; unset it to run the complete measurement with make ${target}.`); + if (expandTargets(target).some(name => shardedTargets.includes(name))) throw new Error(`${stated} applies only to make mutations-ts, make mutations-go and make mutations-rust; unset it to run the complete measurement with make ${target}.`); return []; } if (set.length > 1) throw new Error(`${stated}: MUTATION_SHARD and MUTATION_ONLY exclude each other; a partial run is never merged.`); @@ -116,6 +121,9 @@ export function validationPlan(target, { directory = root, environment = process const node = (label, ...args) => ({ label, command: runnerNode, args }); const pnpm = (label, ...args) => ({ label, command: 'corepack', args: ['pnpm', ...args] }); const go = (label, ...args) => ({ label, command: 'go', args: ['-C', 'go', ...args] }); + // Cargo runs inside rust/ so rustup resolves rust/rust-toolchain.toml; the + // crate's tests locate the repository through CARGO_MANIFEST_DIR, not cwd. + const cargo = (label, subcommand, ...args) => ({ label, command: 'cargo', args: [subcommand, ...args], cwd: 'rust' }); const reportPath = (language, suffix) => `.formal-traces/${language}-${suffix}.json`; const completion = language => ({ ...node(`Validate current ${language} completion`, 'formal/conformance.mjs', 'check', reportPath(language, 'completion'), reportPath(language, 'context')), failureHint: 'A current complete replay is required. Run make formal first; missing or stale reports cannot be reused.' }); @@ -149,6 +157,13 @@ export function validationPlan(target, { directory = root, environment = process 'test', '-race', '-count=1', ...(full ? ['-json', '-timeout=35m', '-skip', '^(TestGeneratedInvalidationVectors|TestVectorBoundaryDriver)$'] : []), './...'), ...(full ? { env: { ...replayEnv, DIALCACHE_WITNESS_EVIDENCE_DIR: witnessDirectory }, stdoutFile: '.formal-traces/go-replay.jsonl' } : {}) }); + // The Rust conformance harness is one cargo test target. Without directory + // selectors it replays the committed smoke histories; with them it replays + // the complete corpus and writes its own JSONL report to DIALCACHE_RUST_REPORT + // (cargo's stdout carries build output, so it is not the report channel). + const nativeRust = full => ({ ...cargo(full ? 'Replay complete Rust corpus' : 'Replay committed Rust fixtures', + 'test', ...(full ? ['--release'] : []), '--all-features', '--test', 'conformance'), + ...(full ? { env: { ...replayEnv, DIALCACHE_WITNESS_EVIDENCE_DIR: witnessDirectory, DIALCACHE_RUST_REPORT: resolve(directory, '.formal-traces/rust-replay.jsonl') } } : {}) }); const node22 = floorExecutable(environment, runnerNode, nodeVersion) ?? ''; const selection = mutationSelectionArguments(target, environment); const differentialShard = differentialShardArguments(target, environment); @@ -156,12 +171,15 @@ export function validationPlan(target, { directory = root, environment = process 'check-ts': [pnpm('Typecheck TypeScript', 'typecheck'), pnpm('Run TypeScript unit tests with coverage', 'test'), pnpm('Build package', 'build'), pnpm('Check packed package on Node 24', 'test:package')], 'check-go': [go('Go vet', 'vet', './...'), { label: 'Check Go formatting', command: 'gofmt', args: ['-l', 'go'], requireEmptyStdout: true }, nativeGo(false)], + 'check-rust': [cargo('Check Rust formatting', 'fmt', '--check'), + cargo('Lint Rust with clippy', 'clippy', '--all-targets', '--all-features', '--', '-D', 'warnings'), + cargo('Run Rust default tests', 'test', '--all-features')], docs: [pnpm('Build documentation', 'docs:build')], audit: ['execution', 'check-source-audit', 'check-semantic-coverage', 'check-feature-coverage', 'check-go-parity'] .map(name => node(`Check ${name}`, `formal/${name}.mjs`)) .concat(node('Verify committed fixture fingerprints', 'formal/generated-fixtures.mjs', '--verify'), node('Check conditional fixture regeneration scope', '--test', '.github/scripts/fixture-scope.test.mjs')), - smoke: [tsReplay(false), nativeGo(false)], + smoke: [tsReplay(false), nativeGo(false), nativeRust(false)], 'fixtures-check': [node('Recompute all committed Quint artifacts', 'formal/generate-artifacts.mjs', '--check')], 'kernel-fixtures': [kernelFixtures], differential: [lintBaseline, kernelFixtures, node('Replay composed profiles against their reference corpus', 'formal/differential.mjs', '--composed', `--reference=${environment.DIFFERENTIAL_REFERENCE ?? 'origin/main'}`, ...differentialShard)], @@ -173,9 +191,9 @@ export function validationPlan(target, { directory = root, environment = process 'formal-check': [node('Check every scheduled Quint model', 'formal/run-models.mjs', 'check'), node('Measure every pinned model fault', 'formal/check-model-properties.mjs'), lintBaseline, kernelFixtures], // Generation is the single shared producer: the corpus, wire artifacts and - // witness evidence depend only on the models. Both ports' replays and both - // mutation measurements read that output and can run in parallel off it. - 'formal-generate': [invalidate('ts', 'go'), + // witness evidence depend only on the models. Every port replay and mutation + // measurement read that output and can run in parallel off it. + 'formal-generate': [invalidate('ts', 'go', 'rust'), node('Generate complete corpus and recompute wire artifacts', 'formal/run-models.mjs', 'generate'), node('Recompute committed Quint smoke and witness fixtures', 'formal/generated-fixtures.mjs', '--check'), witnesses], 'formal-ts': [invalidate('ts'), node('Prepare TypeScript execution context', 'formal/conformance.mjs', 'prepare', 'typescript', reportPath('ts', 'context')), @@ -185,14 +203,25 @@ export function validationPlan(target, { directory = root, environment = process node('Prepare Go execution context', 'formal/conformance.mjs', 'prepare', 'go', reportPath('go', 'context')), nativeGo(true), { ...node('Check complete Go native report', 'formal/check-go-replay.mjs'), stdoutFile: '.formal-traces/go-replay-summary.json' }, { ...node('Adapt Go native assertion report', 'formal/conformance-adapters.mjs', 'go', '.formal-traces/go-replay.jsonl', reportPath('go', 'context')), stdoutFile: reportPath('go', 'completion') }, completion('go')], + // The Go parity ledger is Go-only; Rust has no ledger step. The harness + // names cases by inventory id, so the report gate checks the binding too. + 'formal-rust': [invalidate('rust'), + node('Prepare Rust execution context', 'formal/conformance.mjs', 'prepare', 'rust', reportPath('rust', 'context')), nativeRust(true), + { ...node('Check complete Rust native report', 'formal/check-rust-replay.mjs'), stdoutFile: '.formal-traces/rust-replay-summary.json' }, + { ...node('Adapt Rust native assertion report', 'formal/conformance-adapters.mjs', 'rust', '.formal-traces/rust-replay.jsonl', reportPath('rust', 'context')), stdoutFile: reportPath('rust', 'completion') }, completion('rust')], 'mutations-ts': [node('Measure TypeScript semantic mutations', 'formal/measure-semantics.mjs', ...selection)], 'mutations-go': [node('Measure Go semantic mutations', 'formal/measure-go-semantics.mjs', ...selection)], + 'mutations-rust': [node('Measure Rust semantic mutations', 'formal/measure-rust-semantics.mjs', ...selection)], // The merge needs neither Quint nor Go: it reads shard reports, checks // them against each other and this checkout, and writes the complete report. 'mutations-merge-ts': [node('Merge TypeScript mutation shards', 'formal/merge-mutation-reports.mjs', 'ts')], 'mutations-merge-go': [node('Merge Go mutation shards', 'formal/merge-mutation-reports.mjs', 'go')], + 'mutations-merge-rust': [node('Merge Rust mutation shards', 'formal/merge-mutation-reports.mjs', 'rust')], 'integration-ts': [pnpm('Run TypeScript Redis/Valkey/Cluster integrations', 'test:integration')], 'integration-go': [{ ...go('Run Go Redis/Valkey/Cluster and TypeScript interoperability', 'test', '-race', '-tags', 'integration', '-count=1', '-run', '^TestRedisIntegration$', '-json', './...'), stdoutFile: '.formal-traces/go-integration.jsonl' }], + // The Rust integration tests are #[ignore]d, so a plain cargo test reports + // them as ignored and never needs Docker; this lane runs exactly them. + 'integration-rust': [cargo('Run Rust Redis/Valkey/Cluster integrations', 'test', '--all-features', '--test', 'redis_integration', '--', '--ignored')], 'package-floor': [{ label: 'Require a built package for floor checks', requireFile: 'dist/index.js', failureHint: 'Build first with make check-ts, or run make ci with NODE22_BIN set.' }, { label: 'Check Node 22.15 zstd round trip and output ceiling', command: node22, args: ['--eval', floorSmoke], env: { PATH: floorEnvironment(environment, node22).PATH } }, { label: 'Check packed package on Node 22.15', command: node22, args: ['scripts/test-package.mjs'], env: { PATH: floorEnvironment(environment, node22).PATH } }], @@ -217,6 +246,10 @@ export function checkPrerequisites(target, { directory = root, environment = pro const version = probe('go', ['version'], { directory, environment }); if (!/^go version go1\.27\.1\s/.test(version)) throw new Error(`Validation requires Go 1.27.1; found ${version}. Put the pinned Go toolchain on PATH.`); } + if (targets.some(name => ['check-rust', 'smoke', 'formal-rust', 'integration-rust', 'mutations-rust', 'explore'].includes(name))) { + const version = probe('cargo', ['--version'], { directory: resolve(directory, 'rust'), environment }); + if (!/^cargo 1\.98\.1(?:\s|$)/.test(version)) throw new Error(`Validation requires cargo 1.98.1; found ${version}. Install Rust 1.98.1 (rustup reads rust/rust-toolchain.toml) and put it on PATH.`); + } if (targets.some(name => ['formal-check', 'formal-generate', 'fixtures-check', 'explore', 'model-check', 'differential'].includes(name))) { const requiredQuint = JSON.parse(readFileSync(resolve(directory, 'formal/generated-fixtures.lock.json'), 'utf8')).quintVersion; const version = probe('quint', ['--version'], { directory, environment }); @@ -336,7 +369,7 @@ export async function executeSteps(steps, { directory = root, environment = proc } try { await new Promise((resolveRun, reject) => { - const child = spawn(step.command, step.args, { cwd: directory, env: cleanEnvironment(baseEnvironment, step.env), + const child = spawn(step.command, step.args, { cwd: step.cwd ? resolve(directory, step.cwd) : directory, env: cleanEnvironment(baseEnvironment, step.env), stdio: ['inherit', output ?? (step.requireEmptyStdout ? 'pipe' : 'inherit'), 'inherit'] }); let unexpectedOutput = ''; if (step.requireEmptyStdout) child.stdout.on('data', data => { unexpectedOutput += data; }); @@ -378,10 +411,10 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { if (target === 'help') { const width = Math.max(...Object.keys(targetDescriptions).map(name => name.length)); console.log(Object.entries(targetDescriptions).map(([name, description]) => `make ${name.padEnd(width)} ${description}`).join('\n')); - console.log('\nPrerequisites: frozen pnpm install; Node 24, pinned pnpm; Go 1.27.1 / Docker where required; Quint 0.32.0 for formal-check, formal-generate, fixtures-check, explore and model-check; Java 21 and tar for model-check and ci.'); + console.log('\nPrerequisites: frozen pnpm install; Node 24, pinned pnpm; Go 1.27.1 / cargo 1.98.1 / Docker where required; Quint 0.32.0 for formal-check, formal-generate, fixtures-check, explore and model-check; Java 21 and tar for model-check and ci.'); console.log('formal-check is the Quint evidence lane (models, regressions, challenges); the port and mutation lanes read only the formal-generate output and do not wait for it.'); console.log('Sharded mutation runs: MUTATION_SHARD=/ make mutations-ts for every index, matching the workflow matrix, on any machines with the same corpus, then make mutations-merge-ts; the merged report is the only complete evidence.'); - console.log('One mutant locally: MUTATION_ONLY=M14,M15 make mutations-ts (or mutations-go) writes a partial report under partial/ and leaves the complete report alone.'); + console.log('One mutant locally: MUTATION_ONLY=M01,M02 make mutations-ts (or mutations-go / mutations-rust) writes a partial report under partial/ and leaves the complete report alone.'); console.log('Sharded differential runs: DIFFERENTIAL_SHARD=/ make differential replays one shard balanced by estimated profile replay time, matching the pull request workflow\'s four-shard matrix; every shard checks the lint baseline and the kernel fixtures.'); console.log('Full local CI: make ci NODE22_BIN=/absolute/path/to/node22/bin/node (exact 22.15.0).'); } else { diff --git a/go/README.md b/go/README.md index df5b5497..c87e2397 100644 --- a/go/README.md +++ b/go/README.md @@ -198,7 +198,7 @@ once, then: ```sh make check-go # Native checks, committed cases/smoke and race detection. -make formal # Rust model/corpus checks, then prepared TS and Go replay. +make formal # Quint model/corpus checks, then prepared TS, Go and Rust replay. make model-check # Separate finite symbolic checks. make mutations-go # Go fault catalog over the generated corpus and witness evidence. make integration-go diff --git a/go/redis_interop.ts b/go/redis_interop.ts index 1acdd16f..40fa8081 100644 --- a/go/redis_interop.ts +++ b/go/redis_interop.ts @@ -1,8 +1,9 @@ -// Executed by the Go integration test after bundling against production TS +// Executed by the Go and Rust integration tests after bundling production TS // imports. Inputs are operations and serialized values, never expected state. import { readFileSync } from "node:fs"; import { createClient, createCluster } from "redis"; import { createNodeRedisDialCacheClient } from "../src/node-redis.js"; +import { DialCacheKey, normalizeArgs } from "../src/key.js"; import { JsonSerializer } from "../src/serializer.js"; import { compressPayload, decompressPayload, escapeRawPayload } from "../src/internal/compression.js"; import { isRedisReadMiss } from "../src/redis-client.js"; @@ -20,26 +21,47 @@ async function main() { const results = []; try { for (const action of input.actions) { - if (action.op === "write") { + // Derive keys independently in TypeScript when the caller supplies a + // logical identity. Numeric bits avoid JSON decimal formatting becoming + // an accidental shared oracle for Number::toString regressions. + const identity = action.identity; + const args = { ...identity?.args }; + for (const [name, bits] of Object.entries(identity?.numberBits ?? {})) { + args[name] = Buffer.from(bits as string, "hex").readDoubleBE(); + } + const logical = identity === undefined ? undefined : new DialCacheKey({ + ...identity, args: normalizeArgs(args), + }); + const keys = logical === undefined ? undefined : { + logical: logical.urn, + value: `${logical.urn}:dialcache-frame-v1`, + watermark: logical.trackForInvalidation ? `${logical.prefix}#watermark` : null, + }; + const key = keys?.value ?? action.key; + const watermark = keys?.watermark ?? action.watermark; + if (action.op === "key") { + if (keys === undefined) throw new Error("key operation requires an identity"); + results.push({ kind: "key", keys }); + } else if (action.op === "write") { const serialized = action.binaryHex === undefined ? await codec.dump(action.absent ? undefined : action.value) : Buffer.from(action.binaryHex, "hex"); const payload = action.compress ? compressPayload(serialized, { thresholdBytes: 1, level: 3 }).payload : escapeRawPayload(serialized); - await adapter.write({ valueKey: action.key, value: payload, createdAtMs: action.stamp, cacheTtlMs: 60000 }); - results.push({ kind: "written" }); + await adapter.write({ valueKey: key, value: payload, createdAtMs: action.stamp, cacheTtlMs: 60000 }); + results.push({ kind: "written", ...(keys ? { keys } : {}) }); } else if (action.op === "read") { - const result = await adapter.read({ valueKey: action.key, ...(action.watermark ? { watermarkKey: action.watermark } : {}) }); + const result = await adapter.read({ valueKey: key, ...(watermark ? { watermarkKey: watermark } : {}) }); if (isRedisReadMiss(result)) { results.push(result); continue; } const { payload } = decompressPayload(result.payload); const value = action.binary ? undefined : await codec.load(payload); - results.push({ kind: "hit", stamp: result.createdAtMs, + results.push({ kind: "hit", stamp: result.createdAtMs, ...(keys ? { keys } : {}), ...(action.binary ? { binaryHex: Buffer.from(payload).toString("hex") } : { value: value === undefined ? { absent: true } : value }) }); } else if (action.op === "invalidate") { const realNow = Date.now; Date.now = () => action.stamp; - try { await adapter.invalidate({ watermarkKey: action.watermark, futureBufferMs: action.futureMs }); } + try { await adapter.invalidate({ watermarkKey: watermark, futureBufferMs: action.futureMs }); } finally { Date.now = realNow; } results.push({ kind: "invalidated" }); } else { throw new Error(`Unknown interop operation ${action.op}`); } diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 00000000..3f1f8179 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,1217 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "dialcache" +version = "0.1.0" +dependencies = [ + "dialcache", + "futures", + "hex", + "log", + "lru", + "parking_lot", + "prometheus", + "redis", + "ryu-js", + "serde", + "serde_json", + "sha1", + "sha2", + "slab", + "thiserror 2.0.20", + "tokio", + "zstd", + "zstd-safe 7.3.0", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror 2.0.20", +] + +[[package]] +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.69", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redis" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed" +dependencies = [ + "arc-swap", + "arcstr", + "async-lock", + "backon", + "bytes", + "cfg-if", + "combine", + "crc16", + "futures-channel", + "futures-util", + "itoa", + "log", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "rand", + "ryu", + "sha1_smol", + "socket2", + "tokio", + "tokio-util", + "url", + "xxhash-rust", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "ryu-js" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" +dependencies = [ + "zstd-safe 8.0.0", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.0+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 00000000..4534e9f2 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,62 @@ +[package] +name = "dialcache" +version = "0.1.0" +edition = "2021" +rust-version = "1.85" +license = "MIT" +description = "Explicitly enabled, layered caching with runtime rollout, request coalescing, tracked invalidation, stale recovery and shadow validation. Rust port of DialCache." +repository = "https://github.com/lan17/DialCache" +keywords = ["cache", "redis", "async"] +categories = ["caching"] +publish = false + +[features] +default = ["tokio"] +tokio = ["dep:tokio"] +# Deterministic clock and single-threaded runtime for tests. +test-util = [] +redis = ["dep:redis", "dep:sha1", "dep:hex"] +prometheus = ["dep:prometheus"] + +[dependencies] +futures = "0.3" +parking_lot = "0.12" +slab = "0.4" +lru = "0.18" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["float_roundtrip"] } +ryu-js = "1" +zstd = "0.14" +zstd-safe = "7" +thiserror = "2" +log = "0.4" +tokio = { version = "1", features = ["rt", "time"], optional = true } +redis = { version = "1", features = ["tokio-comp", "cluster-async", "connection-manager", "script"], optional = true } +sha1 = { version = "0.11", optional = true } +hex = { version = "0.4", optional = true } +prometheus = { version = "0.14", optional = true } + +[dev-dependencies] +sha2 = "0.11" +dialcache = { path = ".", features = ["test-util"] } +tokio = { version = "1", features = ["full"] } +hex = "0.4" +serde_json = "1" + +[[test]] +name = "conformance" +path = "tests/conformance.rs" +harness = false + +[[test]] +name = "redis_integration" +path = "tests/redis_integration.rs" +required-features = ["redis"] + +[[example]] +name = "basic" +required-features = ["tokio"] + +[[example]] +name = "redis" +required-features = ["redis", "tokio"] diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 00000000..ec81b60f --- /dev/null +++ b/rust/README.md @@ -0,0 +1,305 @@ +# DialCache for Rust + +Rust implements the same portable behavior as the TypeScript library and the +Go port: explicit request enablement, request/local/Redis layers, +deterministic rollout, sparse runtime policy, request and process coalescing, +tracked invalidation, source and read deadlines, stale recovery, dark and +served-hit shadow validation, compression, and failure-isolated observability. + +The [Quint models](../formal/README.md) are the behavioral source of truth. +The Rust conformance harness replays the same sampled histories and named +public-action regressions as the other ports, plus the fixed scenarios and +Quint-derived protocol vectors, through the shared Node replay coordinator. +These are finite checks of the documented contract, not proof of every +possible input or schedule. + +## Use + +The core crate requires Rust 1.85 or later; the `redis` feature requires Rust +1.88 with the currently locked Redis dependency. CI pins 1.98.1 through +`rust/rust-toolchain.toml`, which rustup honors when cargo runs inside `rust/`. +Applications own their Redis connection and its timeout, retry and +resource budgets. The default runtime is the tokio runtime that is current +while the cache is built. + +```rust +use std::sync::Arc; +use dialcache::{BoxError, DialCache, KeySpec, Policy}; + +#[tokio::main] +async fn main() -> Result<(), BoxError> { + let cache = DialCache::builder().namespace("my-app").build()?; + let display_name = cache + .use_case::("user", "displayName") + .policy(Policy::default().request_local(true).local_ttl_sec(30)) + .key(|id: &u64| KeySpec::new(id)) + .source(|_scope, id| async move { + // Replace this with your database or API call. + Ok(format!("User {id}")) + }) + .register()?; + + let request = cache.enable_guard(); + let name: Arc = display_name.get(request.scope(), 42).await?; + println!("Hello, {name}!"); + Ok(()) +} +``` + +A policy enables no cache layers by default. The example opts into request +caching and a 30-second process-local cache; register the use case once at +startup and create a scope for each request. `Arc` shares one cached value +without requiring `T: Clone`. Settled memory entries holding another Rust type +are misses; compatible remote JSON may still decode into the requested type. +Simultaneous calls sharing a key also share one source result, so an incompatible +coalesced follower returns a type error. Use a consistent value type per key. + +Run the complete [basic example](./examples/basic.rs), which demonstrates a +structured value, an asynchronous source and reuse across two request scopes: + +```sh +cd rust +cargo run --example basic +``` + +The [Redis example](./examples/redis.rs) configures application-owned connection +and command timeouts, enables tracked Redis caching and demonstrates +invalidation. With a Redis server running: + +```sh +REDIS_URL=redis://127.0.0.1/ cargo run --features redis --example redis +``` + +The examples use the crate's existing dependencies. Applications also need +`tokio` with `macros`, `rt-multi-thread` and `time` enabled; structured JSON +values use `serde` with its `derive` feature. The Redis example additionally +needs the `redis` crate with `tokio-comp`, and DialCache's `redis` feature. + +Caching is disabled by default. `DialCache::enable` (closure form) or +`DialCache::enable_guard` (RAII form) opens the outermost enabled scope and +hands out a `Scope`; pass it to every cached call made on behalf of that +request, including calls made inside a source. Completing the callback or +dropping the guard closes the scope: retained `Scope` clones no longer enable +caching and late work cannot publish into the request memo. +`DialCache::enable_in` and `DialCache::disable_in` derive nested scopes that +share the outer request memo. `Scope::outside()` is the pass-through scope of +work that runs on behalf of no request. + +`use_case` registers a typed use case once per instance and returns a +`UseCase` handle; `get_or_load` runs one inline `Operation` without +registration. Both snapshot the static policy and the source budget before any +asynchronous work. Values come back as `Arc`: shared by reference, treat +them as immutable. Both `UseCase` and `Operation` can be cloned +without requiring `T: Clone`. Use case `watermark` is reserved. `coalescing_state` +reports actual process leaders, followers and the oldest leader age. + +Sources are `Fn(Scope, Args) -> Future>`. They may +run again later for served-hit shadow validation, so they must be reusable. +Source errors surface as `Error::Source(Arc)`; every coalesced +caller receives the same shared instance, so `Arc::ptr_eq` identifies one +failure. A source deadline returns `Error::FallbackTimeout` and does not cancel +the source. Dropping the future returned by `get` never cancels the execution: +sources, publications and other callers keep their contracts. + +`Identity::new`, `KeySpec::new` and `DialCache::invalidate` accept strings, +integers and floats, including shared references such as `&u64`. Their `IntoKeyId` conversion +preserves string IDs and exact decimal integers; floats use JavaScript number +spelling, including negative zero and exponents (`f32` is promoted to `f64`). +For custom displayable IDs, pass +`id.to_string()` or implement `IntoKeyId`. `KeySpec::arg` also accepts all +primitive integer and float types, preserving exact integer text and promoting +`f32` to `f64`, as well as borrowed inputs such as `&String` and `&u64`. +`normalize_args` applies the shared scalar spelling to secondary dimensions and +orders names by UTF-16 code units so the same identity produces the same Redis +key in every language. +Use the same namespace, key dimensions, codecs and policy across languages when +sharing entries. + +## Configuration and effects + +`Policy` holds the static leaves: whole-second TTLs (`local_ttl_sec`, +`remote_ttl_sec`), serving ramps, `request_local`, `coalesce`, +`stale_on_error_max_age_sec`, `remote_read_timeout_ms` and `shadow`. +`Policy::from_json` accepts the TypeScript JSON-shaped configuration. +`Policy::enabled(ttl)` and `Policy::disabled()` are the two static helpers. +A `policy_provider` returns a sparse `RuntimePolicy` overlay once per enabled +invocation; `Ok(None)` inherits, present leaves replace operation leaves, and +invalid leaves have the narrower consequences defined in Quint (an invalid TTL +or ramp disables only that layer; an invalid flag or read deadline bypasses +caching for the call). + +Convert a typed policy with `RuntimePolicy::from(policy)` or `policy.into()`. +Omitted leaves remain absent, including `request_local` and `coalesce`, so a +TTL-only overlay preserves the operation's flags. `Policy::to_json` uses the +same sparse representation; library defaults are applied during resolution. + +Defaults are namespace `urn`, local capacity 10,000, 50 ms remote reads, +60,000 ms source calls (`SourceBudget::Default`; `SourceBudget::Unbounded` +disables the deadline), sharing enabled, and shadow capacity one. Policy omits +all cache layers by default. Invalid constructor configuration returns +`ConfigError` from `build`; invalid operation configuration is returned before +execution. + +`Remote` supplies atomic primary snapshots, complete client-stamped frame +writes and surfaced invalidation errors. Writes are one native `SET`; +invalidation dispatches `EVALSHA` and retries once with `EVAL`. No value write +creates or extends a watermark. `DialCache::invalidate` affects shared remote +authority; other processes' local entries and already acquired snapshots +retain the documented lifetime rules. Inline operations with an explicit namespace +can invalidate that same entity through `invalidate_identity`: + +```rust,ignore +let identity = Identity::new("user", 42, "displayName") + .namespace("tenant-b").tracked(true); +// After committing the source mutation: +cache.invalidate_identity(identity, 0).await?; +``` + +An empty namespace inherits the cache instance's namespace. Invalidation covers +all tracked use cases and argument variants for that namespace, entity type and +ID; the identity's `tracked` flag does not restrict the maintenance operation. + +`Clock` separates wall time from elapsed time; `Runtime` supplies detached +task admission and timers. The defaults are `SystemClock` (aligned to the +process-wide millisecond grid used by local expiry) and `TokioRuntime`, which +captures the current tokio runtime handle when the cache is built: building +outside a tokio context is a `ConfigError`, and `TokioRuntime::from_handle` +selects a runtime explicitly. The runtime needs its time driver. +`SystemClock::with_sources` runs the same grid alignment over caller-supplied +time sources. The `test-util` feature ships `testing::TestExecutor`, a +deterministic single-threaded executor with a virtual clock that runs the +cache's detached work to quiescence on demand and delivers timers only when a +test advances time; the conformance harness is built on it, and its +local-clock replay builds `SystemClock` over the virtual clock. + +Detached work is registered RAII-style: if a runtime drops a leader task +before it settles (the cache outlived a shut-down runtime), its flight is +unregistered and its followers receive an error instead of waiting forever. +Local storage hands removed entries back to the cache so a value's destructor +never runs while a cache lock is held. + +## Values, codecs and observability + +`JsonCodec` (serde_json) is the default; `Codec` is asynchronous, and +`FromSync` adapts a synchronous `SyncCodec`. The TypeScript `undefined` +sentinel decodes as JSON `null`, so `Option` destinations read it as +`None`. Compression defaults to a 4,096-byte threshold and zstd level 3; +`disable_compression` stores payloads raw while reads still accept compressed +entries. The wire contract requires interoperable decompression, not identical +compressed bytes. The async engine dispatches payload compression at 64 KiB or +larger (and compression at levels 10–22 once the configured threshold is met) +and every zstd decompression through `Runtime::spawn_blocking`. Small raw +payloads remain inline; synchronous protocol helpers remain synchronous. + +The default CPU executor is shared across cache instances: two worker threads +and two queued jobs. Admission never waits for queue space. Saturation or worker +creation failure fails open: a read falls through to the source, and a failed +compression skips the write while preserving the source result. Custom runtimes +may supply their own bounded CPU executor. `StepRuntime` queues these jobs on its +controlled executor for deterministic tests. A shadow job retains its capacity +until its admitted CPU work finishes or is discarded, including after a deadline +or async runtime shutdown. Codecs supplied by the application still choose their +own scheduling; `FromSync` does not offload application serialization. + +For remote writes, the engine calls `Codec::encode_owned` with the existing +`Arc`. Its default implementation delegates to `encode(&T)`, so existing +codecs work unchanged. Override `encode_owned` to send a non-`Clone` value to a +background CPU job without copying it or changing the cached value type. +`decode` already receives an owned `Payload`. Application codecs own admission +and the lifetime of jobs they start; those jobs do not inherit the library's +shadow-capacity token. Default JSON encoding/decoding and `FromSync` remain +synchronous, so large values can occupy an async worker. + +`Observer` receives every public diagnostic as a typed `Event`. Shadow +validation exists only to be observed, so a job is admitted only when the +observer opts in through `observes_shadow_outcomes`; the bundled exporters do. +`Logger` receives structured `LogEvent`s and defaults to the `log` facade. +Default stale-recovery decode warnings omit error text that could contain cached +values; JSON errors retain their category and line/column location. A custom +`Logger` can inspect the original error when application-controlled details are +needed. +Mismatch logging is opt-in, confirmed, bounded, and previews values through +the operation's `preview` (JSON for serde values). Default JSON previews retain +only an 8 KiB prefix while checking the entire serialization for errors. Preview +callbacks run through the bounded CPU executor after confirmation; they may run +on a worker thread. Queue rejection omits value previews but still logs the +confirmed mismatch. Diagnostic work holds shadow capacity until it finishes, +including after runtime shutdown. Observer and logger failures never change a +cache, source or maintenance result. + +`MetricKind` maps every event to the metric names, labels and values shared +with the TypeScript and Go exporters. `PrometheusObserver` (feature +`prometheus`) registers the nineteen collectors on a `prometheus::Registry` +under an optional prefix; clone one observer for every instance that exports +to the same registry, because the `prometheus` crate cannot hand back an +existing collector and a second registration of the same names is a +`PrometheusError::Conflict`. `DatadogObserver` sends the same metrics through +a caller-supplied `DogStatsdClient`, with `DatadogOptions` choosing histogram +or distribution and a namespace. + +`RedisAdapter` (feature `redis`) implements `Remote` over the `redis` crate +for `ConnectionManager`, `MultiplexedConnection` and cluster connections, +routing tracked reads to slot primaries and sharing the invalidation script +and frame codec with the other ports. The `redis_integration` test replays +every invalidation vector against real Redis, Valkey and Cluster servers, +and reads/writes shared entries with the production TypeScript adapter in both +directions. Both ports construct keys independently, including numeric rounding +boundaries; the payload tests cover JSON, binary escaping, compression and +invalidation by either language. The tests are `#[ignore]`d, and +`make integration-rust` runs them where Docker is available. + +## Validation and reproducing a trace + +Use the repository [Make targets](../Makefile) from its root. CI pins Rust +1.98.1, Go 1.27.1, Node 24 and pnpm 10.33.0. Rust conformance tests use the +shared Node replay coordinator for command mappings and assertions; the crate +itself has no Node dependency. + +```sh +make check-rust # fmt, clippy, unit tests, protocol vectors, fixed scenarios and committed smoke histories +make integration-rust # Real Redis, Valkey and Cluster servers through Docker, plus every invalidation vector +make formal # Quint model checks, full corpus, then TypeScript, Go and Rust replay +make formal-rust # Complete prepared Rust replay of the generated corpus +make mutations-rust # Measure the Rust fault catalog (formal/rust-mutations.json) against the replay +``` + +`make mutations-rust` applies each catalogued single-site fault to an isolated +copy of the crate and requires the conformance harness to detect it +(`DIALCACHE_RUST_SUITE=generated` for the Quint-generated evidence, +`DIALCACHE_RUST_SUITE=fixed` for the fixed scenarios); see +[SEMANTIC-COVERAGE.md](../formal/SEMANTIC-COVERAGE.md). + +Without overrides, `cargo test --all-features --test conformance` replays the +committed smoke histories, every fixed scenario and every protocol vector. +The same `DIALCACHE_*_TRACE_DIR` / `_TRACE_FILE` selectors as Go replay a +directory or one history; `DIALCACHE_WITNESS_EVIDENCE_DIR` binds the shared +witness evidence and `DIALCACHE_RUST_REPORT` names the JSONL assertion report +the completion checker consumes. Reports and traces are kept in +`.formal-traces/`. + +```sh +DIALCACHE_FEATURE_TRACE_FILE="$PWD/.formal-traces/regressions/shadow/confirmationPastFreshnessKeepsOriginalPayloadAndAgeTest.itf.json" \ + cargo test --manifest-path rust/Cargo.toml --all-features --test conformance +``` + +`tests/settlement_control.rs` is the no-settle control required of every +port: a driver that reports observations before the settlement drain fails +every behavior-driver-backed smoke history. + +## Adaptations + +- Explicit `Scope` handles replace the implicit async context of TypeScript + and the `context.Context` of Go. +- Values are `Arc`; sources return `Result`. +- The default shadow comparator is `PartialEq`; `NaN` therefore differs from + itself where the TypeScript default treats it as equal. +- Local storage failures are exposed through the `LocalStore` trait rather + than a clock fault. +- Prometheus collectors are shared by cloning the observer rather than by + registering the same names twice. +- The behavior driver checks source/write causality after every command. + Its test runtime carries driver-owned invocation identities across detached + tasks, independently attributing each write to its actual source callback. +- The core replay, the exporters and the Redis adapter follow their Go + counterparts; real-server integration is a separate lane, as in the other + ports, and not part of the completion claim (see `formal/profiles.json`). diff --git a/rust/examples/basic.rs b/rust/examples/basic.rs new file mode 100644 index 00000000..0bc16f0a --- /dev/null +++ b/rust/examples/basic.rs @@ -0,0 +1,50 @@ +use std::sync::Arc; +use std::time::Duration; + +use dialcache::{BoxError, DialCache, KeySpec, Policy}; +use serde::{Deserialize, Serialize}; + +// Results need serialization and comparison for the default codec and shadow +// comparator. They do not need Clone: cached callers share an Arc. +#[derive(Debug, Serialize, Deserialize, PartialEq)] +struct User { + id: u64, + name: String, +} + +async fn load_user(id: u64) -> Result { + println!("Loading user {id} from the source"); + // Replace this with your database or API call. + tokio::time::sleep(Duration::from_millis(10)).await; + Ok(User { + id, + name: "Ada".to_owned(), + }) +} + +#[tokio::main] +async fn main() -> Result<(), BoxError> { + // Application startup: build the cache and register each use case once. + let cache = DialCache::builder().namespace("my-app").build()?; + let users = cache + .use_case::("user", "byId") + .policy(Policy::default().request_local(true).local_ttl_sec(30)) + .key(|id: &u64| KeySpec::new(id)) + .source(|_scope, id| load_user(id)) + .register()?; + + // Request handler: pass this scope to all cached calls in the request. + let request = cache.enable_guard(); + let user: Arc = users.get(request.scope(), 42).await?; + let again = users.get(request.scope(), 42).await?; + assert!(Arc::ptr_eq(&user, &again)); + println!("Hello, {}! Repeated calls share the same value.", user.name); + drop(request); // Closes the request cache; the returned Arc remains usable. + + // Another request can still use the 30-second process-local entry. + let next_request = cache.enable_guard(); + let next = users.get(next_request.scope(), 42).await?; + assert!(Arc::ptr_eq(&user, &next)); + println!("The next request reused the process-local entry."); + Ok(()) +} diff --git a/rust/examples/redis.rs b/rust/examples/redis.rs new file mode 100644 index 00000000..a34236bc --- /dev/null +++ b/rust/examples/redis.rs @@ -0,0 +1,45 @@ +use std::env; +use std::time::Duration; + +use dialcache::{BoxError, DialCache, KeySpec, Policy, RedisAdapter}; +use redis::AsyncConnectionConfig; + +#[tokio::main] +async fn main() -> Result<(), BoxError> { + let url = env::var("REDIS_URL").unwrap_or_else(|_| "redis://127.0.0.1/".to_owned()); + // Connection establishment, command deadlines and concurrency belong to + // the application. This connection does not automatically retry commands. + let config = AsyncConnectionConfig::new() + .set_connection_timeout(Some(Duration::from_secs(2))) + .set_response_timeout(Some(Duration::from_millis(250))) + .set_concurrency_limit(64); + let connection = redis::Client::open(url)? + .get_multiplexed_async_connection_with_config(&config) + .await?; + let cache = DialCache::builder() + .namespace("dialcache-example") + .remote(RedisAdapter::new(connection)) + .build()?; + let names = cache + .use_case::("user", "displayName") + .policy(Policy::default().remote_ttl_sec(300)) + .tracked(true) + .key(|id: &u64| KeySpec::new(id)) + .source(|_scope, id| async move { + println!("Loading user {id} from the source"); + // Replace this with your database or API call. + Ok(format!("User {id}")) + }) + .register()?; + + let request = cache.enable_guard(); + let name = names.get(request.scope(), 42).await?; + println!("Hello, {name}!"); + drop(request); + + // Invalidate the demo's tracked Redis entry. In an application, call this + // after successfully updating the source. Zero adds no future time buffer. + // Existing request/process-local entries retain their normal lifetimes. + cache.invalidate("user", 42_u64, 0).await?; + Ok(()) +} diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml new file mode 100644 index 00000000..ef662bfe --- /dev/null +++ b/rust/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.98.1" +components = ["clippy", "rustfmt"] diff --git a/rust/src/blocking.rs b/rust/src/blocking.rs new file mode 100644 index 00000000..75490c4c --- /dev/null +++ b/rust/src/blocking.rs @@ -0,0 +1,101 @@ +//! Bounded CPU dispatch shared by production runtimes. + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::{mpsc, Arc, OnceLock}; + +use futures::channel::oneshot; +use parking_lot::Mutex; + +use crate::error::BoxError; +use crate::execution::PanicError; +use crate::flight::{panic_message, DROPPED_MESSAGE}; +use crate::runtime::Runtime; + +type Job = Box; + +struct Pool { + sender: mpsc::SyncSender, +} + +impl Pool { + fn new(workers: usize, queued: usize) -> std::io::Result { + let (sender, receiver) = mpsc::sync_channel::(queued); + let receiver = Arc::new(Mutex::new(receiver)); + for index in 0..workers { + let receiver = receiver.clone(); + std::thread::Builder::new() + .name(format!("dialcache-cpu-{index}")) + .spawn(move || loop { + let next = receiver.lock().recv(); + let Ok(job) = next else { break }; + // A custom job must not take a worker out of the pool. + let _ = catch_unwind(AssertUnwindSafe(job)); + })?; + } + Ok(Self { sender }) + } + + fn submit(&self, job: Job) -> Result<(), BoxError> { + self.sender + .try_send(job) + .map_err(|_| "DialCache CPU queue is full or unavailable".into()) + } +} + +pub(crate) fn submit(job: Job) -> Result<(), BoxError> { + static POOL: OnceLock> = OnceLock::new(); + POOL.get_or_init(|| Pool::new(2, 2).map_err(|e| e.to_string())) + .as_ref() + .map_err(|e| -> BoxError { format!("Could not start DialCache CPU workers: {e}").into() })? + .submit(job) +} + +/// Waiting does not own or cancel the admitted raw job. Its closure owns its +/// payload and any shadow slot until it finishes, even if the runtime closes. +pub(crate) async fn run( + runtime: &dyn Runtime, + job: impl FnOnce() -> Result + Send + 'static, +) -> Result { + let (send, receive) = oneshot::channel(); + catch_unwind(AssertUnwindSafe(|| { + runtime.spawn_blocking(Box::new(move || { + let result = catch_unwind(AssertUnwindSafe(job)) + .unwrap_or_else(|p| Err(Box::new(PanicError(panic_message(p))))); + let _ = send.send(result); + })) + })) + .map_err(|p| -> BoxError { Box::new(PanicError(panic_message(p))) })??; + receive + .await + .map_err(|_| -> BoxError { DROPPED_MESSAGE.into() })? +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn pool_bounds_admission_and_keeps_worker_after_panic() { + let pool = Pool::new(1, 1).unwrap(); + let (started, running) = mpsc::channel(); + let (release, gate) = mpsc::channel(); + pool.submit(Box::new(move || { + started.send(()).unwrap(); + gate.recv_timeout(Duration::from_secs(5)).unwrap(); + panic!("worker must survive"); + })) + .unwrap(); + running.recv_timeout(Duration::from_secs(5)).unwrap(); + let (finished, done) = mpsc::channel(); + pool.submit(Box::new(move || { + finished.send(()).unwrap(); + })) + .unwrap(); + assert!(pool + .submit(Box::new(|| panic!("rejected task ran"))) + .is_err()); + release.send(()).unwrap(); + done.recv_timeout(Duration::from_secs(5)).unwrap(); + } +} diff --git a/rust/src/cancel.rs b/rust/src/cancel.rs new file mode 100644 index 00000000..e5fc6540 --- /dev/null +++ b/rust/src/cancel.rs @@ -0,0 +1,112 @@ +//! Cooperative cancellation for remote reads. + +use std::future::Future; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll, Waker}; + +use parking_lot::Mutex; + +#[derive(Default)] +struct Inner { + cancelled: bool, + wakers: Vec, + callbacks: Vec>, +} + +/// A cancellation request handed to remote adapters with each read. +/// +/// The cache cancels the token once the read deadline passes. Cancellation is +/// a request; it does not prove that a dispatched command stopped. +#[derive(Clone, Default)] +pub struct CancelToken { + inner: Arc>, +} + +impl std::fmt::Debug for CancelToken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CancelToken") + .field("cancelled", &self.is_cancelled()) + .finish() + } +} + +impl CancelToken { + /// A token that has not been cancelled; clones share its state. + pub fn new() -> Self { + Self::default() + } + + /// Whether cancellation was requested. + pub fn is_cancelled(&self) -> bool { + self.inner.lock().cancelled + } + + /// Request cancellation once. Later calls do nothing. + pub fn cancel(&self) { + let (wakers, callbacks) = { + let mut inner = self.inner.lock(); + if inner.cancelled { + return; + } + inner.cancelled = true; + ( + std::mem::take(&mut inner.wakers), + std::mem::take(&mut inner.callbacks), + ) + }; + for waker in wakers { + waker.wake(); + } + // A callback belongs to an adapter; its panic must not become the + // outcome of the read that is being abandoned. + for callback in callbacks { + let _ = catch_unwind(AssertUnwindSafe(callback)); + } + } + + /// Run `callback` when cancellation is requested, or immediately if it already was. + pub fn on_cancel(&self, callback: impl FnOnce() + Send + 'static) { + let mut callback = Some(Box::new(callback) as Box); + { + let mut inner = self.inner.lock(); + if !inner.cancelled { + inner + .callbacks + .push(callback.take().expect("callback present")); + } + } + if let Some(callback) = callback { + let _ = catch_unwind(AssertUnwindSafe(callback)); + } + } + + /// Completes once cancellation is requested. + pub fn cancelled(&self) -> Cancelled { + Cancelled { + token: self.clone(), + } + } +} + +/// Future returned by [`CancelToken::cancelled`]. +#[derive(Debug)] +pub struct Cancelled { + token: CancelToken, +} + +impl Future for Cancelled { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + let mut inner = self.token.inner.lock(); + if inner.cancelled { + return Poll::Ready(()); + } + if !inner.wakers.iter().any(|w| w.will_wake(cx.waker())) { + inner.wakers.push(cx.waker().clone()); + } + Poll::Pending + } +} diff --git a/rust/src/clock.rs b/rust/src/clock.rs new file mode 100644 index 00000000..29d31d90 --- /dev/null +++ b/rust/src/clock.rs @@ -0,0 +1,144 @@ +//! Wall and elapsed clocks. + +use std::fmt; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +/// Separates wall timestamps from monotonic elapsed time. +/// +/// Wall time stamps frames and invalidation markers. Elapsed time governs +/// deadlines and process-local expiry. Timers live on the [`Runtime`](crate::Runtime). +pub trait Clock: Send + Sync + 'static { + /// Epoch milliseconds from the application wall clock. + fn wall_ms(&self) -> i64; + /// Monotonic elapsed time since this clock's origin, with native precision. + fn elapsed(&self) -> Duration; + /// Whole-millisecond elapsed reading used by process-local storage. + /// + /// Insertion and lookup share this integer grid, so an entry inserted at + /// 0.7 ms with a 1,000 ms TTL expires when this reading reaches 1,000. + fn elapsed_ms(&self) -> i64 { + self.elapsed().as_millis().min(i64::MAX as u128) as i64 + } +} + +static PROCESS_ORIGIN: OnceLock = OnceLock::new(); + +/// The origin of a clock constructed `now_ns` after the process origin so that +/// its whole-millisecond readings fall on the process-wide grid: the nearest +/// aligned instant at or before construction. +pub fn grid_origin_ns(now_ns: u128) -> u128 { + now_ns - now_ns % 1_000_000 +} + +type MonotonicSource = Arc Duration + Send + Sync>; +type WallSource = Arc i64 + Send + Sync>; + +/// The default clock: system wall time and a monotonic origin aligned to the +/// process-wide millisecond grid, so default instances share one local expiry grid. +#[derive(Clone)] +pub struct SystemClock { + monotonic: MonotonicSource, + wall: WallSource, + /// Nanoseconds from the monotonic source's zero to this clock's origin. + origin_ns: u128, +} + +impl SystemClock { + /// System sources: `SystemTime` for wall time and the `Instant` elapsed + /// since the first default clock of this process for monotonic time. + pub fn new() -> Self { + let process = *PROCESS_ORIGIN.get_or_init(Instant::now); + Self::with_sources(move || process.elapsed(), system_wall_ms) + } + + /// The production alignment over caller-supplied sources. + /// + /// `monotonic` reads elapsed time since one fixed instant shared by every + /// clock built over it; this clock's origin is the nearest whole + /// millisecond of that reading at or before construction, so clocks built + /// at different fractional times share one expiry grid. `wall` supplies + /// epoch milliseconds. Controlled tests use this to run the default + /// alignment over a virtual clock. + pub fn with_sources( + monotonic: impl Fn() -> Duration + Send + Sync + 'static, + wall: impl Fn() -> i64 + Send + Sync + 'static, + ) -> Self { + let since_process = monotonic().as_nanos(); + SystemClock { + monotonic: Arc::new(monotonic), + wall: Arc::new(wall), + origin_ns: grid_origin_ns(since_process), + } + } +} + +fn system_wall_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis().min(i64::MAX as u128) as i64) + .unwrap_or(0) +} + +impl Default for SystemClock { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for SystemClock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SystemClock") + .field("origin_ns", &self.origin_ns) + .finish_non_exhaustive() + } +} + +impl Clock for SystemClock { + fn wall_ms(&self) -> i64 { + (self.wall)() + } + + fn elapsed(&self) -> Duration { + let now_ns = (self.monotonic)().as_nanos(); + Duration::from_nanos(now_ns.saturating_sub(self.origin_ns).min(u64::MAX as u128) as u64) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use parking_lot::Mutex; + + #[test] + fn clocks_built_at_different_fractions_share_one_grid() { + let now = Arc::new(Mutex::new(Duration::from_micros(700))); + let source = { + let now = now.clone(); + move || *now.lock() + }; + let first = SystemClock::with_sources(source.clone(), || 0); + *now.lock() = Duration::from_micros(1_300); + let second = SystemClock::with_sources(source, || 0); + *now.lock() = Duration::from_micros(1_000_900); + // Origins are 0 ms and 1 ms: the same grid, one millisecond apart. + assert_eq!(first.elapsed_ms(), 1_000); + assert_eq!(second.elapsed_ms(), 999); + *now.lock() = Duration::from_micros(1_001_000); + assert_eq!(second.elapsed_ms(), 1_000); + } + + #[test] + fn default_clock_is_monotonic_and_shares_the_process_origin() { + let first = SystemClock::new(); + let second = SystemClock::new(); + let a = first.elapsed(); + assert!(first.elapsed() >= a); + // Both origins lie on the process grid: whole milliseconds after the + // process origin, so the two instances share one expiry grid. + assert_eq!(first.origin_ns % 1_000_000, 0); + assert_eq!(second.origin_ns % 1_000_000, 0); + assert!(second.origin_ns >= first.origin_ns); + assert!(first.wall_ms() > 1_700_000_000_000); + } +} diff --git a/rust/src/codec.rs b/rust/src/codec.rs new file mode 100644 index 00000000..edcd1fd4 --- /dev/null +++ b/rust/src/codec.rs @@ -0,0 +1,150 @@ +//! Serialized payloads and value codecs. + +use std::borrow::Cow; +use std::sync::Arc; + +use futures::future::BoxFuture; +use serde::de::DeserializeOwned; +use serde::Serialize; + +use crate::error::BoxError; + +/// Serializer output: UTF-8 text or opaque binary bytes. +/// +/// Text payloads are stored with the frame's UTF-8 tag; binary payloads keep +/// their exact bytes. Compression envelopes wrap either form transparently. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +pub struct Payload { + /// The serializer output. Read as UTF-8 unless `binary`; ill-formed text + /// sequences are replaced when the payload is stored. + pub bytes: Vec, + /// `true` for binary output, `false` for UTF-8 text. + pub binary: bool, +} + +impl Payload { + /// A UTF-8 text payload. + pub fn text(text: impl Into) -> Self { + Payload { + bytes: text.into().into_bytes(), + binary: false, + } + } + + /// An opaque binary payload, stored byte-exact. + pub fn binary(bytes: impl Into>) -> Self { + Payload { + bytes: bytes.into(), + binary: true, + } + } + + /// Length in bytes. + pub fn len(&self) -> usize { + self.bytes.len() + } + + /// Whether the payload holds no bytes. + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// The bytes as text, replacing ill-formed UTF-8 in binary payloads. + pub fn as_text(&self) -> Cow<'_, str> { + String::from_utf8_lossy(&self.bytes) + } +} + +/// Encodes cached values to payloads and decodes them back. +/// +/// Encoding and decoding are asynchronous so a codec may await. For ordinary +/// synchronous serializers implement [`SyncCodec`] and wrap the codec in +/// [`FromSync`]. Every decode must return an independent value: the cache may +/// retain one payload and decode it more than once. +pub trait Codec: Send + Sync + 'static { + /// Serialize `value`. A failure counts as a `serialization_dump` error + /// and skips the remote write. + fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result>; + + /// Serialize a shared value owned by the cache. + /// + /// The engine calls this method for remote writes. By default it delegates + /// to [`encode`](Self::encode) without copying the value. Override it to + /// move the shared handle into a background job without requiring `T: Clone`. + /// The codec owns scheduling, admission and the lifetime of any jobs it + /// starts; those jobs do not inherit the cache's shadow-capacity token. + fn encode_owned(&self, value: Arc) -> BoxFuture<'_, Result> + where + T: Send + Sync + 'static, + { + Box::pin(async move { self.encode(value.as_ref()).await }) + } + + /// Deserialize `payload` into an independent value. A failure counts as + /// a `serialization_load` error and is treated as a miss. + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result>; +} + +/// A synchronous codec. Wrap it in [`FromSync`] where a [`Codec`] is expected. +pub trait SyncCodec: Send + Sync + 'static { + /// Serialize `value` without awaiting. + fn encode(&self, value: &T) -> Result; + /// Deserialize `payload` into an independent value without awaiting. + fn decode(&self, payload: Payload) -> Result; +} + +/// Adapts a [`SyncCodec`] to [`Codec`]. +#[derive(Debug, Clone, Copy, Default)] +pub struct FromSync(pub C); + +impl> Codec for FromSync { + fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result> { + let result = self.0.encode(value); + Box::pin(std::future::ready(result)) + } + + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result> { + let result = self.0.decode(payload); + Box::pin(std::future::ready(result)) + } +} + +/// Payload text that TypeScript writes for an `undefined` value. +pub const JSON_UNDEFINED_SENTINEL: &str = "__dialcache_json_undefined_v1__"; + +/// The default codec: `serde_json` text. +/// +/// Values written by TypeScript's default serializer decode as long as they +/// are valid JSON for the destination type. The TypeScript `undefined` +/// sentinel decodes as JSON `null`: an `Option` destination reads it as +/// `None`, a destination that accepts `null` (such as `serde_json::Value`) +/// decodes it, and any other destination fails open to the source. +#[derive(Debug, Clone, Copy, Default)] +pub struct JsonCodec; + +impl JsonCodec { + /// Serialize any `Serialize` value as compact JSON text. + pub fn encode_value(value: &T) -> Result { + Ok(Payload::text(serde_json::to_string(value)?)) + } + + /// Deserialize JSON text, reading the TypeScript `undefined` sentinel as + /// JSON `null`. + pub fn decode_value(payload: &Payload) -> Result { + let text = payload.as_text(); + if text == JSON_UNDEFINED_SENTINEL { + return Ok(T::deserialize(serde_json::Value::Null)?); + } + Ok(serde_json::from_str(&text)?) + } +} + +impl Codec for JsonCodec { + fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result> { + Box::pin(std::future::ready(JsonCodec::encode_value(value))) + } + + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result> { + Box::pin(std::future::ready(JsonCodec::decode_value(&payload))) + } +} diff --git a/rust/src/cpu_tests.rs b/rust/src/cpu_tests.rs new file mode 100644 index 00000000..ed24409a --- /dev/null +++ b/rust/src/cpu_tests.rs @@ -0,0 +1,540 @@ +//! Native scheduling and ownership tests for built-in CPU work. +use std::collections::VecDeque; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::BoxFuture; +use parking_lot::Mutex; + +use crate::observe::ShadowOutcome; +use crate::protocol::{compress_payload, CompressionConfig}; +use crate::testing::{StepRuntime, TestExecutor, WALL_EPOCH_MS}; +use crate::*; + +type Job = Box; +struct CpuRuntime { + step: Arc, + jobs: Mutex>, + reject: AtomicBool, +} +impl Runtime for CpuRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.step.spawn(task); + } + fn defer(&self, task: BoxFuture<'static, ()>) { + self.step.defer(task); + } + fn sleep(&self, delay: Duration) -> BoxFuture<'static, ()> { + self.step.sleep(delay) + } + fn spawn_blocking(&self, task: Job) -> Result<(), BoxError> { + if self.reject.load(Ordering::SeqCst) { + return Err("test CPU rejection".into()); + } + self.jobs.lock().push_back(task); + Ok(()) + } +} +impl CpuRuntime { + fn take(&self) -> Job { + self.jobs.lock().pop_front().expect("CPU work admitted") + } +} +struct SnapshotRemote { + result: ReadResult, + writes: AtomicUsize, +} +impl Remote for SnapshotRemote { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(std::future::ready(Ok(self.result.clone()))) + } + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.writes.fetch_add(1, Ordering::SeqCst); + Box::pin(std::future::ready(Ok(()))) + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(std::future::ready(Ok(()))) + } +} +#[derive(Default)] +struct Events(Mutex>); +impl Observer for Events { + fn observes_shadow_outcomes(&self) -> bool { + true + } + fn observe(&self, event: &Event) { + if let Event::ShadowValidation { outcome, .. } = event { + self.0.lock().push(*outcome); + } + } +} +struct TextCodec(Arc); +impl SyncCodec for TextCodec { + fn encode(&self, text: &String) -> Result { + Ok(Payload::text(text.clone())) + } + fn decode(&self, payload: Payload) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(payload.as_text().into_owned()) + } +} +fn text() -> String { + "a".repeat(128 * 1024) +} +fn snapshot(decode: bool) -> ReadResult { + if decode { + ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64, + payload: compress_payload( + Payload::text(text()), + &CompressionConfig::default(), + limits::MAX_DECOMPRESSED_BYTES, + ) + .unwrap() + .payload, + }) + } else { + ReadResult::miss(MissReason::ValueAbsent) + } +} +fn operation(decodes: &Arc, shadow: bool) -> Operation { + let mut policy = Policy::default().remote_ttl_sec(60); + if shadow { + policy = policy.remote_ramp(0.0).shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: None, + }); + } + Operation::with_codec( + Identity::new("thing", "one", "CPU"), + Arc::new(FromSync(TextCodec(decodes.clone()))), + |a, b| a == b, + ) + .policy(policy) + .budget(SourceBudget::Millis(5)) +} +fn setup( + decode: bool, +) -> ( + TestExecutor, + DialCache, + Arc, + Arc, + Arc, +) { + let executor = TestExecutor::new(WALL_EPOCH_MS); + let runtime = Arc::new(CpuRuntime { + step: executor.runtime.clone(), + jobs: Mutex::new(VecDeque::new()), + reject: AtomicBool::new(false), + }); + let remote = Arc::new(SnapshotRemote { + result: snapshot(decode), + writes: AtomicUsize::new(0), + }); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(runtime.clone()) + .remote_arc(remote.clone()) + .observer_arc(events.clone()) + .build() + .unwrap(); + (executor, cache, runtime, remote, events) +} +fn invoke(executor: &mut TestExecutor, cache: &DialCache, op: Operation) -> Arc { + let cache = cache.clone(); + executor.block_on(async move { + let scope = cache.enable_guard(); + cache + .get_or_load(scope.scope(), op, |_| async { Ok(text()) }) + .await + .unwrap() + }) +} + +#[test] +fn rejected_cpu_work_fails_open_through_the_cache_api() { + for decode in [false, true] { + let (mut executor, cache, runtime, remote, events) = setup(decode); + runtime.reject.store(true, Ordering::SeqCst); + let decodes = Arc::new(AtomicUsize::new(0)); + for shadow in [false, true] { + for _ in 0..2 { + assert_eq!( + *invoke(&mut executor, &cache, operation(&decodes, shadow)), + text() + ); + } + assert!(cache.core.state.lock().flights.is_empty()); + assert!(cache.core.state.lock().shadows.is_empty()); + } + assert_eq!(remote.writes.load(Ordering::SeqCst), 0); + assert_eq!(decodes.load(Ordering::SeqCst), 0); + assert_eq!( + *events.0.lock(), + vec![ + if decode { + ShadowOutcome::DeserializationError + } else { + ShadowOutcome::FillError + }; + 2 + ] + ); + } +} + +#[test] +fn shadow_deadline_keeps_cpu_ownership_and_prevents_subsequent_phases() { + for decode in [false, true] { + for deliver in [false, true] { + let (mut executor, cache, runtime, remote, events) = setup(decode); + let decodes = Arc::new(AtomicUsize::new(0)); + invoke(&mut executor, &cache, operation(&decodes, true)); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + assert_eq!(runtime.jobs.lock().len(), 1); + executor.advance(5, deliver); + assert_eq!( + cache.core.state.lock().shadows.len(), + 1, + "timeout released raw CPU ownership" + ); + runtime.take()(); + executor.drain(); + assert_eq!(*events.0.lock(), vec![ShadowOutcome::Timeout]); + assert_eq!( + decodes.load(Ordering::SeqCst), + 0, + "codec ran after decode deadline" + ); + assert_eq!( + remote.writes.load(Ordering::SeqCst), + 0, + "write ran after compression deadline" + ); + assert!(cache.core.state.lock().shadows.is_empty()); + } + } +} + +#[test] +fn queued_cpu_job_keeps_shadow_slot_after_executor_shutdown() { + for discard in [false, true] { + let (mut executor, cache, runtime, _, _) = setup(false); + invoke( + &mut executor, + &cache, + operation(&Arc::new(AtomicUsize::new(0)), true), + ); + drop(executor); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + let job = runtime.take(); + if discard { + drop(job); + } else { + job(); + } + assert!(cache.core.state.lock().shadows.is_empty()); + } +} + +#[test] +fn running_cpu_job_keeps_shadow_slot_after_executor_shutdown() { + let (mut executor, cache, runtime, _, _) = setup(false); + invoke( + &mut executor, + &cache, + operation(&Arc::new(AtomicUsize::new(0)), true), + ); + let job = runtime.take(); + let (started, running) = std::sync::mpsc::channel(); + let (release, gate) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + started.send(()).unwrap(); + gate.recv_timeout(Duration::from_secs(5)).unwrap(); + job(); + }); + running.recv_timeout(Duration::from_secs(5)).unwrap(); + drop(executor); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + release.send(()).unwrap(); + worker.join().unwrap(); + assert!(cache.core.state.lock().shadows.is_empty()); +} + +#[cfg(feature = "tokio")] +#[tokio::test(flavor = "current_thread")] +async fn large_codec_phases_allow_an_independent_short_timer_to_progress() { + struct GatedRuntime { + inner: TokioRuntime, + started: tokio::sync::mpsc::UnboundedSender, + release: Arc>>, + } + impl Runtime for GatedRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.inner.spawn(task); + } + fn sleep(&self, delay: Duration) -> BoxFuture<'static, ()> { + self.inner.sleep(delay) + } + fn spawn_blocking(&self, task: Job) -> Result<(), BoxError> { + let started = self.started.clone(); + let release = self.release.clone(); + self.inner.spawn_blocking(Box::new(move || { + started.send(std::thread::current().id()).unwrap(); + release.lock().recv_timeout(Duration::from_secs(5)).unwrap(); + task(); + })) + } + } + for decode in [false, true] { + let (started, mut started_rx) = tokio::sync::mpsc::unbounded_channel(); + let (release, gate) = std::sync::mpsc::channel(); + let runtime = GatedRuntime { + inner: TokioRuntime::current().unwrap(), + started, + release: Arc::new(Mutex::new(gate)), + }; + let remote = Arc::new(SnapshotRemote { + result: snapshot(decode), + writes: AtomicUsize::new(0), + }); + let cache = DialCache::builder() + .clock_arc(crate::testing::VirtualClock::new(WALL_EPOCH_MS)) + .runtime(runtime) + .remote_arc(remote) + .build() + .unwrap(); + let decodes = Arc::new(AtomicUsize::new(0)); + let op = operation(&decodes, false); + let pending = tokio::spawn(async move { + let guard = cache.enable_guard(); + cache + .get_or_load(guard.scope(), op, |_| async { Ok(text()) }) + .await + .unwrap() + }); + let worker = tokio::time::timeout(Duration::from_secs(5), started_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_ne!(worker, std::thread::current().id()); + tokio::time::sleep(Duration::from_millis(2)).await; + assert!(!pending.is_finished(), "the CPU gate should still be held"); + release.send(()).unwrap(); + assert_eq!( + *tokio::time::timeout(Duration::from_secs(5), pending) + .await + .unwrap() + .unwrap(), + text() + ); + assert_eq!(decodes.load(Ordering::SeqCst), usize::from(decode)); + } +} + +#[derive(Default)] +struct Logs(Mutex>); +impl Logger for Logs { + fn log(&self, event: &LogEvent) { + if let LogEvent::ShadowMismatch(details) = event { + self.0.lock().push(details.clone()); + } + } +} +fn preview_setup() -> ( + TestExecutor, + DialCache, + Arc, + Arc, + Arc, +) { + let executor = TestExecutor::new(WALL_EPOCH_MS); + let runtime = Arc::new(CpuRuntime { + step: executor.runtime.clone(), + jobs: Mutex::new(VecDeque::new()), + reject: AtomicBool::new(false), + }); + let events = Arc::new(Events::default()); + let logs = Arc::new(Logs::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(runtime.clone()) + .observer_arc(events.clone()) + .logger_arc(logs.clone()) + .remote(SnapshotRemote { + result: ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64, + payload: Payload::text(serde_json::to_string(&"é".repeat(10_000)).unwrap()), + }), + writes: AtomicUsize::new(0), + }) + .build() + .unwrap(); + (executor, cache, runtime, events, logs) +} +fn preview_call( + executor: &mut TestExecutor, + cache: &DialCache, + registered: bool, + preview: Option>, +) { + let cache = cache.clone(); + executor.block_on(async move { + let guard = cache.enable_guard(); + let policy = Policy::default().remote_ttl_sec(60).shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: Some(true), + }); + let result = if registered { + let use_case = cache + .use_case::<(), String>("thing", "Preview") + .policy(policy) + .key(|_| KeySpec::new("one")) + .source(|_, ()| async { Ok("source".to_owned()) }) + .register() + .unwrap(); + use_case.get(guard.scope(), ()).await + } else { + let mut operation = + Operation::::new(Identity::new("thing", "one", "Preview")).policy(policy); + if let Some(preview) = preview { + operation.preview = Some(preview); + } + cache + .get_or_load(guard.scope(), operation, |_| async { + Ok("source".to_owned()) + }) + .await + }; + assert_eq!(*result.unwrap(), "é".repeat(10_000)); + }); +} + +#[test] +fn both_default_apis_prepare_bounded_previews_without_holding_the_caller() { + for registered in [false, true] { + let (mut executor, cache, runtime, events, logs) = preview_setup(); + preview_call(&mut executor, &cache, registered, None); + assert_eq!(*events.0.lock(), vec![ShadowOutcome::Mismatch]); + assert!(logs.0.lock().is_empty()); + assert_eq!(runtime.jobs.lock().len(), 1); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + // An inline call uses the same identity as the registered handle. + preview_call(&mut executor, &cache, false, None); + assert_eq!( + *events.0.lock(), + vec![ShadowOutcome::Mismatch, ShadowOutcome::Dropped] + ); + executor.advance(10_000, true); + runtime.take()(); + executor.drain(); + assert!(cache.core.state.lock().shadows.is_empty()); + let messages = logs.0.lock(); + assert_eq!(messages.len(), 1); + assert_eq!( + messages[0].cached_value_json, + crate::preview::json_preview(&"é".repeat(10_000)) + ); + assert_eq!(messages[0].source_value_json.as_deref(), Some("\"source\"")); + assert_eq!( + *events.0.lock(), + vec![ShadowOutcome::Mismatch, ShadowOutcome::Dropped] + ); + } +} + +#[test] +fn rejected_preview_cpu_work_still_logs_the_confirmed_mismatch() { + let (mut executor, cache, runtime, events, logs) = preview_setup(); + runtime.reject.store(true, Ordering::SeqCst); + preview_call(&mut executor, &cache, false, None); + assert_eq!(*events.0.lock(), vec![ShadowOutcome::Mismatch]); + assert!(runtime.jobs.lock().is_empty()); + assert!(cache.core.state.lock().shadows.is_empty()); + let messages = logs.0.lock(); + assert_eq!(messages.len(), 1); + assert!(messages[0].cached_value_json.is_none()); + assert!(messages[0].source_value_json.is_none()); +} + +#[test] +fn custom_preview_failures_are_isolated_per_value_and_success_is_clamped() { + for panic in [false, true] { + let (mut executor, cache, runtime, _, logs) = preview_setup(); + preview_call( + &mut executor, + &cache, + false, + Some(Arc::new(move |value| { + if value == "source" { + Some("🙂".repeat(10_000)) + } else if panic { + panic!("bad cached preview"); + } else { + None + } + })), + ); + runtime.take()(); + executor.drain(); + let messages = logs.0.lock(); + assert_eq!(messages.len(), 1); + assert!(messages[0].cached_value_json.is_none()); + assert_eq!( + messages[0].source_value_json, + Some(crate::preview::preview_value(&"🙂".repeat(10_000))) + ); + assert!(cache.core.state.lock().shadows.is_empty()); + } +} + +#[test] +fn queued_preview_keeps_capacity_after_executor_shutdown_until_run_or_discarded() { + for discard in [false, true] { + let (mut executor, cache, runtime, _, _) = preview_setup(); + preview_call(&mut executor, &cache, false, None); + drop(executor); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + let job = runtime.take(); + if discard { + drop(job); + } else { + job(); + } + assert!(cache.core.state.lock().shadows.is_empty()); + } +} + +#[test] +fn running_preview_keeps_capacity_after_executor_shutdown() { + let (mut executor, cache, runtime, _, _) = preview_setup(); + let (started, running) = std::sync::mpsc::channel(); + let (release, gate) = std::sync::mpsc::channel(); + let gate = Mutex::new(gate); + preview_call( + &mut executor, + &cache, + false, + Some(Arc::new(move |value| { + if value != "source" { + started.send(std::thread::current().id()).unwrap(); + gate.lock().recv_timeout(Duration::from_secs(5)).unwrap(); + } + Some(value.clone()) + })), + ); + let job = runtime.take(); + let worker = std::thread::spawn(job); + assert_ne!( + running.recv_timeout(Duration::from_secs(5)).unwrap(), + std::thread::current().id() + ); + drop(executor); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + release.send(()).unwrap(); + worker.join().unwrap(); + assert!(cache.core.state.lock().shadows.is_empty()); +} diff --git a/rust/src/datadog.rs b/rust/src/datadog.rs new file mode 100644 index 00000000..60f49044 --- /dev/null +++ b/rust/src/datadog.rs @@ -0,0 +1,243 @@ +//! Datadog (DogStatsD) metric exporter. +//! +//! [`DatadogObserver`] publishes every [`Event`] to a caller-supplied +//! [`DogStatsdClient`] under the metric names, units and tags of the +//! TypeScript and Go adapters. Transport, buffering, flushing and ownership of +//! the client stay with the caller. + +use std::fmt; + +use crate::metrics::MetricKind; +use crate::observe::{Event, Observer}; + +/// The subset of the DogStatsD client API the exporter uses. +/// +/// `tags` are `(name, value)` pairs in wire order. Implementations format +/// them as `name:value` for their transport. +pub trait DogStatsdClient: Send + Sync + 'static { + /// Add `value` to the counter `name`. + fn increment(&self, name: &str, value: f64, tags: &[(String, String)]); + /// Record `value` in the histogram `name`. + fn histogram(&self, name: &str, value: f64, tags: &[(String, String)]); + /// Record `value` in the distribution `name`. + fn distribution(&self, name: &str, value: f64, tags: &[(String, String)]); +} + +/// How observations (timers, ages, sizes and ratios) are published. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ObservationMetricType { + /// Agent-side percentiles per host. + Histogram, + /// Server-side global percentiles. + Distribution, +} + +/// Construction options of a [`DatadogObserver`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DatadogOptions { + /// Whether observations go to DogStatsD histograms or distributions. + pub observation_metric_type: ObservationMetricType, + /// Metric-name namespace; unrelated to the cache namespace label. + /// `None` selects [`DEFAULT_NAMESPACE`]; an explicit empty string is an error. + pub namespace: Option, +} + +impl DatadogOptions { + /// Options under the default metric-name namespace, [`DEFAULT_NAMESPACE`]. + pub fn new(observation_metric_type: ObservationMetricType) -> Self { + DatadogOptions { + observation_metric_type, + namespace: None, + } + } + + /// Set the metric-name namespace; it must satisfy [`is_valid_namespace`]. + pub fn namespace(mut self, namespace: impl Into) -> Self { + self.namespace = Some(namespace.into()); + self + } +} + +/// The namespace used when [`DatadogOptions::namespace`] is `None`. +pub const DEFAULT_NAMESPACE: &str = "dialcache"; +/// Datadog's metric name length limit. +pub const METRIC_NAME_MAX_LENGTH: usize = 200; + +/// Construction failures of a [`DatadogObserver`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DatadogError { + /// The namespace does not match `^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z0-9_]+)*$`. + InvalidNamespace(String), + /// A metric name would exceed [`METRIC_NAME_MAX_LENGTH`] characters. + MetricNameTooLong(String), +} + +impl fmt::Display for DatadogError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DatadogError::InvalidNamespace(namespace) => write!( + f, + "Datadog namespace {namespace:?} must start with a letter and contain only \ + letters, numbers, underscores, and dot-separated non-empty segments" + ), + DatadogError::MetricNameTooLong(name) => write!( + f, + "Datadog metric name {name:?} exceeds the {METRIC_NAME_MAX_LENGTH}-character limit" + ), + } + } +} + +impl std::error::Error for DatadogError {} + +/// The metric-name suffix of each kind, appended to the namespace. +pub fn metric_suffix(kind: MetricKind) -> &'static str { + match kind { + MetricKind::Request => "request.count", + MetricKind::Miss => "miss.count", + MetricKind::Disabled => "disabled.count", + MetricKind::Error => "error.count", + MetricKind::Invalidation => "invalidation.count", + MetricKind::Coalesced => "coalesced.count", + MetricKind::ShadowValidation => "shadow.count", + MetricKind::ShadowValueAge => "shadow.value_age", + MetricKind::FutureTimestampOffset => "future_timestamp_offset", + MetricKind::StaleRecovery => "stale_recovery.count", + MetricKind::StaleRecoveryValueAge => "stale_recovery.value_age", + MetricKind::Compression => "compression.count", + MetricKind::Get => "get.duration", + MetricKind::Fallback => "fallback.duration", + MetricKind::Serialization => "serialization.duration", + MetricKind::Size => "serialization.size", + MetricKind::StoredSize => "stored.size", + MetricKind::CompressionRatio => "compression.ratio", + MetricKind::CompressionDuration => "compression.duration", + } +} + +/// Whether `namespace` matches `^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z0-9_]+)*$`. +pub fn is_valid_namespace(namespace: &str) -> bool { + let mut segments = namespace.split('.'); + let Some(first) = segments.next() else { + return false; + }; + if !first + .chars() + .next() + .is_some_and(|c| c.is_ascii_alphabetic()) + { + return false; + } + let segment_ok = |segment: &str| { + !segment.is_empty() + && segment + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_') + }; + segment_ok(first) && segments.all(segment_ok) +} + +/// Publishes DialCache diagnostics to DogStatsD. Counters increment by one +/// per event; observations use the configured metric type. +pub struct DatadogObserver { + client: Box, + names: Vec, + distribution: bool, +} + +impl fmt::Debug for DatadogObserver { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DatadogObserver") + .field("distribution", &self.distribution) + .field("names", &self.names) + .finish_non_exhaustive() + } +} + +impl DatadogObserver { + /// Validate the namespace and precompute every metric name. Fails when + /// the namespace is invalid or a name would exceed + /// [`METRIC_NAME_MAX_LENGTH`]. + pub fn new( + client: impl DogStatsdClient, + options: DatadogOptions, + ) -> Result { + let namespace = options.namespace.as_deref().unwrap_or(DEFAULT_NAMESPACE); + if !is_valid_namespace(namespace) { + return Err(DatadogError::InvalidNamespace(namespace.to_string())); + } + let mut names = Vec::with_capacity(MetricKind::ALL.len()); + for kind in MetricKind::ALL { + let name = format!("{namespace}.{}", metric_suffix(kind)); + if name.chars().count() > METRIC_NAME_MAX_LENGTH { + return Err(DatadogError::MetricNameTooLong(name)); + } + names.push(name); + } + Ok(DatadogObserver { + client: Box::new(client), + names, + distribution: options.observation_metric_type == ObservationMetricType::Distribution, + }) + } + + /// The full metric name of `kind` under this observer's namespace. + pub fn metric_name(&self, kind: MetricKind) -> &str { + &self.names[kind.index()] + } +} + +impl Observer for DatadogObserver { + fn observe(&self, event: &Event) { + let kind = MetricKind::of(event); + let name = self.metric_name(kind); + let tags: Vec<(String, String)> = MetricKind::labels(event) + .into_iter() + .map(|(label, value)| (label.to_string(), value)) + .collect(); + if kind.is_counter() { + self.client.increment(name, 1.0, &tags); + } else if self.distribution { + self.client + .distribution(name, MetricKind::value(event), &tags); + } else { + self.client.histogram(name, MetricKind::value(event), &tags); + } + } + + fn observes_shadow_outcomes(&self) -> bool { + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn namespace_pattern_matches_the_reference_regex() { + for valid in ["dialcache", "app.cache", "A1_.b_2", "a", "x.y.z"] { + assert!(is_valid_namespace(valid), "{valid}"); + } + for invalid in [ + "", + "1bad", + "has-dash", + "two..dots", + ".lead", + "trail.", + "_under", + "sp ace", + "ünï", + ] { + assert!(!is_valid_namespace(invalid), "{invalid}"); + } + } + + #[test] + fn every_kind_has_a_distinct_suffix() { + let suffixes: std::collections::HashSet<&str> = + MetricKind::ALL.iter().map(|k| metric_suffix(*k)).collect(); + assert_eq!(suffixes.len(), MetricKind::ALL.len()); + } +} diff --git a/rust/src/deadline.rs b/rust/src/deadline.rs new file mode 100644 index 00000000..3f2f7eae --- /dev/null +++ b/rust/src/deadline.rs @@ -0,0 +1,61 @@ +//! Monotonic deadlines over controlled clocks. + +use std::time::Duration; + +use futures::future::{select, Either}; + +use crate::clock::Clock; +use crate::flight::Settled; +use crate::runtime::Runtime; + +/// Elapsed time since `start`, never negative. +pub(crate) fn since(clock: &dyn Clock, start: Duration) -> Duration { + clock.elapsed().saturating_sub(start) +} + +pub(crate) fn seconds_since(clock: &dyn Clock, start: Duration) -> f64 { + since(clock, start).as_secs_f64() +} + +/// Wait for `pending`, accepting only a result observed strictly before the +/// deadline `started + budget_ms`. Raw work keeps its resources after the +/// caller stops waiting. `None` budget waits without bound. +pub(crate) async fn await_deadline( + clock: &dyn Clock, + runtime: &dyn Runtime, + pending: &Settled, + started: Duration, + budget_ms: Option, + on_timeout: impl FnOnce() -> T, +) -> T { + let Some(budget_ms) = budget_ms else { + return pending.wait().await; + }; + let budget = Duration::from_millis(budget_ms); + loop { + if let Some(value) = pending.peek() { + if since(clock, started) < budget { + return value; + } + break; + } + let remaining = budget.saturating_sub(since(clock, started)); + let timer = runtime.sleep(remaining); + match select(pending.wait(), timer).await { + Either::Left((value, _timer)) => { + if since(clock, started) < budget { + return value; + } + break; + } + Either::Right(((), _wait)) => { + // Timer precision and delivery do not define the semantic boundary. + if since(clock, started) < budget { + continue; + } + break; + } + } + } + on_timeout() +} diff --git a/rust/src/engine.rs b/rust/src/engine.rs new file mode 100644 index 00000000..c7b6dd43 --- /dev/null +++ b/rust/src/engine.rs @@ -0,0 +1,770 @@ +//! The cache instance: construction, scopes, maintenance and call admission. + +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::num::NonZeroUsize; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use futures::future::BoxFuture; +use parking_lot::Mutex; + +use crate::clock::Clock; +use crate::error::{BoxError, ConfigError, Error}; +use crate::flight::{start_pending, Flight, Settled, ValueResult}; +use crate::identity::{Identity, IntoKeyId}; +use crate::limits::{ + DEFAULT_LOCAL_CAPACITY, DEFAULT_REMOTE_READ_TIMEOUT_MS, DEFAULT_SHADOW_MAX_IN_FLIGHT, + MAX_DEADLINE_MS, MAX_SAFE_INTEGER, MAX_SUPPORTED_DURATION_MS, WATERMARK_USE_CASE, +}; +use crate::local::{LocalEntry, LocalRead, LocalStore, LruLocalStore, StoredValue}; +use crate::observe::{Event, Labels, Layer, LogEvent, Logger, Observer, OutcomeLabels}; +use crate::operation::{downcast_value, erase_load, ErasedOperation, Operation, RecoveryPredicate}; +use crate::policy::RuntimePolicy; +use crate::protocol::CompressionConfig; +use crate::remote::{InvalidateRequest, Remote}; +use crate::runtime::Runtime; +use crate::scope::{Owner, Scope}; +use crate::shadow::ShadowFlight; + +/// Resolves a sparse runtime policy overlay once per enabled call. +pub type PolicyProvider = Arc< + dyn Fn(Identity) -> BoxFuture<'static, Result, BoxError>> + Send + Sync, +>; + +static NEXT_CACHE_ID: AtomicU64 = AtomicU64::new(1); + +pub(crate) struct CoreState { + pub(crate) local: Option>, + pub(crate) flights: HashMap>, + pub(crate) shadows: HashMap>, + pub(crate) registered: HashSet, +} + +pub(crate) struct Core { + pub(crate) id: u64, + pub(crate) namespace: Arc, + pub(crate) remote: Option>, + pub(crate) observer: Option>, + pub(crate) logger: Arc, + pub(crate) provider: Option, + pub(crate) remote_read_timeout_ms: u64, + pub(crate) shadow_max_in_flight: usize, + pub(crate) should_recover: Option, + /// `None` disables new compressed writes; reads still decompress. + pub(crate) compression: Option, + pub(crate) clock: Arc, + pub(crate) runtime: Arc, + pub(crate) state: Mutex, +} + +impl Core { + /// Deliver a diagnostic. Observer failures never change results. + pub(crate) fn emit(&self, event: Event) { + if let Some(observer) = &self.observer { + let _ = catch_unwind(AssertUnwindSafe(|| observer.observe(&event))); + } + } + + pub(crate) fn log(&self, event: LogEvent) { + let _ = catch_unwind(AssertUnwindSafe(|| self.logger.log(&event))); + } + + pub(crate) fn shadow_hook_enabled(&self) -> bool { + match &self.observer { + Some(observer) => { + catch_unwind(AssertUnwindSafe(|| observer.observes_shadow_outcomes())) + .unwrap_or(false) + } + None => false, + } + } + + /// Read a live local entry, promoting it. `Ok(None)` is a miss. + pub(crate) fn local_get(&self, key: &str) -> Result, BoxError> { + // Read the clock before taking the lock: it is the only external code + // on this path, and a failure must not poison shared state. + let now_ms = self.clock.elapsed_ms(); + let read = { + let mut state = self.state.lock(); + match state.local.as_mut() { + None => return Ok(None), + Some(local) => catch_unwind(AssertUnwindSafe(|| local.get(key, now_ms))) + .unwrap_or_else(|payload| { + Err(crate::flight::panic_message(payload).to_string().into()) + }), + } + }; + // An expired entry drops here, outside the lock: a value's destructor + // may call back into the cache. + match read? { + LocalRead::Live(value) => Ok(Some(value)), + LocalRead::Absent | LocalRead::Expired(_) => Ok(None), + } + } + + pub(crate) fn local_put( + &self, + key: &str, + value: StoredValue, + ttl_ms: u64, + ) -> Result<(), BoxError> { + let now_ms = self.clock.elapsed_ms(); + let displaced = { + let mut state = self.state.lock(); + match state.local.as_mut() { + None => return Ok(()), + Some(local) => { + let entry = LocalEntry { + value, + inserted_ms: now_ms, + ttl_ms: ttl_ms.min(i64::MAX as u64) as i64, + }; + catch_unwind(AssertUnwindSafe(|| local.put(key.to_string(), entry))) + .unwrap_or_else(|payload| { + Err(crate::flight::panic_message(payload).to_string().into()) + }) + } + } + }; + // The displaced entry drops here, outside the lock. + displaced.map(|_| ()) + } +} + +/// Exact process-scoped single-flight state of one instance. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ProcessCoalescingState { + /// Executions currently in flight on this instance, one per logical key. + pub active_leaders: usize, + /// Callers currently waiting on those executions instead of running their own. + pub active_followers: usize, + /// Milliseconds since the oldest in-flight execution started; `None` + /// when nothing is in flight. + pub oldest_leader_age_ms: Option, +} + +/// A point-in-time snapshot of cache-owned coalescing state. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct CoalescingState { + /// Instance-wide single-flight state; request-scoped flights are not counted. + pub process: ProcessCoalescingState, +} + +/// Configures a [`DialCache`]. +pub struct DialCacheBuilder { + namespace: String, + remote: Option>, + observer: Option>, + logger: Option>, + provider: Option, + local_capacity: Option, + local_store: Option>, + remote_read_timeout_ms: Option, + shadow_max_in_flight: Option, + should_recover: Option, + compression: Option>, + clock: Option>, + runtime: Option>, +} + +impl std::fmt::Debug for DialCacheBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DialCacheBuilder") + .field("namespace", &self.namespace) + .finish_non_exhaustive() + } +} + +impl DialCacheBuilder { + fn new() -> Self { + DialCacheBuilder { + namespace: "urn".to_string(), + remote: None, + observer: None, + logger: None, + provider: None, + local_capacity: None, + local_store: None, + remote_read_timeout_ms: None, + shadow_max_in_flight: None, + should_recover: None, + compression: None, + clock: None, + runtime: None, + } + } + + /// Logical namespace used in keys, invalidation identity, cohorts and + /// labels. Defaults to `"urn"`; may not contain `{` or `}`. + pub fn namespace(mut self, namespace: impl Into) -> Self { + self.namespace = namespace.into(); + self + } + + /// The remote layer adapter. Without one, only request and local layers serve. + pub fn remote(mut self, remote: impl Remote) -> Self { + self.remote = Some(Arc::new(remote)); + self + } + + /// The remote layer adapter, shared. + pub fn remote_arc(mut self, remote: Arc) -> Self { + self.remote = Some(remote); + self + } + + /// Receives every public diagnostic; also the shadow outcome hook. + pub fn observer(mut self, observer: impl Observer) -> Self { + self.observer = Some(Arc::new(observer)); + self + } + + /// Receives every public diagnostic, shared. + pub fn observer_arc(mut self, observer: Arc) -> Self { + self.observer = Some(observer); + self + } + + /// Receives structured log events. Defaults to the `log` crate facade + /// under target `dialcache`. + pub fn logger(mut self, logger: impl Logger) -> Self { + self.logger = Some(Arc::new(logger)); + self + } + + /// Receives structured log events, shared. + pub fn logger_arc(mut self, logger: Arc) -> Self { + self.logger = Some(logger); + self + } + + /// Runtime policy overlay resolved once per enabled call. `Ok(None)` + /// inherits the operation policy; an error bypasses caching for that call. + pub fn policy_provider(mut self, provider: F) -> Self + where + F: Fn(Identity) -> Fut + Send + Sync + 'static, + Fut: Future, BoxError>> + Send + 'static, + { + self.provider = Some(Arc::new(move |identity| Box::pin(provider(identity)))); + self + } + + /// Maximum process-local entries across every use case. Zero disables + /// local storage while preserving coalescing. Defaults to 10,000. + pub fn local_capacity(mut self, capacity: usize) -> Self { + self.local_capacity = Some(capacity); + self + } + + /// Replace the default LRU store. Advanced: the store must preserve the + /// documented LRU, expiry and promotion rules. + pub fn local_store(mut self, store: Box) -> Self { + self.local_store = Some(store); + self + } + + /// Instance default remote read budget. Defaults to 50 ms. + pub fn remote_read_timeout_ms(mut self, ms: u64) -> Self { + self.remote_read_timeout_ms = Some(ms); + self + } + + /// Concurrent shadow jobs per instance; excess work is dropped. Defaults to 1. + pub fn shadow_max_in_flight(mut self, jobs: usize) -> Self { + self.shadow_max_in_flight = Some(jobs); + self + } + + /// Instance default classifier deciding whether a source failure may use + /// a retained stale value. Omission admits only the source deadline error. + pub fn should_recover( + mut self, + predicate: impl Fn(&Error) -> bool + Send + Sync + 'static, + ) -> Self { + self.should_recover = Some(Arc::new(predicate)); + self + } + + /// Write-side compression policy. Enabled by default at a 4,096-byte threshold. + pub fn compression(mut self, config: CompressionConfig) -> Self { + self.compression = Some(Some(config)); + self + } + + /// Store every payload uncompressed. Reads still accept compressed entries. + pub fn disable_compression(mut self) -> Self { + self.compression = Some(None); + self + } + + /// Replace the wall and elapsed clock. Defaults to + /// [`SystemClock`](crate::SystemClock). + pub fn clock(mut self, clock: impl Clock) -> Self { + self.clock = Some(Arc::new(clock)); + self + } + + /// Share a clock between instances (for example a controlled test clock). + pub fn clock_arc(mut self, clock: Arc) -> Self { + self.clock = Some(clock); + self + } + + /// Share a runtime between instances. + pub fn runtime_arc(mut self, runtime: Arc) -> Self { + self.runtime = Some(runtime); + self + } + + /// Replace the executor and timer source. Defaults to tokio. + pub fn runtime(mut self, runtime: impl Runtime) -> Self { + self.runtime = Some(Arc::new(runtime)); + self + } + + /// Validate the configuration and create the instance. Rejects a + /// namespace containing `{` or `}`, a read budget outside + /// `1..=2_147_483_647` ms, a zero shadow cap, an invalid compression + /// config, or no usable runtime: with the `tokio` feature, no tokio + /// runtime is current and none was supplied through `runtime` or + /// `runtime_arc`; without it, no runtime was supplied. + pub fn build(self) -> Result { + if self.namespace.contains(['{', '}']) { + return Err(ConfigError::invalid( + "DialCache namespace must not contain \"{\" or \"}\"", + )); + } + let compression = match self.compression { + None => Some(CompressionConfig::default()), + Some(None) => None, + Some(Some(config)) => { + config + .validate() + .map_err(|e| ConfigError::invalid(e.to_string()))?; + Some(config) + } + }; + let remote_read_timeout_ms = self + .remote_read_timeout_ms + .unwrap_or(DEFAULT_REMOTE_READ_TIMEOUT_MS); + if !(1..=MAX_DEADLINE_MS).contains(&remote_read_timeout_ms) { + return Err(ConfigError::invalid(format!( + "DialCache remote read timeout must be a positive integer no greater than {MAX_DEADLINE_MS} ms" + ))); + } + let shadow_max_in_flight = self + .shadow_max_in_flight + .unwrap_or(DEFAULT_SHADOW_MAX_IN_FLIGHT); + if shadow_max_in_flight == 0 { + return Err(ConfigError::invalid( + "DialCache shadow_max_in_flight must be positive", + )); + } + let local: Option> = match self.local_store { + Some(store) => Some(store), + None => { + let capacity = self.local_capacity.unwrap_or(DEFAULT_LOCAL_CAPACITY); + if capacity as u64 > MAX_SAFE_INTEGER { + return Err(ConfigError::invalid( + "DialCache local capacity must be a safe integer", + )); + } + NonZeroUsize::new(capacity) + .map(|capacity| Box::new(LruLocalStore::new(capacity)) as Box) + } + }; + let clock = match self.clock { + Some(clock) => clock, + None => default_clock()?, + }; + let runtime = match self.runtime { + Some(runtime) => runtime, + None => default_runtime()?, + }; + let core = Core { + id: NEXT_CACHE_ID.fetch_add(1, Ordering::Relaxed), + namespace: Arc::from(self.namespace.as_str()), + remote: self.remote, + observer: self.observer, + logger: self + .logger + .unwrap_or_else(|| Arc::new(crate::observe::LogFacadeLogger)), + provider: self.provider, + remote_read_timeout_ms, + shadow_max_in_flight, + should_recover: self.should_recover, + compression, + clock, + runtime, + state: Mutex::new(CoreState { + local, + flights: HashMap::new(), + shadows: HashMap::new(), + registered: HashSet::new(), + }), + }; + Ok(DialCache { + core: Arc::new(core), + }) + } +} + +fn default_clock() -> Result, ConfigError> { + Ok(Arc::new(crate::clock::SystemClock::new())) +} + +#[cfg(feature = "tokio")] +fn default_runtime() -> Result, ConfigError> { + Ok(Arc::new(crate::runtime::TokioRuntime::current()?)) +} + +#[cfg(not(feature = "tokio"))] +fn default_runtime() -> Result, ConfigError> { + Err(ConfigError::invalid("DialCache needs a runtime: enable the tokio feature or supply one with DialCacheBuilder::runtime")) +} + +/// Holds the outermost enabled scope open until dropped. +#[derive(Debug)] +pub struct ScopeGuard { + scope: Scope, + owner: Arc, +} + +impl ScopeGuard { + /// The enabled scope of this request. + pub fn scope(&self) -> &Scope { + &self.scope + } +} + +impl Drop for ScopeGuard { + fn drop(&mut self) { + self.owner.close(); + } +} + +/// A cache instance: request, process-local and remote layers behind explicit enablement. +/// +/// Instances are cheap to clone and share one state. +#[derive(Clone)] +pub struct DialCache { + pub(crate) core: Arc, +} + +impl std::fmt::Debug for DialCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DialCache") + .field("namespace", &self.core.namespace) + .finish_non_exhaustive() + } +} + +impl DialCache { + /// Start configuring an instance with the defaults documented on + /// [`DialCacheBuilder`]'s setters. + pub fn builder() -> DialCacheBuilder { + DialCacheBuilder::new() + } + + /// The instance namespace. + pub fn namespace(&self) -> &str { + &self.core.namespace + } + + /// Open the outermost enabled scope and hold it in a guard. The scope + /// closes when the guard drops; retained [`Scope`] clones then pass through. + /// + /// ```ignore + /// let request = cache.enable_guard(); + /// let name = display_name.get(request.scope(), id).await?; + /// ``` + pub fn enable_guard(&self) -> ScopeGuard { + let owner = Owner::new(); + let scope = Scope { + cache_id: self.core.id, + owner: Some(owner.clone()), + enabled: true, + }; + ScopeGuard { scope, owner } + } + + /// Open the outermost enabled scope for `f`. The scope closes when the + /// returned future completes, is dropped before completion, or unwinds; + /// retained clones then pass through. + pub async fn enable(&self, f: F) -> R + where + F: FnOnce(Scope) -> Fut, + Fut: Future, + { + let guard = self.enable_guard(); + let scope = guard.scope().clone(); + let result = f(scope).await; + drop(guard); + result + } + + /// Enable caching inside `parent`, reusing its live outer request memo. + /// When `parent` has no live outer scope, this opens a new one. + pub async fn enable_in(&self, parent: &Scope, f: F) -> R + where + F: FnOnce(Scope) -> Fut, + Fut: Future, + { + let live = if parent.cache_id == self.core.id { + parent.live_owner() + } else { + None + }; + match live { + Some(owner) => { + let scope = Scope { + cache_id: self.core.id, + owner: Some(owner), + enabled: true, + }; + f(scope).await + } + None => self.enable(f).await, + } + } + + /// Disable caching inside `parent` while preserving its outer request memo + /// for nested re-enablement. + pub async fn disable_in(&self, parent: &Scope, f: F) -> R + where + F: FnOnce(Scope) -> Fut, + Fut: Future, + { + let owner = if parent.cache_id == self.core.id { + parent.owner.clone() + } else { + None + }; + let scope = Scope { + cache_id: self.core.id, + owner, + enabled: false, + }; + f(scope).await + } + + /// Whether calls made with `scope` on this instance use caching. + pub fn is_enabled(&self, scope: &Scope) -> bool { + scope.cache_id == self.core.id && scope.is_enabled() + } + + /// Process-scoped single-flight state of this instance. + pub fn coalescing_state(&self) -> CoalescingState { + let now = self.core.clock.elapsed(); + let state = self.core.state.lock(); + let mut process = ProcessCoalescingState { + active_leaders: state.flights.len(), + ..Default::default() + }; + for flight in state.flights.values() { + process.active_followers += flight.followers(); + let age = now + .saturating_sub(flight.started) + .as_millis() + .min(u64::MAX as u128) as u64; + if process + .oldest_leader_age_ms + .is_none_or(|oldest| age > oldest) + { + process.oldest_leader_age_ms = Some(age); + } + } + CoalescingState { process } + } + + /// Advance the invalidation watermark of every tracked variant of one + /// entity, in this instance's namespace. Call it after the source + /// mutation commits. Requires a remote adapter; failures are returned. + /// IDs use the same [`IntoKeyId`] conversion as [`crate::KeySpec::new`]. + pub async fn invalidate( + &self, + key_type: &str, + id: impl IntoKeyId, + future_buffer_ms: u64, + ) -> Result<(), Error> { + let identity = Identity::new(key_type, id.into_key_id(), WATERMARK_USE_CASE) + .tracked(true) + .namespace(self.core.namespace.to_string()); + self.invalidate_identity(identity, future_buffer_ms).await + } + + /// Advance the watermark for the entity named by `identity` after its + /// source mutation commits. An empty namespace inherits this instance's; + /// an explicit namespace is preserved, exactly as in `get_or_load`. + /// + /// All use cases and argument variants share the entity's watermark. + /// `tracked` is ignored: this operation always invalidates tracked remote + /// values. Local entries and already acquired snapshots retain their lifetimes. + /// Requires a remote adapter; mutation failures are returned to the caller. + pub async fn invalidate_identity( + &self, + mut identity: Identity, + future_buffer_ms: u64, + ) -> Result<(), Error> { + let core = &self.core; + if future_buffer_ms > MAX_SUPPORTED_DURATION_MS { + return Err(ConfigError::invalid( + "DialCache invalidation future buffer must be no greater than 365 days", + ) + .into()); + } + if identity.namespace.is_empty() { + identity.namespace = core.namespace.to_string(); + } + identity.tracked = true; + let namespace: Arc = Arc::from(identity.namespace.as_str()); + let key_type: Arc = Arc::from(identity.key_type.as_str()); + core.emit(Event::Invalidation { + namespace: namespace.clone(), + key_type: key_type.clone(), + layer: Layer::Remote, + }); + let result: Result<(), Error> = async { + let remote = core.remote.clone().ok_or(Error::MissingRemote)?; + let keys = identity + .keys() + .map_err(|e| Error::Config(ConfigError::invalid(e.to_string())))?; + let watermark_key = keys.watermark.ok_or_else(|| { + Error::Config(ConfigError::invalid( + "tracked identity has no watermark key", + )) + })?; + let now = core.clock.wall_ms(); + if now < 0 || now as u64 > MAX_SAFE_INTEGER - future_buffer_ms { + return Err(ConfigError::invalid( + "DialCache invalidation timestamp is outside the safe integer domain", + ) + .into()); + } + let request = InvalidateRequest { + watermark_key, + invalidated_at_ms: now as u64, + future_buffer_ms, + }; + let pending: Settled> = start_pending( + core.runtime.as_ref(), + async move { + remote + .invalidate(request) + .await + .map_err(|e| Error::Remote(Arc::from(e))) + }, + |message| Err(Error::Panic(message)), + ); + pending.wait().await + } + .await; + if let Err(error) = &result { + core.log(LogEvent::InvalidationFailed(error.to_string().into())); + core.emit(Event::Error { + labels: Labels { + namespace, + use_case: Arc::from(WATERMARK_USE_CASE), + key_type, + layer: Layer::Remote, + }, + error: crate::observe::ErrorKind::Invalidation, + in_fallback: false, + }); + } + result + } + + /// Execute one inline cached call. + /// + /// `load` is the source of truth; it may be invoked again later by + /// served-hit shadow validation, so it must be reusable. The returned + /// value is shared by reference: treat it as immutable. + pub async fn get_or_load( + &self, + scope: &Scope, + operation: Operation, + load: F, + ) -> Result, Error> + where + T: Send + Sync + 'static, + F: Fn(Scope) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + let (identity, metadata) = operation.erase(); + let erased = ErasedOperation { + identity, + identity_provider: None, + metadata, + load: erase_load(load), + }; + let value = self.execute(scope, erased).await?; + downcast_value::(value) + } + + /// Validate static operation configuration, then run the execution as + /// detached work whose result every caller awaits. Dropping the returned + /// future does not cancel the execution: sources and publications keep + /// their contracts, as in the reference implementations. + pub(crate) async fn execute( + &self, + scope: &Scope, + mut operation: ErasedOperation, + ) -> ValueResult { + validate_operation(&operation)?; + if operation.identity.namespace.is_empty() { + operation.identity.namespace = self.core.namespace.to_string(); + } + // A scope owned by another instance disables this call but still + // reaches the source, so nested calls on its own instance keep caching. + let core = self.core.clone(); + let scope = scope.clone(); + let pending: Settled = start_pending( + core.runtime.clone().as_ref(), + crate::execution::run(core, scope, Arc::new(operation)), + |message| Err(Error::Panic(message)), + ); + pending.wait().await + } + + pub(crate) fn register_use_case(&self, use_case: &str) -> Result<(), ConfigError> { + if use_case == WATERMARK_USE_CASE { + return Err(ConfigError::ReservedUseCase(use_case.to_string())); + } + let mut state = self.core.state.lock(); + if !state.registered.insert(use_case.to_string()) { + return Err(ConfigError::UseCaseAlreadyRegistered(use_case.to_string())); + } + Ok(()) + } +} + +pub(crate) fn validate_operation(operation: &ErasedOperation) -> Result<(), Error> { + operation + .metadata + .policy + .validate() + .map_err(|e| ConfigError::invalid(e.to_string()))?; + if let crate::operation::SourceBudget::Millis(ms) = operation.metadata.budget { + if !(1..=MAX_DEADLINE_MS).contains(&ms) { + return Err(ConfigError::invalid(format!( + "DialCache source budget must be a positive integer no greater than {MAX_DEADLINE_MS} ms" + )) + .into()); + } + } + if operation.identity.use_case == WATERMARK_USE_CASE { + return Err(ConfigError::ReservedUseCase(operation.identity.use_case.clone()).into()); + } + Ok(()) +} + +pub(crate) fn outcome_labels(identity: &Identity) -> OutcomeLabels { + OutcomeLabels { + namespace: Arc::from(identity.namespace.as_str()), + use_case: Arc::from(identity.use_case.as_str()), + key_type: Arc::from(identity.key_type.as_str()), + } +} diff --git a/rust/src/error.rs b/rust/src/error.rs new file mode 100644 index 00000000..4133f6db --- /dev/null +++ b/rust/src/error.rs @@ -0,0 +1,131 @@ +//! Public error types. + +use std::fmt; +use std::sync::Arc; + +/// Boxed error returned by sources, codecs, providers and adapters. +pub type BoxError = Box; +/// Shared error handed to every caller that joined one execution. +pub type SharedError = Arc; + +/// Every failure a cached call or maintenance operation can report. +/// +/// Cache plumbing fails open, so a call fails only when its source fails, +/// its source deadline elapses, a callback panics, or static configuration +/// is rejected before execution. Maintenance operations surface their own +/// remote failures. +#[derive(Debug, Clone)] +pub enum Error { + /// The source returned an error. Every coalesced caller receives the same + /// shared error, so identity comparisons through [`Arc::ptr_eq`] work. + Source(SharedError), + /// The source did not settle before its deadline. Followers of one + /// execution share the leader's error instance. + FallbackTimeout(Arc), + /// A source, codec, provider or comparator panicked. + Panic(Arc), + /// Static operation or instance configuration was rejected before execution. + Config(ConfigError), + /// Invalidation was requested without a configured remote adapter. + MissingRemote, + /// An explicit maintenance mutation failed at the remote adapter. + Remote(SharedError), +} + +impl Error { + /// The source error, when the source itself failed. + pub fn source_error(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> { + match self { + Error::Source(error) => Some(error.as_ref()), + _ => None, + } + } + + /// Whether this is a source deadline error: the library's own, or one a + /// source propagated from a nested cached call, either as the bare + /// [`FallbackTimeout`] or boxed inside a nested [`Error`]. + pub fn is_fallback_timeout(&self) -> bool { + match self { + Error::FallbackTimeout(_) => true, + Error::Source(error) => { + error.downcast_ref::().is_some() + || error + .downcast_ref::() + .is_some_and(Error::is_fallback_timeout) + } + _ => false, + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Source(error) => write!(f, "DialCache source failed: {error}"), + Error::FallbackTimeout(timeout) => timeout.fmt(f), + Error::Panic(message) => write!(f, "DialCache callback panicked: {message}"), + Error::Config(error) => error.fmt(f), + Error::MissingRemote => { + f.write_str("DialCache invalidation requires a configured remote adapter") + } + Error::Remote(error) => write!(f, "DialCache remote maintenance failed: {error}"), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Error::Source(error) | Error::Remote(error) => Some(error.as_ref()), + Error::FallbackTimeout(timeout) => Some(timeout.as_ref()), + Error::Config(error) => Some(error), + _ => None, + } + } +} + +impl From for Error { + fn from(error: ConfigError) -> Self { + Error::Config(error) + } +} + +/// The source deadline elapsed before the source settled. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("DialCache fallback for use case {use_case:?} timed out after {timeout_ms} ms")] +pub struct FallbackTimeout { + /// The use case whose source timed out. + pub use_case: String, + /// The source budget that elapsed, in milliseconds. + pub timeout_ms: u64, +} + +/// The remote read deadline elapsed before the adapter answered. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("DialCache remote read for use case {use_case:?} timed out after {timeout_ms} ms")] +pub struct RemoteReadTimeout { + /// The use case whose remote read timed out. + pub use_case: String, + /// The read budget that elapsed, in milliseconds. + pub timeout_ms: u64, +} + +/// Static configuration rejected at construction or registration. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ConfigError { + /// A value is outside its domain; the message names it. + #[error("{0}")] + Invalid(String), + /// The use case name `watermark` is owned by invalidation. + #[error("DialCache use case name is reserved: {0}")] + ReservedUseCase(String), + /// A use case with this name is already registered on the instance. + #[error("DialCache use case already registered: {0}")] + UseCaseAlreadyRegistered(String), +} + +impl ConfigError { + pub(crate) fn invalid(message: impl Into) -> Self { + ConfigError::Invalid(message.into()) + } +} diff --git a/rust/src/execution.rs b/rust/src/execution.rs new file mode 100644 index 00000000..fec0c8f9 --- /dev/null +++ b/rust/src/execution.rs @@ -0,0 +1,1141 @@ +//! One enabled call: admission, policy capture, traversal, source, publication +//! and stale recovery. Ported transition by transition from the reference +//! implementations; the Quint models define the behavior. + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::BoxFuture; +use futures::FutureExt; + +use crate::cancel::CancelToken; +use crate::deadline::{await_deadline, seconds_since}; +use crate::engine::Core; +use crate::error::{BoxError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; +use crate::flight::{panic_message, start_pending, Flight, Settled, ValueResult, DROPPED_MESSAGE}; +use crate::identity::{Identity, Keys}; +use crate::limits::{MAX_DECOMPRESSED_BYTES, MAX_SAFE_INTEGER, MAX_TRACKED_VALUE_TTL_MS}; +use crate::local::StoredValue; +use crate::observe::{ + CoalescingScope, CompressionOperation, DisabledReason, ErrorKind, Event, Labels, Layer, + LogEvent, MissReason, OutcomeLabels, RecoveryOutcome, SerializationOperation, +}; +use crate::operation::ErasedOperation; +use crate::policy::{resolve_policy, PolicyDefaults, ResolvedLayer, ResolvedPolicy}; +use crate::protocol::{ + compress_payload, decompress_payload, escape_raw_payload, normalize_read_result, +}; +use crate::remote::{Frame, ReadContext, ReadRequest, ReadResult, WriteRequest}; +use crate::scope::{Owner, Scope}; +use crate::shadow::ShadowWork; + +/// An immutable acquired frame shared by readers of the same settled result. +/// The public adapter result stays owned; only internal observations share it. +#[derive(Debug, Clone)] +pub(crate) enum ReadSnapshot { + Hit(Arc), + Miss { + reason: MissReason, + observed_watermark_ms: Option, + }, +} + +impl ReadSnapshot { + pub(crate) fn miss(reason: MissReason) -> Self { + Self::Miss { + reason, + observed_watermark_ms: None, + } + } +} + +impl From for ReadSnapshot { + fn from(read: ReadResult) -> Self { + match read { + ReadResult::Hit(frame) => Self::Hit(Arc::new(frame)), + ReadResult::Miss { + reason, + observed_watermark_ms, + } => Self::Miss { + reason, + observed_watermark_ms, + }, + } + } +} + +/// The normalized, shared outcome of one raw adapter read. +pub(crate) type RawRead = Result; + +#[derive(Debug, thiserror::Error)] +#[error("DialCache callback panicked: {0}")] +pub(crate) struct PanicError(pub Arc); + +/// The registered leader of one key in the request or process flight table. +/// +/// Dropping it before [`Leader::finish`] means the leader's detached task was +/// dropped unpolled (the runtime shut down while the cache outlived it); the +/// flight is then unregistered and settled with an error so later callers, +/// possibly on another runtime, start a fresh execution instead of joining a +/// flight that can never complete. +struct Leader { + core: Arc, + owner: Option>, + key: String, + flight: Arc, + settled: bool, +} + +impl Leader { + fn unregister(&self) { + match &self.owner { + Some(owner) => { + let mut state = owner.state.lock(); + if state + .flights + .get(&self.key) + .is_some_and(|f| Arc::ptr_eq(f, &self.flight)) + { + state.flights.remove(&self.key); + } + } + None => { + let mut state = self.core.state.lock(); + if state + .flights + .get(&self.key) + .is_some_and(|f| Arc::ptr_eq(f, &self.flight)) + { + state.flights.remove(&self.key); + } + } + } + } + + /// Unregister first, then settle: a caller arriving between the two + /// starts a new flight rather than joining a settled one. + fn finish(&mut self, result: ValueResult) { + self.settled = true; + self.unregister(); + self.flight.result.settle(result); + } +} + +impl Drop for Leader { + fn drop(&mut self) { + if !self.settled { + self.unregister(); + self.flight + .result + .settle(Err(Error::Panic(Arc::from(DROPPED_MESSAGE)))); + } + } +} + +/// One admitted enabled call. +pub(crate) struct Execution { + pub(crate) core: Arc, + pub(crate) scope: Scope, + pub(crate) op: Arc, + pub(crate) identity: Identity, + pub(crate) keys: Keys, + pub(crate) policy: ResolvedPolicy, + pub(crate) labels: OutcomeLabels, +} + +/// What the serving remote read produced. +pub(crate) enum RemoteValue { + Hit { + value: StoredValue, + frame: Arc, + }, + /// A valid stale frame retained only as a recovery candidate. + Retained { + frame: Arc, + }, + Miss { + fence: Option, + }, + DecodeError, + Error, +} + +/// Run one call from admission to result. +pub(crate) async fn run(core: Arc, scope: Scope, op: Arc) -> ValueResult { + let mut identity = op.identity.clone(); + let noop_labels = crate::engine::outcome_labels(&identity); + let noop = |reason: DisabledReason| Event::Disabled { + labels: layer_labels(&noop_labels, Layer::Noop), + reason, + }; + if !core.id_enabled(&scope) { + core.emit(noop(DisabledReason::Context)); + return call_load(&op, scope).await; + } + if let Some(provider) = &op.identity_provider { + let computed = catch_unwind(AssertUnwindSafe(|| provider())) + .unwrap_or_else(|payload| Err(Box::new(PanicError(panic_message(payload))) as BoxError)) + .and_then(|identity| { + if identity.use_case == crate::limits::WATERMARK_USE_CASE { + Err("reserved use case: watermark".into()) + } else { + Ok(identity) + } + }); + match computed { + Ok(mut computed) => { + if computed.namespace.is_empty() { + computed.namespace = core.namespace.to_string(); + } + identity = computed; + } + Err(error) => { + core.log(LogEvent::KeyConstructionFailed(error)); + core.emit(Event::Error { + labels: layer_labels(&noop_labels, Layer::Noop), + error: ErrorKind::KeyConstruction, + in_fallback: false, + }); + return uncached_source(&core, &op, scope, &noop_labels).await; + } + } + } + let labels = crate::engine::outcome_labels(&identity); + let keys = match identity.keys() { + Ok(keys) => keys, + Err(error) => { + core.log(LogEvent::KeyConstructionFailed(Box::new(error))); + core.emit(Event::Error { + labels: layer_labels(&labels, Layer::Noop), + error: ErrorKind::KeyConstruction, + in_fallback: false, + }); + return uncached_source(&core, &op, scope, &labels).await; + } + }; + let overlay: Result, BoxError> = match &core.provider { + Some(provider) => AssertUnwindSafe(async { provider(identity.clone()).await }) + .catch_unwind() + .await + .unwrap_or_else( + |payload| Err(Box::new(PanicError(panic_message(payload))) as BoxError), + ), + None => Ok(None), + }; + let resolved = overlay.and_then(|overlay| { + resolve_policy( + &op.metadata.policy, + overlay.as_ref(), + &keys.logical, + PolicyDefaults { + remote_read_timeout_ms: core.remote_read_timeout_ms, + }, + ) + .map_err(|e| Box::new(e) as BoxError) + }); + let policy = match resolved { + Ok(policy) => policy, + Err(error) => { + core.log(LogEvent::PolicyResolutionFailed(error)); + core.emit(Event::Error { + labels: layer_labels(&labels, Layer::Noop), + error: ErrorKind::ConfigResolution, + in_fallback: false, + }); + core.emit(Event::Disabled { + labels: layer_labels(&labels, Layer::Noop), + reason: DisabledReason::ConfigError, + }); + return uncached_source(&core, &op, scope, &labels).await; + } + }; + if !core.id_enabled(&scope) { + core.emit(Event::Disabled { + labels: layer_labels(&labels, Layer::Noop), + reason: DisabledReason::Context, + }); + return uncached_source(&core, &op, scope, &labels).await; + } + let execution = Arc::new(Execution { + core: core.clone(), + scope: scope.clone(), + op, + identity, + keys, + policy, + labels, + }); + if !execution.policy.request_local { + return execution.shared(Layer::Local).await; + } + let Some(owner) = scope.owner.clone() else { + return execution.shared(Layer::Local).await; + }; + let request_run = { + let x = execution.clone(); + let owner = owner.clone(); + move || -> BoxFuture<'static, ValueResult> { + Box::pin(async move { x.through_request(owner).await }) + } + }; + if !execution.policy.coalesce { + return request_run().await; + } + execution + .single_flight(Some(owner), CoalescingScope::RequestLocal, request_run) + .await +} + +#[derive(Debug)] +struct SharedErrorWrapper(SharedError); + +impl std::fmt::Display for SharedErrorWrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl std::error::Error for SharedErrorWrapper {} + +impl Core { + pub(crate) fn id_enabled(&self, scope: &Scope) -> bool { + scope.cache_id == self.id && scope.is_enabled() + } +} + +pub(crate) fn layer_labels(labels: &OutcomeLabels, layer: Layer) -> Labels { + Labels { + namespace: labels.namespace.clone(), + use_case: labels.use_case.clone(), + key_type: labels.key_type.clone(), + layer, + } +} + +/// Invoke the source directly with panic isolation; errors keep their identity. +pub(crate) async fn call_load(op: &ErasedOperation, scope: Scope) -> ValueResult { + // Constructing a callback's future can panic before its first poll. + match AssertUnwindSafe(async { (op.load)(scope).await }) + .catch_unwind() + .await + { + Ok(Ok(value)) => Ok(value), + Ok(Err(error)) => Err(Error::Source(Arc::from(error))), + Err(payload) => Err(Error::Panic(panic_message(payload))), + } +} + +/// The enabled but uncached path: the source runs with its enabled deadline. +async fn uncached_source( + core: &Arc, + op: &Arc, + scope: Scope, + labels: &OutcomeLabels, +) -> ValueResult { + source_with_budget(core, op, scope, labels, Layer::Noop).await +} + +/// Run the source under the operation budget, reporting the fallback trail. +async fn source_with_budget( + core: &Arc, + op: &Arc, + scope: Scope, + labels: &OutcomeLabels, + layer: Layer, +) -> ValueResult { + let clock = core.clock.clone(); + let start = clock.elapsed(); + let budget = op.metadata.budget.millis(); + let pending: Settled = start_pending( + core.runtime.as_ref(), + { + let op = op.clone(); + async move { call_load(&op, scope).await } + }, + |message| Err(Error::Panic(message)), + ); + let use_case = labels.use_case.to_string(); + let result = await_deadline( + clock.as_ref(), + core.runtime.as_ref(), + &pending, + start, + budget, + || { + Err(Error::FallbackTimeout(Arc::new(FallbackTimeout { + use_case, + timeout_ms: budget.unwrap_or(0), + }))) + }, + ) + .await; + if result.is_err() { + core.emit(Event::Error { + labels: layer_labels(labels, layer), + error: ErrorKind::Fallback, + in_fallback: true, + }); + } + core.emit(Event::Fallback { + labels: layer_labels(labels, layer), + seconds: seconds_since(clock.as_ref(), start), + }); + result +} + +impl Execution { + pub(crate) fn labels(&self, layer: Layer) -> Labels { + layer_labels(&self.labels, layer) + } + + pub(crate) fn emit(&self, event: Event) { + self.core.emit(event); + } + + pub(crate) fn error_event(&self, layer: Layer, error: ErrorKind, in_fallback: bool) { + self.emit(Event::Error { + labels: self.labels(layer), + error, + in_fallback, + }); + } + + pub(crate) fn elapsed(&self) -> Duration { + self.core.clock.elapsed() + } + + pub(crate) fn seconds_since(&self, start: Duration) -> f64 { + seconds_since(self.core.clock.as_ref(), start) + } + + pub(crate) fn wall_ms(&self) -> i64 { + self.core.clock.wall_ms() + } + + /// The source under the operation budget, attributed to `layer`. + pub(crate) async fn source(&self, layer: Layer) -> ValueResult { + source_with_budget( + &self.core, + &self.op, + self.scope.clone(), + &self.labels, + layer, + ) + .await + } + + async fn through_request(self: Arc, owner: Arc) -> ValueResult { + let start = self.elapsed(); + let (memo, live) = { + let state = owner.state.lock(); + (state.memo.get(&self.keys.logical).cloned(), state.live) + }; + self.emit(Event::Request { + labels: self.labels(Layer::RequestLocal), + }); + self.emit(Event::Get { + labels: self.labels(Layer::RequestLocal), + seconds: self.seconds_since(start), + }); + let reason = match (live, memo) { + (true, Some(value)) if value.as_ref().type_id() == self.op.metadata.value_type => { + return Ok(value); + } + (true, Some(_)) => MissReason::Unclassified, + _ => MissReason::ValueAbsent, + }; + self.emit(Event::Miss { + labels: self.labels(Layer::RequestLocal), + reason, + }); + let result = self.clone().shared(Layer::RequestLocal).await; + if let Ok(value) = &result { + let displaced = { + let mut state = owner.state.lock(); + if state.live && value.as_ref().type_id() == self.op.metadata.value_type { + state.memo.insert(self.keys.logical.clone(), value.clone()) + } else { + None + } + }; + // A replaced memo value drops here, outside the lock. + drop(displaced); + } + result + } + + /// Join an existing execution for this key or lead a new one. + pub(crate) async fn single_flight( + self: &Arc, + owner: Option>, + label: CoalescingScope, + run: F, + ) -> ValueResult + where + F: FnOnce() -> BoxFuture<'static, ValueResult>, + { + // Read the clock before taking any lock: it is external code. + let started = self.elapsed(); + let key = self.keys.logical.clone(); + let flight = { + let joined = match &owner { + Some(owner) => { + let mut state = owner.state.lock(); + if !state.live { + None + } else if let Some(existing) = state.flights.get(&key) { + existing.join(); + Some(Ok(existing.clone())) + } else { + let flight = Flight::new(started); + state.flights.insert(key.clone(), flight.clone()); + Some(Err(flight)) + } + } + None => { + let mut state = self.core.state.lock(); + if let Some(existing) = state.flights.get(&key) { + existing.join(); + Some(Ok(existing.clone())) + } else { + let flight = Flight::new(started); + state.flights.insert(key.clone(), flight.clone()); + Some(Err(flight)) + } + } + }; + match joined { + None => return run().await, + Some(Ok(existing)) => { + self.emit(Event::Coalesced { + labels: self.labels.clone(), + scope: label, + }); + return existing.result.wait().await; + } + Some(Err(flight)) => flight, + } + }; + let mut leader = Leader { + core: self.core.clone(), + owner, + key, + flight, + settled: false, + }; + let result = match AssertUnwindSafe(run()).catch_unwind().await { + Ok(result) => result, + Err(payload) => Err(Error::Panic(panic_message(payload))), + }; + leader.finish(result.clone()); + result + } + + fn layer_event(&self, layer: Layer, resolved: &ResolvedLayer) { + if !resolved.enabled { + let reason = resolved.reason.unwrap_or(DisabledReason::PolicyDisabled); + self.emit(Event::Disabled { + labels: self.labels(layer), + reason, + }); + if matches!( + reason, + DisabledReason::InvalidTtl | DisabledReason::InvalidRamp + ) { + self.error_event(layer, ErrorKind::ConfigResolution, false); + } + } + } + + /// Traverse the shared layers below the request layer. + pub(crate) async fn shared(self: Arc, fallback_layer: Layer) -> ValueResult { + let p = self.policy.clone(); + self.layer_event(Layer::Local, &p.local); + let run = { + let x = self.clone(); + move || -> BoxFuture<'static, ValueResult> { + Box::pin(async move { x.shared_run(fallback_layer).await }) + } + }; + if p.local.enabled { + if p.coalesce { + return self + .single_flight(None, CoalescingScope::Process, run) + .await; + } + return run().await; + } + // Remote admission is decided before joining; traversal belongs to the leader. + if p.remote.enabled && self.core.remote.is_some() && p.coalesce { + return self + .single_flight(None, CoalescingScope::Process, run) + .await; + } + run().await + } + + async fn shared_run(self: Arc, mut fallback_layer: Layer) -> ValueResult { + let p = self.policy.clone(); + let mut local_miss = false; + if p.local.enabled { + let start = self.elapsed(); + match self.core.local_get(&self.keys.logical) { + Err(error) => { + self.core.log(LogEvent::LocalReadFailed(error)); + self.error_event(Layer::Local, ErrorKind::CacheRead, false); + self.emit(Event::Disabled { + labels: self.labels(Layer::Local), + reason: DisabledReason::ConfigError, + }); + } + Ok(found) => { + self.emit(Event::Request { + labels: self.labels(Layer::Local), + }); + self.emit(Event::Get { + labels: self.labels(Layer::Local), + seconds: self.seconds_since(start), + }); + let reason = match found { + Some(value) if value.as_ref().type_id() == self.op.metadata.value_type => { + return Ok(value); + } + Some(_) => MissReason::Unclassified, + None => MissReason::ValueAbsent, + }; + local_miss = true; + self.emit(Event::Miss { + labels: self.labels(Layer::Local), + reason, + }); + } + } + fallback_layer = Layer::Local; + } + if self.core.remote.is_none() { + let result = self.source(fallback_layer).await; + if let Ok(value) = &result { + if local_miss { + self.put_local(value.clone()); + } + } + return result; + } + if p.stale_on_error_config_error { + self.error_event(Layer::Remote, ErrorKind::ConfigResolution, false); + } + self.layer_event(Layer::Remote, &p.remote); + if !p.remote.enabled { + if p.remote.reason == Some(DisabledReason::RampedDown) { + return self.dark_source(fallback_layer, local_miss).await; + } + let result = self.source(fallback_layer).await; + if let Ok(value) = &result { + if local_miss { + self.put_local(value.clone()); + } + } + return result; + } + let remote = self.read_serving().await; + if let RemoteValue::Hit { value, frame } = remote { + if local_miss { + self.put_local(value.clone()); + } + self.schedule_shadow(Some(frame), None, Duration::ZERO); + return Ok(value); + } + let result = self.source(Layer::Remote).await; + match result { + Err(error) => { + let recoverable = !matches!(remote, RemoteValue::Error | RemoteValue::DecodeError); + if recoverable && p.stale_on_error_max_age_ms > 0 && self.can_recover(&error) { + let frame = match &remote { + RemoteValue::Retained { frame } => Some(frame.as_ref()), + _ => None, + }; + if let Some(value) = self.recover(frame).await { + return Ok(value); + } + } + Err(error) + } + Ok(value) => { + if !matches!(remote, RemoteValue::Error) { + let fence = match &remote { + RemoteValue::Miss { fence } => *fence, + _ => None, + }; + if let Err(error) = self.put_remote(&value, fence, Layer::Remote, None).await { + self.core.log(LogEvent::RemoteWriteFailed(error)); + } + } + if local_miss && !self.identity.tracked { + self.put_local(value.clone()); + } + Ok(value) + } + } + } + + pub(crate) fn put_local(&self, value: StoredValue) { + if let Err(error) = self + .core + .local_put(&self.keys.logical, value, self.policy.local.ttl_ms) + { + self.core.log(LogEvent::LocalWriteFailed(error)); + self.error_event(Layer::Local, ErrorKind::CacheWrite, false); + } + } + + /// Start one adapter read under the read budget. Returns the bounded + /// result and the raw work, which keeps running after a deadline. + pub(crate) fn raw_read(&self) -> (BoxFuture<'static, RawRead>, Settled) { + let remote = self + .core + .remote + .clone() + .expect("remote layer requires an adapter"); + let cancel = CancelToken::new(); + let timeout_ms = self.policy.remote_read_timeout_ms; + let context = ReadContext { + timeout_ms, + cancel: cancel.clone(), + }; + let request = ReadRequest { + value_key: self.keys.value.clone(), + watermark_key: if self.identity.tracked { + self.keys.watermark.clone() + } else { + None + }, + }; + let clock = self.core.clock.clone(); + let runtime = self.core.runtime.clone(); + let start = clock.elapsed(); + let tracked = self.identity.tracked; + let raw: Settled = start_pending( + self.core.runtime.as_ref(), + async move { + remote + .read(request, context) + .await + .map(|read| ReadSnapshot::from(normalize_read_result(read, tracked))) + .map_err(|e| Arc::from(e) as SharedError) + }, + |message| Err(Arc::new(PanicError(message)) as SharedError), + ); + let use_case = self.labels.use_case.to_string(); + let waited = raw.clone(); + let bounded = Box::pin(async move { + await_deadline( + clock.as_ref(), + runtime.as_ref(), + &waited, + start, + Some(timeout_ms), + || { + cancel.cancel(); + Err(Arc::new(RemoteReadTimeout { + use_case, + timeout_ms, + }) as SharedError) + }, + ) + .await + }); + (bounded, raw) + } + + /// Age of a frame against the wall clock; `false` for invalid or future stamps. + pub(crate) fn frame_age(&self, frame: &Frame, layer: Layer) -> (i64, bool) { + if frame.created_at_ms > MAX_SAFE_INTEGER { + return (0, false); + } + let age = self.wall_ms().saturating_sub(frame.created_at_ms as i64); + if age < 0 { + self.emit(Event::FutureTimestampOffset { + labels: self.labels(layer), + seconds: (-age) as f64 / 1000.0, + }); + return (age, false); + } + (age, true) + } + + async fn read_serving(&self) -> RemoteValue { + let start = self.elapsed(); + self.emit(Event::Request { + labels: self.labels(Layer::Remote), + }); + let (bounded, _raw) = self.raw_read(); + let result = bounded.await; + let value = match result { + Err(error) => { + let kind = if error.downcast_ref::().is_some() { + ErrorKind::CacheReadTimeout + } else { + ErrorKind::CacheRead + }; + self.core + .log(LogEvent::RemoteReadFailed(Box::new(SharedErrorWrapper( + error, + )))); + self.error_event(Layer::Remote, kind, false); + RemoteValue::Error + } + Ok(ReadSnapshot::Miss { + reason, + observed_watermark_ms, + }) => { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason, + }); + RemoteValue::Miss { + fence: observed_watermark_ms, + } + } + Ok(ReadSnapshot::Hit(frame)) => { + let (age, valid) = self.frame_age(&frame, Layer::Remote); + if !valid { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason: MissReason::Unclassified, + }); + RemoteValue::Miss { fence: None } + } else { + let max_age = if self.policy.stale_on_error_max_age_ms > 0 { + self.policy.stale_on_error_max_age_ms + } else { + self.policy.remote.ttl_ms + }; + if age as u64 >= max_age { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason: MissReason::Expired, + }); + RemoteValue::Miss { fence: None } + } else if age as u64 >= self.policy.remote.ttl_ms { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason: MissReason::Expired, + }); + RemoteValue::Retained { frame } + } else { + match self.decode(&frame, Layer::Remote, None).await { + Ok(value) => RemoteValue::Hit { value, frame }, + Err(_) => { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason: MissReason::Unclassified, + }); + RemoteValue::DecodeError + } + } + } + } + } + }; + self.emit(Event::Get { + labels: self.labels(Layer::Remote), + seconds: self.seconds_since(start), + }); + value + } + + /// Decode a frame's payload: envelope, then codec, with diagnostics. + pub(crate) async fn decode( + &self, + frame: &Frame, + layer: Layer, + shadow: Option<&ShadowWork>, + ) -> Result { + let decompress_started = self.elapsed(); + let payload = frame.payload.clone(); + let expanded = if payload.binary && matches!(payload.bytes.first(), Some(1 | 2)) { + let shadow = shadow.cloned(); + match crate::blocking::run(self.core.runtime.as_ref(), move || { + if shadow.as_ref().is_some_and(ShadowWork::expired) { + return Err("shadow deadline elapsed before decompression".into()); + } + Ok(decompress_payload(payload, MAX_DECOMPRESSED_BYTES)) + }) + .await + { + Ok(expanded) => expanded, + Err(error) => { + self.error_event(layer, ErrorKind::Compression, false); + return Err(error); + } + } + } else { + decompress_payload(payload, MAX_DECOMPRESSED_BYTES) + }; + if let Some(outcome) = expanded.outcome { + self.emit(Event::Compression { + labels: self.labels(layer), + outcome, + }); + self.emit(Event::CompressionDuration { + labels: self.labels(layer), + operation: CompressionOperation::Decompress, + seconds: self.seconds_since(decompress_started), + }); + } + if shadow.is_some_and(ShadowWork::expired) { + return Err("shadow deadline elapsed after decompression".into()); + } + let start = self.elapsed(); + let codec = self.op.metadata.codec.clone(); + let decoded = match AssertUnwindSafe(async { codec.decode(expanded.payload).await }) + .catch_unwind() + .await + { + Ok(result) => result, + Err(payload) => Err(Box::new(PanicError(panic_message(payload))) as BoxError), + }; + if decoded.is_err() { + self.error_event(layer, ErrorKind::SerializationLoad, false); + } + self.emit(Event::Serialization { + labels: self.labels(layer), + operation: SerializationOperation::Load, + seconds: self.seconds_since(start), + }); + decoded + } + + fn write_timestamp(&self, layer: Layer) -> Result { + let stamp = self.wall_ms(); + if stamp < 0 || stamp as u64 > MAX_SAFE_INTEGER { + self.error_event(layer, ErrorKind::CacheWrite, false); + return Err("DialCache write timestamp is outside the safe integer domain".into()); + } + Ok(stamp as u64) + } + + /// Prepare and dispatch one remote write. `Ok(false)` means the write was + /// fenced or its shadow deadline expired; `Ok(true)` means it was dispatched. + pub(crate) async fn put_remote( + &self, + value: &StoredValue, + fence: Option, + layer: Layer, + shadow: Option<&ShadowWork>, + ) -> Result { + let fence = if self.identity.tracked { fence } else { None }; + if let Some(fence) = fence { + let stamp = self.write_timestamp(layer)?; + if stamp <= fence { + return Ok(false); + } + } + let start = self.elapsed(); + let codec = self.op.metadata.codec.clone(); + let encoded = match AssertUnwindSafe(async { codec.encode(value.clone()).await }) + .catch_unwind() + .await + { + Ok(result) => result, + Err(payload) => Err(Box::new(PanicError(panic_message(payload))) as BoxError), + }; + if encoded.is_err() { + self.error_event(layer, ErrorKind::SerializationDump, false); + } + self.emit(Event::Serialization { + labels: self.labels(layer), + operation: SerializationOperation::Dump, + seconds: self.seconds_since(start), + }); + let mut payload = encoded?; + if shadow.is_some_and(ShadowWork::expired) { + return Ok(false); + } + self.emit(Event::Size { + labels: self.labels(layer), + bytes: payload.len() as u64, + }); + match &self.core.compression { + Some(config) => { + let compress_started = self.elapsed(); + let config = *config; + let offload = payload.len() >= 64 * 1024 + || (config.level >= 10 && payload.len() >= config.threshold_bytes); + let result = if offload { + let shadow = shadow.cloned(); + crate::blocking::run(self.core.runtime.as_ref(), move || { + if shadow.as_ref().is_some_and(ShadowWork::expired) { + return Err("shadow deadline elapsed before compression".into()); + } + compress_payload(payload, &config, MAX_DECOMPRESSED_BYTES) + .map_err(|e| Box::new(e) as BoxError) + }) + .await + } else { + compress_payload(payload, &config, MAX_DECOMPRESSED_BYTES) + .map_err(|e| Box::new(e) as BoxError) + }; + let compressed = match result { + Ok(compressed) => compressed, + Err(error) => { + self.error_event(layer, ErrorKind::Compression, false); + return Err(error); + } + }; + payload = compressed.payload; + self.emit(Event::Compression { + labels: self.labels(layer), + outcome: compressed.outcome, + }); + if matches!( + compressed.outcome, + crate::observe::CompressionOutcome::Compressed + | crate::observe::CompressionOutcome::NotSmaller + ) { + self.emit(Event::CompressionDuration { + labels: self.labels(layer), + operation: CompressionOperation::Compress, + seconds: self.seconds_since(compress_started), + }); + } + if compressed.outcome == crate::observe::CompressionOutcome::Compressed { + self.emit(Event::CompressionRatio { + labels: self.labels(layer), + ratio: compressed.stored_bytes as f64 / compressed.original_bytes as f64, + }); + } + } + None => payload = escape_raw_payload(payload), + } + self.emit(Event::StoredSize { + labels: self.labels(layer), + bytes: payload.len() as u64, + }); + if shadow.is_some_and(ShadowWork::expired) { + return Ok(false); + } + let stamp = self.write_timestamp(layer)?; + if let Some(fence) = fence { + if stamp <= fence { + return Ok(false); + } + } + let mut ttl = if self.policy.stale_on_error_max_age_ms > 0 { + self.policy.stale_on_error_max_age_ms + } else { + self.policy.remote.ttl_ms + }; + if self.identity.tracked && ttl > MAX_TRACKED_VALUE_TTL_MS { + ttl = MAX_TRACKED_VALUE_TTL_MS; + self.error_event(layer, ErrorKind::TrackedTtlClamped, false); + } + let remote = self + .core + .remote + .clone() + .expect("remote layer requires an adapter"); + let request = WriteRequest { + value_key: self.keys.value.clone(), + frame: Frame { + created_at_ms: stamp, + payload, + }, + ttl_ms: ttl, + }; + let shadow = shadow.cloned(); + let pending: Settled> = start_pending( + self.core.runtime.as_ref(), + async move { + // Task admission can precede its first poll. Keep the raw task's + // shadow ownership and check again at the adapter boundary. + if shadow.as_ref().is_some_and(ShadowWork::expired) { + return Ok(false); + } + remote + .write(request) + .await + .map(|()| true) + .map_err(|e| Arc::from(e) as SharedError) + }, + |message| Err(Arc::new(PanicError(message)) as SharedError), + ); + match pending.wait().await { + Ok(dispatched) => Ok(dispatched), + Err(error) => { + self.error_event(layer, ErrorKind::CacheWrite, false); + Err(Box::new(SharedErrorWrapper(error))) + } + } + } + + pub(crate) fn can_recover(&self, error: &Error) -> bool { + let predicate = self + .op + .metadata + .should_recover + .clone() + .or_else(|| self.core.should_recover.clone()); + match predicate { + None => error.is_fallback_timeout(), + Some(predicate) => match catch_unwind(AssertUnwindSafe(|| predicate(error))) { + Ok(allowed) => allowed, + Err(payload) => { + self.core + .log(LogEvent::RecoveryPredicateFailed(Box::new(PanicError( + panic_message(payload), + )))); + false + } + }, + } + } + + fn recovery_event(&self, outcome: RecoveryOutcome, age_ms: Option) { + self.emit(Event::StaleRecovery { + labels: self.labels.clone(), + outcome, + }); + if let Some(age) = age_ms { + self.emit(Event::StaleRecoveryValueAge { + labels: self.labels.clone(), + outcome, + seconds: age.max(0) as f64 / 1000.0, + }); + } + } + + /// Serve the retained stale candidate after an authorized source failure. + async fn recover(&self, frame: Option<&Frame>) -> Option { + let Some(frame) = frame else { + self.recovery_event(RecoveryOutcome::Miss, None); + return None; + }; + let max_age = self.policy.stale_on_error_max_age_ms; + let (age, valid) = self.frame_age(frame, Layer::Remote); + if !valid || age as u64 >= max_age { + self.recovery_event(RecoveryOutcome::Miss, None); + return None; + } + let value = match self.decode(frame, Layer::Remote, None).await { + Ok(value) => value, + Err(error) => { + self.core.log(LogEvent::RecoveryDecodeFailed(error)); + self.recovery_event(RecoveryOutcome::DeserializationError, None); + return None; + } + }; + let (age, valid) = self.frame_age(frame, Layer::Remote); + if !valid || age as u64 >= max_age { + self.recovery_event(RecoveryOutcome::Miss, None); + return None; + } + self.recovery_event(RecoveryOutcome::Served, Some(age)); + Some(value) + } +} + +#[cfg(test)] +#[path = "read_snapshot_tests.rs"] +mod read_snapshot_tests; diff --git a/rust/src/flight.rs b/rust/src/flight.rs new file mode 100644 index 00000000..60e723a8 --- /dev/null +++ b/rust/src/flight.rs @@ -0,0 +1,279 @@ +//! Settle-once cells shared by coalesced callers and detached raw work. + +use std::future::Future; +use std::panic::AssertUnwindSafe; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll, Waker}; +use std::time::Duration; + +use futures::FutureExt; +use parking_lot::Mutex; +use slab::Slab; + +use crate::error::Error; +use crate::local::StoredValue; +use crate::runtime::Runtime; + +struct SettledState { + value: Option, + wakers: Slab, +} + +/// A value that is set at most once and observed by any number of waiters. +pub(crate) struct Settled { + inner: Arc>>, +} + +impl Clone for Settled { + fn clone(&self) -> Self { + Settled { + inner: self.inner.clone(), + } + } +} + +impl Settled { + pub(crate) fn new() -> Self { + Settled { + inner: Arc::new(Mutex::new(SettledState { + value: None, + wakers: Slab::new(), + })), + } + } + + /// Store the value and wake every waiter. Returns false if already settled. + pub(crate) fn settle(&self, value: T) -> bool { + let wakers = { + let mut state = self.inner.lock(); + if state.value.is_some() { + return false; + } + state.value = Some(value); + std::mem::take(&mut state.wakers) + }; + for (_, waker) in wakers { + waker.wake(); + } + true + } + + pub(crate) fn peek(&self) -> Option { + self.inner.lock().value.clone() + } + + /// A future that completes with a clone of the settled value. + pub(crate) fn wait(&self) -> Wait { + Wait { + settled: self.clone(), + registration: None, + } + } +} + +pub(crate) struct Wait { + settled: Settled, + registration: Option, +} + +impl Future for Wait { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + let mut state = this.settled.inner.lock(); + if let Some(value) = &state.value { + return Poll::Ready(value.clone()); + } + let replaced = match this.registration { + Some(slot) => Some(std::mem::replace( + &mut state.wakers[slot], + cx.waker().clone(), + )), + None => { + this.registration = Some(state.wakers.insert(cx.waker().clone())); + None + } + }; + drop(state); + drop(replaced); + Poll::Pending + } +} + +impl Drop for Wait { + fn drop(&mut self) { + let removed = { + let mut state = self.settled.inner.lock(); + if state.value.is_none() { + self.registration + .take() + .map(|slot| state.wakers.remove(slot)) + } else { + None + } + }; + drop(removed); + } +} + +impl Unpin for Wait {} + +/// Describe a panic payload. +pub(crate) fn panic_message(payload: Box) -> Arc { + if let Some(text) = payload.downcast_ref::<&str>() { + Arc::from(*text) + } else if let Some(text) = payload.downcast_ref::() { + Arc::from(text.as_str()) + } else { + Arc::from("non-string panic payload") + } +} + +/// The message settled into a cell whose detached task was dropped before it +/// completed: the runtime shut down while the cache outlived it. +pub(crate) const DROPPED_MESSAGE: &str = + "DialCache detached work was dropped before it settled (runtime shut down)"; + +/// Settles the cell from `Drop` when the spawned task is dropped unpolled or +/// mid-await, so waiters on another runtime never hang on a dead task. +struct SettleOnDrop) -> T> { + cell: Settled, + on_panic: Option

, +} + +impl) -> T> Drop for SettleOnDrop { + fn drop(&mut self) { + if let Some(on_panic) = self.on_panic.take() { + self.cell.settle(on_panic(Arc::from(DROPPED_MESSAGE))); + } + } +} + +/// Start `work` as detached raw work. The returned cell settles with its +/// result, with `on_panic` when it panics, and with `on_panic` and +/// [`DROPPED_MESSAGE`] when the runtime drops the task before it completes. +/// Callers that stop waiting keep no ownership of the work. +pub(crate) fn start_pending( + runtime: &dyn Runtime, + work: F, + on_panic: impl FnOnce(Arc) -> T + Send + 'static, +) -> Settled +where + T: Clone + Send + 'static, + F: Future + Send + 'static, +{ + let settled = Settled::new(); + let mut guard = SettleOnDrop { + cell: settled.clone(), + on_panic: Some(on_panic), + }; + runtime.spawn(Box::pin(async move { + let outcome = match AssertUnwindSafe(work).catch_unwind().await { + Ok(value) => value, + Err(payload) => (guard.on_panic.take().expect("armed guard"))(panic_message(payload)), + }; + // Disarm before settling so the cell settles exactly once. + guard.on_panic = None; + guard.cell.settle(outcome); + })); + settled +} + +/// Complete once work that is already runnable has progressed (the executor's +/// deferred queue drained under a controlled scheduler). +pub(crate) async fn yield_deferred(runtime: &dyn Runtime) { + let done: Settled<()> = Settled::new(); + let cell = done.clone(); + runtime.defer(Box::pin(async move { + cell.settle(()); + })); + done.wait().await; +} + +/// Result of one execution shared by every coalesced caller. +pub(crate) type ValueResult = Result; + +/// One registered execution and its followers. +pub(crate) struct Flight { + pub(crate) result: Settled, + pub(crate) started: Duration, + pub(crate) followers: AtomicUsize, +} + +impl Flight { + pub(crate) fn new(started: Duration) -> Arc { + Arc::new(Flight { + result: Settled::new(), + started, + followers: AtomicUsize::new(0), + }) + } + + pub(crate) fn join(&self) { + self.followers.fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn followers(&self) -> usize { + self.followers.load(Ordering::Relaxed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicUsize; + use std::task::Wake; + + #[derive(Default)] + struct Counter(AtomicUsize); + impl Wake for Counter { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + #[test] + fn many_waiters_replace_and_remove_their_own_registration() { + let cell = Settled::new(); + let mut waiters: Vec<_> = (0..10_000).map(|_| cell.wait()).collect(); + let counters: Vec<_> = (0..10_000).map(|_| Arc::new(Counter::default())).collect(); + for (wait, counter) in waiters.iter_mut().zip(&counters) { + let waker = Waker::from(counter.clone()); + for _ in 0..3 { + assert!(Pin::new(&mut *wait) + .poll(&mut Context::from_waker(&waker)) + .is_pending()); + } + } + let replacement = Arc::new(Counter::default()); + let waker = Waker::from(replacement.clone()); + assert!(Pin::new(&mut waiters[0]) + .poll(&mut Context::from_waker(&waker)) + .is_pending()); + // Cancellation frees slots; the surviving waiter must not remove a reused slot. + waiters.truncate(5_000); + let mut later = cell.wait(); + assert!(Pin::new(&mut later) + .poll(&mut Context::from_waker(&waker)) + .is_pending()); + assert!(cell.settle(7)); + assert!(!cell.settle(8)); + assert_eq!(replacement.0.load(Ordering::SeqCst), 2); + for (i, counter) in counters.iter().enumerate() { + assert_eq!( + counter.0.load(Ordering::SeqCst), + usize::from(i > 0 && i < 5_000) + ); + } + for mut wait in waiters { + assert_eq!( + Pin::new(&mut wait).poll(&mut Context::from_waker(&waker)), + Poll::Ready(7) + ); + } + drop(later); // Settlement removed registrations; stale indices must be ignored. + } +} diff --git a/rust/src/identity.rs b/rust/src/identity.rs new file mode 100644 index 00000000..1a1c88ce --- /dev/null +++ b/rust/src/identity.rs @@ -0,0 +1,481 @@ +//! Logical identities, shared cache keys and rollout cohorts. +//! +//! Key construction follows the portable protocol (W01–W03): URI component +//! escaping, tracked entity hash tags, ordered argument pairs, the frame key +//! suffix, and FNV-1a cohorts over UTF-16 code units. + +use std::borrow::Cow; +use std::cmp::Ordering; +use std::fmt::Write as _; + +use crate::limits::FRAME_KEY_SUFFIX; + +/// Converts an entity identifier to the text used in a cache key. +/// +/// Strings are preserved, integers use exact decimal notation, and floats use +/// JavaScript `String(number)` spelling (`f32` is promoted to `f64`). Shared +/// references to these types are also accepted. For other displayable IDs, +/// pass `id.to_string()` or implement this trait with the desired spelling. +pub trait IntoKeyId { + /// Consume the identifier and return its canonical text. + fn into_key_id(self) -> String; +} + +impl IntoKeyId for String { + fn into_key_id(self) -> String { + self + } +} + +impl IntoKeyId for &str { + fn into_key_id(self) -> String { + self.to_owned() + } +} + +impl IntoKeyId for Box { + fn into_key_id(self) -> String { + self.into_string() + } +} + +impl IntoKeyId for Cow<'_, str> { + fn into_key_id(self) -> String { + self.into_owned() + } +} + +impl IntoKeyId for char { + fn into_key_id(self) -> String { + self.to_string() + } +} + +impl IntoKeyId for &T { + fn into_key_id(self) -> String { + self.clone().into_key_id() + } +} + +macro_rules! integer_key_id { + ($($t:ty),*) => { $(impl IntoKeyId for $t { + fn into_key_id(self) -> String { self.to_string() } + })* }; +} +integer_key_id!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); + +impl IntoKeyId for f64 { + fn into_key_id(self) -> String { + js_number_to_string(self) + } +} + +impl IntoKeyId for f32 { + fn into_key_id(self) -> String { + js_number_to_string(f64::from(self)) + } +} + +/// A normalized logical identity. Ordered arguments retain caller order; +/// use [`normalize_args`] to build them from a host-language record. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct Identity { + /// The instance namespace; empty means the cache fills in its own at execution. + #[serde(default)] + pub namespace: String, + /// The entity type; serialized as `keyType`. + #[serde(rename = "keyType")] + pub key_type: String, + /// The entity identifier. + pub id: String, + /// The operation name; serialized as `useCase`. + #[serde(rename = "useCase")] + pub use_case: String, + /// Whether the identity is fenced by its entity's invalidation watermark; + /// serialized as `trackForInvalidation`. Defaults to false. + #[serde(rename = "trackForInvalidation", default)] + pub tracked: bool, + /// Secondary dimensions as `(name, spelled value)` pairs, already + /// normalized and in key order. + #[serde(default)] + pub args: Vec<(String, String)>, +} + +impl Identity { + /// An untracked identity with an empty namespace and no arguments. + /// IDs use the same [`IntoKeyId`] conversion as [`crate::KeySpec::new`] + /// and [`crate::DialCache::invalidate`]. + pub fn new( + key_type: impl Into, + id: impl IntoKeyId, + use_case: impl Into, + ) -> Self { + Identity { + namespace: String::new(), + key_type: key_type.into(), + id: id.into_key_id(), + use_case: use_case.into(), + tracked: false, + args: Vec::new(), + } + } + + /// Set whether the identity shares its entity's invalidation watermark. + pub fn tracked(mut self, tracked: bool) -> Self { + self.tracked = tracked; + self + } + + /// Set the namespace explicitly instead of inheriting the instance's. + pub fn namespace(mut self, namespace: impl Into) -> Self { + self.namespace = namespace.into(); + self + } + + /// Replace the ordered argument pairs; build them with [`normalize_args`]. + pub fn args(mut self, args: Vec<(String, String)>) -> Self { + self.args = args; + self + } + + /// The logical key, the stored value key and, for tracked identities, the watermark key. + pub fn keys(&self) -> Result { + keys(self) + } +} + +/// Derived key strings of one identity. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Keys { + /// Logical key: `namespace:keyType:id[?args]#useCase`, hash-tagged when tracked. + pub logical: String, + /// Stored value key: the logical key plus the frame suffix. + pub value: String, + /// Entity watermark key for tracked identities. + pub watermark: Option, +} + +/// Why an identity cannot form a key. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum IdentityError { + /// A component contains `{` or `}` where it would form the Redis Cluster + /// hash tag: the namespace always, the key type and ID when tracked. + #[error("DialCache identity component contains a reserved hash-tag delimiter")] + ReservedDelimiter, + /// An argument value cannot be spelled; the text names the offending value. + #[error("DialCache identity contains an unsupported argument value: {0}")] + UnsupportedArgument(String), +} + +/// A host scalar accepted as a key argument. +#[derive(Debug, Clone, PartialEq)] +pub enum ArgValue { + /// Omitted from the key (JavaScript `undefined`). + Absent, + /// JSON null, spelled `null`. + Null, + /// Spelled `true` or `false`. + Bool(bool), + /// Spelled with JavaScript `Number` formatting. + Number(f64), + /// Spelled as a decimal integer. + Int(i64), + /// An arbitrary-precision integer as decimal text. + BigInt(String), + /// Used verbatim, then percent-escaped like every other component. + Str(String), +} + +impl ArgValue { + /// The JavaScript `String(value)` spelling, or `None` for an absent value. + fn spell(self) -> Result, IdentityError> { + Ok(Some(match self { + ArgValue::Absent => return Ok(None), + ArgValue::Null => "null".to_owned(), + ArgValue::Bool(value) => if value { "true" } else { "false" }.to_owned(), + ArgValue::Number(value) => js_number_to_string(value), + ArgValue::Int(value) => value.to_string(), + ArgValue::BigInt(text) => { + if !is_decimal_integer(&text) { + return Err(IdentityError::UnsupportedArgument(format!( + "bigint text {text:?} is not a decimal integer" + ))); + } + text + } + ArgValue::Str(text) => text, + })) + } +} + +fn is_decimal_integer(text: &str) -> bool { + let digits = text.strip_prefix('-').unwrap_or(text); + !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) +} + +/// Normalize a record of key arguments: omit absent values, spell scalars +/// the JavaScript way, and sort names by UTF-16 code units. +/// +/// The sort is stable, so entries whose names compare equal keep their input order. +pub fn normalize_args(args: I) -> Result, IdentityError> +where + I: IntoIterator, + K: Into, +{ + let mut pairs = Vec::new(); + for (name, value) in args { + if let Some(text) = value.spell()? { + pairs.push((name.into(), text)); + } + } + pairs.sort_by(|left, right| compare_utf16(&left.0, &right.0)); + Ok(pairs) +} + +fn contains_brace(component: &str) -> bool { + component.contains(['{', '}']) +} + +/// Build every key of an identity. +/// +/// The namespace may never contain a hash-tag brace. The key type and ID +/// are restricted the same way only when the identity is tracked, because +/// only then do they form the Redis Cluster hash tag; untracked braces are +/// percent-escaped like any other reserved character. +pub fn keys(identity: &Identity) -> Result { + if contains_brace(&identity.namespace) + || (identity.tracked + && (contains_brace(&identity.key_type) || contains_brace(&identity.id))) + { + return Err(IdentityError::ReservedDelimiter); + } + let entity = format!( + "{}:{}:{}", + escape_component(&identity.namespace), + escape_component(&identity.key_type), + escape_component(&identity.id) + ); + let (mut logical, watermark) = if identity.tracked { + let tagged = format!("{{{entity}}}"); + let watermark = format!("{tagged}#watermark"); + (tagged, Some(watermark)) + } else { + (entity, None) + }; + for (index, (name, value)) in identity.args.iter().enumerate() { + logical.push(if index == 0 { '?' } else { '&' }); + logical.push_str(&escape_component(name)); + logical.push('='); + logical.push_str(&escape_component(value)); + } + logical.push('#'); + logical.push_str(&escape_component(&identity.use_case)); + let value = format!("{logical}{FRAME_KEY_SUFFIX}"); + Ok(Keys { + logical, + value, + watermark, + }) +} + +/// `encodeURIComponent` escaping of one component. +/// +/// Bytes of the UTF-8 encoding are kept literally when they are +/// `A-Z a-z 0-9 - _ . ! ~ * ' ( )` and percent-encoded with uppercase hex otherwise. +pub fn escape_component(component: &str) -> String { + let mut escaped = String::with_capacity(component.len()); + for byte in component.bytes() { + if byte.is_ascii_alphanumeric() || b"-_.!~*'()".contains(&byte) { + escaped.push(byte as char); + } else { + let _ = write!(escaped, "%{byte:02X}"); + } + } + escaped +} + +/// FNV-1a 32-bit hash over the UTF-16 code units of `logical_key:discriminator`; +/// the integer numerator behind [`cohort`]. +pub fn cohort_hash(logical_key: &str, discriminator: &str) -> u32 { + let units = logical_key + .encode_utf16() + .chain(":".encode_utf16()) + .chain(discriminator.encode_utf16()); + units.fold(0x811c_9dc5_u32, |hash, unit| { + (hash ^ u32::from(unit)).wrapping_mul(0x0100_0193) + }) +} + +/// Deterministic rollout sample in `[0, 100)` for a logical key and layer or +/// shadow discriminator: FNV-1a over the UTF-16 units of `key:discriminator`. +pub fn cohort(logical_key: &str, discriminator: &str) -> f64 { + f64::from(cohort_hash(logical_key, discriminator)) / 4_294_967_296.0 * 100.0 +} + +/// Spell a number the way JavaScript's `String(number)` does. +/// +/// This is ECMA-262 `Number::toString(10)`: the shortest digit string that +/// round-trips, laid out as plain decimal for exponents in `(-7, 21]` and as +/// `d.ddde±X` otherwise. Negative zero spells `0`. +pub fn js_number_to_string(value: f64) -> String { + // Rust's standard shortest formatter rounds some decimal ties away from + // zero; ECMAScript chooses the even significand. Those spellings are part + // of the shared key identity, so use the ECMAScript variant of Ryū. + ryu_js::Buffer::new().format(value).to_owned() +} + +/// Compare two strings by UTF-16 code units, as JavaScript's `<` does. +pub fn compare_utf16(left: &str, right: &str) -> Ordering { + left.encode_utf16().cmp(right.encode_utf16()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn js_numbers_follow_ecmascript_spelling() { + let cases: &[(f64, &str)] = &[ + (f64::NAN, "NaN"), + (f64::INFINITY, "Infinity"), + (f64::NEG_INFINITY, "-Infinity"), + (0.0, "0"), + (-0.0, "0"), + (1.5, "1.5"), + (-1.5, "-1.5"), + (100.0, "100"), + (1e21, "1e+21"), + (1e-7, "1e-7"), + (-1e-7, "-1e-7"), + (0.000001, "0.000001"), + (123456789012345680000.0, "123456789012345680000"), + (9007199254740991.0, "9007199254740991"), + (1.25, "1.25"), + (0.1, "0.1"), + (1.2345e-7, "1.2345e-7"), + (1.5e300, "1.5e+300"), + (5e-324, "5e-324"), + (f64::MAX, "1.7976931348623157e+308"), + (0.5, "0.5"), + (123.456, "123.456"), + (1e20, "100000000000000000000"), + // Exact binary fractions whose shortest decimal candidates tie. + (f64::from_bits(0x430c6bf526340002), "1000000000000000.2"), + (f64::from_bits(0xc30c6bf526340002), "-1000000000000000.2"), + (f64::from_bits(0x430c6bf526340006), "1000000000000000.8"), + (f64::from_bits(0xc30c6bf526340006), "-1000000000000000.8"), + (f64::from_bits(0x42d6bcc41e900008), "100000000000000.12"), + (f64::from_bits(0x42d6bcc41e900018), "100000000000000.38"), + ]; + for (value, expected) in cases { + assert_eq!( + js_number_to_string(*value), + *expected, + "spelling of {value:?}" + ); + } + } + + #[test] + fn escaping_matches_encode_uri_component() { + assert_eq!(escape_component("é"), "%C3%A9"); + assert_eq!(escape_component("😀"), "%F0%9F%98%80"); + assert_eq!(escape_component("~!*'()-._"), "~!*'()-._"); + assert_eq!( + escape_component("a b+c/d?e=f&g#h%"), + "a%20b%2Bc%2Fd%3Fe%3Df%26g%23h%25" + ); + assert_eq!(escape_component("{1}"), "%7B1%7D"); + assert_eq!(escape_component(""), ""); + } + + #[test] + fn utf16_ordering_places_astral_before_private_use() { + assert_eq!(compare_utf16("😀", "\u{e000}"), Ordering::Less); + assert_eq!(compare_utf16("\u{ffff}", "😀"), Ordering::Greater); + assert_eq!(compare_utf16("a", "ab"), Ordering::Less); + assert_eq!(compare_utf16("b", "ab"), Ordering::Greater); + assert_eq!(compare_utf16("", ""), Ordering::Equal); + } + + #[test] + fn normalization_spells_scalars_and_sorts_names() { + let normalized = normalize_args(vec![ + ("z", ArgValue::Str("last".into())), + ("missing", ArgValue::Absent), + ("a", ArgValue::Int(1)), + ("nil", ArgValue::Null), + ("flag", ArgValue::Bool(false)), + ("wide", ArgValue::BigInt("-9007199254740993".into())), + ("tiny", ArgValue::Number(1e-7)), + ]) + .unwrap(); + let expected: Vec<(String, String)> = [ + ("a", "1"), + ("flag", "false"), + ("nil", "null"), + ("tiny", "1e-7"), + ("wide", "-9007199254740993"), + ("z", "last"), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), value.to_owned())) + .collect(); + assert_eq!(normalized, expected); + } + + #[test] + fn malformed_bigint_text_is_rejected() { + for text in ["", "-", "1.5", "0x10", "1e3", "+1", "１"] { + assert!( + matches!( + normalize_args(vec![("n", ArgValue::BigInt(text.into()))]), + Err(IdentityError::UnsupportedArgument(_)) + ), + "bigint text {text:?} should be rejected" + ); + } + } + + #[test] + fn braces_are_reserved_only_where_they_form_a_hash_tag() { + let untracked = Identity::new("{kind}", "{1}", "Get").namespace("urn"); + assert_eq!( + untracked.keys().unwrap().logical, + "urn:%7Bkind%7D:%7B1%7D#Get" + ); + assert_eq!( + untracked.clone().tracked(true).keys(), + Err(IdentityError::ReservedDelimiter) + ); + assert_eq!( + Identity::new("k", "1", "Get").namespace("bad{ns").keys(), + Err(IdentityError::ReservedDelimiter) + ); + let tracked = Identity::new("user_id", "123", "GetUser") + .namespace("users-api") + .tracked(true) + .args(vec![("locale".into(), "en".into())]); + let keys = tracked.keys().unwrap(); + assert_eq!(keys.logical, "{users-api:user_id:123}?locale=en#GetUser"); + assert_eq!( + keys.value, + "{users-api:user_id:123}?locale=en#GetUser:dialcache-frame-v1" + ); + assert_eq!( + keys.watermark.as_deref(), + Some("{users-api:user_id:123}#watermark") + ); + } + + #[test] + fn cohort_is_fnv1a_over_utf16_units() { + assert_eq!(cohort_hash("urn:id:1#Get", "local"), 1_652_509_740); + assert_eq!(cohort("urn:id:1#Get", "local"), 38.475490640848875); + assert_eq!(cohort_hash("", ""), { + let mut hash = 0x811c_9dc5_u32; + hash ^= u32::from(b':'); + hash.wrapping_mul(0x0100_0193) + }); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 00000000..2a4d8ed1 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,80 @@ +//! DialCache: explicitly enabled, layered caching with runtime rollout, +//! request coalescing, tracked invalidation, stale recovery and shadow +//! validation. +//! +//! This crate is a port of the TypeScript library. Its portable behavior is +//! defined by the Quint models under `formal/` and checked by replaying the +//! same generated histories through the real API. + +#![forbid(unsafe_code)] +#![warn(missing_debug_implementations)] +#![warn(missing_docs)] + +mod blocking; +pub mod cancel; +pub mod clock; +pub mod codec; +#[cfg(test)] +mod cpu_tests; +pub mod datadog; +mod deadline; +mod engine; +pub mod error; +mod execution; +mod flight; +pub mod identity; +pub mod limits; +pub mod local; +pub mod metrics; +pub mod observe; +pub mod operation; +pub mod policy; +pub mod preview; +#[cfg(feature = "prometheus")] +pub mod prometheus; +pub mod protocol; +#[cfg(feature = "redis")] +pub mod redis; +pub mod remote; +pub mod runtime; +mod scope; +mod shadow; +#[cfg(feature = "test-util")] +pub mod testing; +mod use_case; + +#[cfg(feature = "prometheus")] +pub use self::prometheus::{CollectorSchema, PrometheusError, PrometheusObserver}; +pub use cancel::CancelToken; +pub use clock::{Clock, SystemClock}; +pub use codec::{Codec, FromSync, JsonCodec, Payload, SyncCodec}; +pub use datadog::{ + DatadogError, DatadogObserver, DatadogOptions, DogStatsdClient, ObservationMetricType, +}; +pub use engine::{ + CoalescingState, DialCache, DialCacheBuilder, PolicyProvider, ProcessCoalescingState, + ScopeGuard, +}; +pub use error::{BoxError, ConfigError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; +pub use identity::{normalize_args, ArgValue, Identity, IdentityError, IntoKeyId, Keys}; +pub use local::{LocalEntry, LocalRead, LocalStore, LruLocalStore, StoredValue}; +pub use metrics::MetricKind; +pub use observe::{Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; +pub use operation::{Comparator, Operation, Preview, RecoveryPredicate, SourceBudget}; +pub use policy::{ + Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy, +}; +#[cfg(feature = "redis")] +pub use redis::{ + invalidation_script_sha1, RedisAdapter, RedisConnection, RedisProtocolError, + RedisReadCancelled, INVALIDATION_SCRIPT, +}; +pub use remote::{ + Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, Remote, + WriteRequest, +}; +pub use runtime::Runtime; +#[cfg(feature = "tokio")] +pub use runtime::TokioRuntime; +pub use scope::Scope; +pub use use_case::{KeySpec, UseCase, UseCaseBuilder}; diff --git a/rust/src/limits.rs b/rust/src/limits.rs new file mode 100644 index 00000000..50214a76 --- /dev/null +++ b/rust/src/limits.rs @@ -0,0 +1,36 @@ +//! Numeric domains shared by every DialCache implementation. + +/// Largest integer JavaScript represents exactly; timestamps and counters +/// beyond it are rejected rather than rounded. +pub const MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991; +/// Fixed 365-day ceiling shared by cache TTLs and invalidation buffers, in milliseconds. +pub const MAX_SUPPORTED_DURATION_MS: u64 = 365 * 24 * 60 * 60 * 1_000; +/// Cache TTL ceiling in whole seconds. +pub const MAX_CACHE_TTL_SEC: u64 = MAX_SUPPORTED_DURATION_MS / 1_000; +/// Tracked Redis values are physically retained for at most one hour so +/// invalidation markers can safely age out. +pub const MAX_TRACKED_VALUE_TTL_MS: u64 = 60 * 60 * 1_000; +/// Largest source or read deadline, in milliseconds. +pub const MAX_DEADLINE_MS: u64 = 2_147_483_647; +/// Library default remote read budget. +pub const DEFAULT_REMOTE_READ_TIMEOUT_MS: u64 = 50; +/// Library default source budget. +pub const DEFAULT_FALLBACK_TIMEOUT_MS: u64 = 60_000; +/// Default process-local capacity per instance. +pub const DEFAULT_LOCAL_CAPACITY: usize = 10_000; +/// Default concurrent shadow jobs per instance. +pub const DEFAULT_SHADOW_MAX_IN_FLIGHT: usize = 1; +/// Ceiling on one decompressed payload (Redis's own value limit). +pub const MAX_DECOMPRESSED_BYTES: usize = 512 * 1024 * 1024; +/// Default compression threshold in serialized bytes. +pub const DEFAULT_COMPRESSION_THRESHOLD_BYTES: usize = 4096; +/// Default zstd level. +pub const DEFAULT_ZSTD_LEVEL: i32 = 3; +/// Reserved use case name owned by invalidation watermarks. +pub const WATERMARK_USE_CASE: &str = "watermark"; +/// Key suffix of every stored value frame. +pub const FRAME_KEY_SUFFIX: &str = ":dialcache-frame-v1"; +/// Minimum watermark retention: twice the tracked value cap. +pub const MIN_WATERMARK_TTL_MS: u64 = 2 * MAX_TRACKED_VALUE_TTL_MS; +/// Slack added to derived watermark retention. +pub const WATERMARK_TTL_MARGIN_MS: u64 = 60_000; diff --git a/rust/src/local.rs b/rust/src/local.rs new file mode 100644 index 00000000..494d47ef --- /dev/null +++ b/rust/src/local.rs @@ -0,0 +1,125 @@ +//! Process-local storage. + +use std::any::Any; +use std::num::NonZeroUsize; +use std::sync::Arc; + +use crate::error::BoxError; + +/// A type-erased cached value shared by reference. +pub type StoredValue = Arc; + +/// One process-local entry with its insertion-time TTL. +#[derive(Clone)] +pub struct LocalEntry { + /// The cached value, shared by reference with every reader. + pub value: StoredValue, + /// Whole-millisecond elapsed reading at insertion. + pub inserted_ms: i64, + /// Lifetime in whole milliseconds; the entry is expired once + /// `now_ms - inserted_ms >= ttl_ms`. + pub ttl_ms: i64, +} + +/// Process-local storage for one cache instance. +/// +/// The default [`LruLocalStore`] evicts the least recently used entry at +/// capacity regardless of its remaining TTL, checks expiry lazily on read +/// against whole elapsed milliseconds, promotes on read and write, and never +/// renews a TTL on read. Implementations must preserve those rules. +pub trait LocalStore: Send + 'static { + /// Read one key. A live entry is promoted and returned; an expired entry + /// is removed without promotion and handed back so the cache can drop it + /// outside its lock. + fn get(&mut self, key: &str, now_ms: i64) -> Result; + /// Insert or replace an entry, promoting it and evicting at capacity. + /// Returns the displaced entry (the replaced value or the evicted tail), + /// which the cache drops outside its lock. + fn put(&mut self, key: String, entry: LocalEntry) -> Result, BoxError>; +} + +/// The outcome of one [`LocalStore::get`]. +/// +/// Removed entries travel back to the cache instead of dropping inside the +/// store, because a value's destructor may call back into the cache and the +/// store runs under the cache's lock. +#[derive(Debug)] +pub enum LocalRead { + /// No entry under the key. + Absent, + /// The entry had expired and was removed. + Expired(LocalEntry), + /// A live entry, promoted. + Live(StoredValue), +} + +/// The default LRU store. +pub struct LruLocalStore { + entries: lru::LruCache, +} + +impl LruLocalStore { + /// A store holding at most `capacity` entries. Capacity must be positive; + /// storage is allocated as entries arrive rather than for the full limit. + pub fn new(capacity: NonZeroUsize) -> Self { + LruLocalStore { + entries: lru::LruCache::sparse(capacity), + } + } + + /// Entries currently held, expired ones included until a read removes them. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Whether no entries are held. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } +} + +impl LocalStore for LruLocalStore { + fn get(&mut self, key: &str, now_ms: i64) -> Result { + // Peek, check freshness, then promote: an expired entry leaves the LRU + // order untouched apart from its own removal. + let expired = match self.entries.peek(key) { + None => return Ok(LocalRead::Absent), + Some(entry) => now_ms.saturating_sub(entry.inserted_ms) >= entry.ttl_ms, + }; + if expired { + return Ok(match self.entries.pop(key) { + Some(entry) => LocalRead::Expired(entry), + None => LocalRead::Absent, + }); + } + Ok(match self.entries.get(key) { + Some(entry) => LocalRead::Live(entry.value.clone()), + None => LocalRead::Absent, + }) + } + + fn put(&mut self, key: String, entry: LocalEntry) -> Result, BoxError> { + Ok(self + .entries + .push(key, entry) + .map(|(_, displaced)| displaced)) + } +} + +impl std::fmt::Debug for LocalEntry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LocalEntry") + .field("inserted_ms", &self.inserted_ms) + .field("ttl_ms", &self.ttl_ms) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for LruLocalStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LruLocalStore") + .field("len", &self.entries.len()) + .field("cap", &self.entries.cap()) + .finish() + } +} diff --git a/rust/src/metrics.rs b/rust/src/metrics.rs new file mode 100644 index 00000000..1b1e8688 --- /dev/null +++ b/rust/src/metrics.rs @@ -0,0 +1,623 @@ +//! The shared event-to-metric mapping of the bundled exporters. +//! +//! Every [`Event`] maps to exactly one [`MetricKind`]: a counter or an +//! observation (timer, age, size, ratio). The kind fixes the metric's label +//! set and value so the Prometheus and Datadog exporters, and the TypeScript +//! and Go ports, publish the same wire contract. Logical cache keys never +//! appear here: labels are bounded enumerations plus the namespace, use case +//! and key type. + +use crate::observe::Event; + +/// The metric an [`Event`] feeds. Variant names match the TypeScript adapter +/// method names and the Go `metricKinds` table. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MetricKind { + /// Counter of lookups that reached an enabled layer ([`Event::Request`]). + Request, + /// Counter of classified misses ([`Event::Miss`]). + Miss, + /// Counter of skipped layers and no-layer calls ([`Event::Disabled`]). + Disabled, + /// Counter of bounded failure sites ([`Event::Error`]). + Error, + /// Counter of explicit invalidation attempts ([`Event::Invalidation`]). + Invalidation, + /// Counter of callers that joined an in-flight execution ([`Event::Coalesced`]). + Coalesced, + /// Counter of terminal shadow job outcomes ([`Event::ShadowValidation`]). + ShadowValidation, + /// Seconds: validated value age at a match or mismatch verdict ([`Event::ShadowValueAge`]). + ShadowValueAge, + /// Seconds: how far a frame is dated ahead of the observing clock + /// ([`Event::FutureTimestampOffset`]). + FutureTimestampOffset, + /// Counter of authorized stale-recovery checks ([`Event::StaleRecovery`]). + StaleRecovery, + /// Seconds: age of a stale value when recovery served it + /// ([`Event::StaleRecoveryValueAge`]). + StaleRecoveryValueAge, + /// Counter of envelope outcomes on writes and reads ([`Event::Compression`]). + Compression, + /// Seconds: layer lookup latency ([`Event::Get`]). + Get, + /// Seconds: time until the source settled or timed out ([`Event::Fallback`]). + Fallback, + /// Seconds: codec encode or decode latency ([`Event::Serialization`]). + Serialization, + /// Bytes: serializer output before compression and escaping ([`Event::Size`]). + Size, + /// Bytes: prepared payload after compression and escaping ([`Event::StoredSize`]). + StoredSize, + /// Ratio: stored over original bytes of a compressed write ([`Event::CompressionRatio`]). + CompressionRatio, + /// Seconds: zstd compress or decompress latency ([`Event::CompressionDuration`]). + CompressionDuration, +} + +/// The base label names every layer-scoped metric starts with. +const LABEL_CACHE_NAMESPACE: &str = "cache_namespace"; +const LABEL_USE_CASE: &str = "use_case"; +const LABEL_KEY_TYPE: &str = "key_type"; + +impl MetricKind { + /// Every kind, in declaration order. + pub const ALL: [MetricKind; 19] = [ + MetricKind::Request, + MetricKind::Miss, + MetricKind::Disabled, + MetricKind::Error, + MetricKind::Invalidation, + MetricKind::Coalesced, + MetricKind::ShadowValidation, + MetricKind::ShadowValueAge, + MetricKind::FutureTimestampOffset, + MetricKind::StaleRecovery, + MetricKind::StaleRecoveryValueAge, + MetricKind::Compression, + MetricKind::Get, + MetricKind::Fallback, + MetricKind::Serialization, + MetricKind::Size, + MetricKind::StoredSize, + MetricKind::CompressionRatio, + MetricKind::CompressionDuration, + ]; + + /// The metric an event feeds. + pub fn of(event: &Event) -> MetricKind { + match event { + Event::Request { .. } => MetricKind::Request, + Event::Miss { .. } => MetricKind::Miss, + Event::Disabled { .. } => MetricKind::Disabled, + Event::Error { .. } => MetricKind::Error, + Event::Invalidation { .. } => MetricKind::Invalidation, + Event::Coalesced { .. } => MetricKind::Coalesced, + Event::ShadowValidation { .. } => MetricKind::ShadowValidation, + Event::ShadowValueAge { .. } => MetricKind::ShadowValueAge, + Event::FutureTimestampOffset { .. } => MetricKind::FutureTimestampOffset, + Event::StaleRecovery { .. } => MetricKind::StaleRecovery, + Event::StaleRecoveryValueAge { .. } => MetricKind::StaleRecoveryValueAge, + Event::Compression { .. } => MetricKind::Compression, + Event::Get { .. } => MetricKind::Get, + Event::Fallback { .. } => MetricKind::Fallback, + Event::Serialization { .. } => MetricKind::Serialization, + Event::Size { .. } => MetricKind::Size, + Event::StoredSize { .. } => MetricKind::StoredSize, + Event::CompressionRatio { .. } => MetricKind::CompressionRatio, + Event::CompressionDuration { .. } => MetricKind::CompressionDuration, + } + } + + /// The kind's name in the TypeScript adapter and the Go `metricKinds` table. + pub fn as_str(self) -> &'static str { + match self { + MetricKind::Request => "request", + MetricKind::Miss => "miss", + MetricKind::Disabled => "disabled", + MetricKind::Error => "error", + MetricKind::Invalidation => "invalidation", + MetricKind::Coalesced => "coalesced", + MetricKind::ShadowValidation => "shadowValidation", + MetricKind::ShadowValueAge => "shadowValueAge", + MetricKind::FutureTimestampOffset => "futureTimestampOffset", + MetricKind::StaleRecovery => "staleRecovery", + MetricKind::StaleRecoveryValueAge => "staleRecoveryValueAge", + MetricKind::Compression => "compression", + MetricKind::Get => "get", + MetricKind::Fallback => "fallback", + MetricKind::Serialization => "serialization", + MetricKind::Size => "size", + MetricKind::StoredSize => "storedSize", + MetricKind::CompressionRatio => "compressionRatio", + MetricKind::CompressionDuration => "compressionDuration", + } + } + + /// Counters increment by one per event; every other kind records + /// [`value`](Self::value). + pub fn is_counter(self) -> bool { + matches!( + self, + MetricKind::Request + | MetricKind::Miss + | MetricKind::Disabled + | MetricKind::Error + | MetricKind::Invalidation + | MetricKind::Coalesced + | MetricKind::ShadowValidation + | MetricKind::StaleRecovery + | MetricKind::Compression + ) + } + + /// Position of the kind in [`ALL`](Self::ALL). + pub fn index(self) -> usize { + self as usize + } + + /// The label names of this kind, in wire order: the Prometheus label + /// schema and the Datadog tag set of the TypeScript adapters. + pub fn label_names(self) -> &'static [&'static str] { + const BASE: &[&str] = &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + ]; + const OUTCOME: &[&str] = &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "outcome", + ]; + match self { + MetricKind::Request + | MetricKind::FutureTimestampOffset + | MetricKind::Get + | MetricKind::Fallback + | MetricKind::Size + | MetricKind::StoredSize + | MetricKind::CompressionRatio => BASE, + MetricKind::Miss | MetricKind::Disabled => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + "reason", + ], + MetricKind::Error => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + "error", + "in_fallback", + ], + MetricKind::Invalidation => &[LABEL_CACHE_NAMESPACE, LABEL_KEY_TYPE, "layer"], + MetricKind::Coalesced => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "scope", + ], + MetricKind::ShadowValidation + | MetricKind::ShadowValueAge + | MetricKind::StaleRecovery + | MetricKind::StaleRecoveryValueAge => OUTCOME, + MetricKind::Compression => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + "outcome", + ], + MetricKind::Serialization | MetricKind::CompressionDuration => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + "operation", + ], + } + } + + /// The labels of an event, in the order of [`label_names`](Self::label_names) + /// for [`of(event)`](Self::of). Values are bounded enumerations plus the + /// namespace, use case and key type; the logical key is never one. + pub fn labels(event: &Event) -> Vec<(&'static str, String)> { + let mut labels: Vec<(&'static str, String)> = Vec::with_capacity(6); + match event { + Event::Invalidation { + namespace, + key_type, + layer, + } => { + labels.push((LABEL_CACHE_NAMESPACE, namespace.to_string())); + labels.push((LABEL_KEY_TYPE, key_type.to_string())); + labels.push(("layer", layer.as_str().to_string())); + } + Event::Coalesced { + labels: outcome, + scope, + } => { + push_outcome_base(&mut labels, outcome); + labels.push(("scope", scope.as_str().to_string())); + } + Event::ShadowValidation { + labels: outcome, + outcome: verdict, + } + | Event::ShadowValueAge { + labels: outcome, + outcome: verdict, + .. + } => { + push_outcome_base(&mut labels, outcome); + labels.push(("outcome", verdict.as_str().to_string())); + } + Event::StaleRecovery { + labels: outcome, + outcome: verdict, + } + | Event::StaleRecoveryValueAge { + labels: outcome, + outcome: verdict, + .. + } => { + push_outcome_base(&mut labels, outcome); + labels.push(("outcome", verdict.as_str().to_string())); + } + Event::Request { labels: base } + | Event::FutureTimestampOffset { labels: base, .. } + | Event::Get { labels: base, .. } + | Event::Fallback { labels: base, .. } + | Event::Size { labels: base, .. } + | Event::StoredSize { labels: base, .. } + | Event::CompressionRatio { labels: base, .. } => push_layer_base(&mut labels, base), + Event::Miss { + labels: base, + reason, + } => { + push_layer_base(&mut labels, base); + labels.push(("reason", reason.as_str().to_string())); + } + Event::Disabled { + labels: base, + reason, + } => { + push_layer_base(&mut labels, base); + labels.push(("reason", reason.as_str().to_string())); + } + Event::Error { + labels: base, + error, + in_fallback, + } => { + push_layer_base(&mut labels, base); + labels.push(("error", error.as_str().to_string())); + labels.push(("in_fallback", in_fallback.to_string())); + } + Event::Compression { + labels: base, + outcome, + } => { + push_layer_base(&mut labels, base); + labels.push(("outcome", outcome.as_str().to_string())); + } + Event::Serialization { + labels: base, + operation, + .. + } => { + push_layer_base(&mut labels, base); + labels.push(("operation", operation.as_str().to_string())); + } + Event::CompressionDuration { + labels: base, + operation, + .. + } => { + push_layer_base(&mut labels, base); + labels.push(("operation", operation.as_str().to_string())); + } + } + labels + } + + /// The value an event records: seconds for timers, ages and offsets, + /// bytes for sizes, the compressed-to-original ratio for + /// [`CompressionRatio`](Self::CompressionRatio), and the increment `1` + /// for counters. + pub fn value(event: &Event) -> f64 { + match event { + Event::ShadowValueAge { seconds, .. } + | Event::FutureTimestampOffset { seconds, .. } + | Event::StaleRecoveryValueAge { seconds, .. } + | Event::Get { seconds, .. } + | Event::Fallback { seconds, .. } + | Event::Serialization { seconds, .. } + | Event::CompressionDuration { seconds, .. } => *seconds, + Event::Size { bytes, .. } | Event::StoredSize { bytes, .. } => *bytes as f64, + Event::CompressionRatio { ratio, .. } => *ratio, + Event::Request { .. } + | Event::Miss { .. } + | Event::Disabled { .. } + | Event::Error { .. } + | Event::Invalidation { .. } + | Event::Coalesced { .. } + | Event::ShadowValidation { .. } + | Event::StaleRecovery { .. } + | Event::Compression { .. } => 1.0, + } + } +} + +fn push_layer_base(labels: &mut Vec<(&'static str, String)>, base: &crate::observe::Labels) { + labels.push((LABEL_CACHE_NAMESPACE, base.namespace.to_string())); + labels.push((LABEL_USE_CASE, base.use_case.to_string())); + labels.push((LABEL_KEY_TYPE, base.key_type.to_string())); + labels.push(("layer", base.layer.as_str().to_string())); +} + +fn push_outcome_base( + labels: &mut Vec<(&'static str, String)>, + base: &crate::observe::OutcomeLabels, +) { + labels.push((LABEL_CACHE_NAMESPACE, base.namespace.to_string())); + labels.push((LABEL_USE_CASE, base.use_case.to_string())); + labels.push((LABEL_KEY_TYPE, base.key_type.to_string())); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::observe::{ + CoalescingScope, CompressionOperation, CompressionOutcome, DisabledReason, ErrorKind, + Labels, Layer, MissReason, OutcomeLabels, RecoveryOutcome, SerializationOperation, + ShadowOutcome, + }; + use std::sync::Arc; + + fn base() -> Labels { + Labels { + namespace: Arc::from("logical"), + use_case: Arc::from("lookup"), + key_type: Arc::from("item"), + layer: Layer::Remote, + } + } + + fn outcome() -> OutcomeLabels { + OutcomeLabels { + namespace: Arc::from("logical"), + use_case: Arc::from("lookup"), + key_type: Arc::from("item"), + } + } + + /// One event per kind with fixed, recognizable field values. + pub(crate) fn event_of(kind: MetricKind) -> Event { + match kind { + MetricKind::Request => Event::Request { labels: base() }, + MetricKind::Miss => Event::Miss { + labels: base(), + reason: MissReason::Expired, + }, + MetricKind::Disabled => Event::Disabled { + labels: base(), + reason: DisabledReason::RampedDown, + }, + MetricKind::Error => Event::Error { + labels: base(), + error: ErrorKind::Fallback, + in_fallback: true, + }, + MetricKind::Invalidation => Event::Invalidation { + namespace: Arc::from("logical"), + key_type: Arc::from("item"), + layer: Layer::Remote, + }, + MetricKind::Coalesced => Event::Coalesced { + labels: outcome(), + scope: CoalescingScope::Process, + }, + MetricKind::ShadowValidation => Event::ShadowValidation { + labels: outcome(), + outcome: ShadowOutcome::Mismatch, + }, + MetricKind::ShadowValueAge => Event::ShadowValueAge { + labels: outcome(), + outcome: ShadowOutcome::Mismatch, + seconds: 0.25, + }, + MetricKind::FutureTimestampOffset => Event::FutureTimestampOffset { + labels: base(), + seconds: 0.25, + }, + MetricKind::StaleRecovery => Event::StaleRecovery { + labels: outcome(), + outcome: RecoveryOutcome::Served, + }, + MetricKind::StaleRecoveryValueAge => Event::StaleRecoveryValueAge { + labels: outcome(), + outcome: RecoveryOutcome::Served, + seconds: 0.25, + }, + MetricKind::Compression => Event::Compression { + labels: base(), + outcome: CompressionOutcome::Compressed, + }, + MetricKind::Get => Event::Get { + labels: base(), + seconds: 0.25, + }, + MetricKind::Fallback => Event::Fallback { + labels: base(), + seconds: 0.25, + }, + MetricKind::Serialization => Event::Serialization { + labels: base(), + operation: SerializationOperation::Dump, + seconds: 0.25, + }, + MetricKind::Size => Event::Size { + labels: base(), + bytes: 123, + }, + MetricKind::StoredSize => Event::StoredSize { + labels: base(), + bytes: 123, + }, + MetricKind::CompressionRatio => Event::CompressionRatio { + labels: base(), + ratio: 0.25, + }, + MetricKind::CompressionDuration => Event::CompressionDuration { + labels: base(), + operation: CompressionOperation::Compress, + seconds: 0.25, + }, + } + } + + #[test] + fn every_kind_round_trips_through_of_and_index() { + for (index, kind) in MetricKind::ALL.iter().enumerate() { + assert_eq!(MetricKind::of(&event_of(*kind)), *kind); + assert_eq!(kind.index(), index); + } + let names: std::collections::HashSet<&str> = + MetricKind::ALL.iter().map(|k| k.as_str()).collect(); + assert_eq!(names.len(), 19); + } + + #[test] + fn label_values_follow_the_declared_label_names() { + for kind in MetricKind::ALL { + let labels = MetricKind::labels(&event_of(kind)); + let names: Vec<&str> = labels.iter().map(|(name, _)| *name).collect(); + assert_eq!(names, kind.label_names(), "{}", kind.as_str()); + } + } + + #[test] + fn labels_derive_as_the_go_port_does() { + let pairs = |kind: MetricKind| -> Vec<(String, String)> { + MetricKind::labels(&event_of(kind)) + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect() + }; + let owned = |items: &[(&str, &str)]| -> Vec<(String, String)> { + items + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + }; + assert_eq!( + pairs(MetricKind::Error), + owned(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("layer", "remote"), + ("error", "fallback"), + ("in_fallback", "true"), + ]) + ); + assert_eq!( + pairs(MetricKind::Invalidation), + owned(&[ + ("cache_namespace", "logical"), + ("key_type", "item"), + ("layer", "remote"), + ]) + ); + assert_eq!( + pairs(MetricKind::Coalesced), + owned(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("scope", "process"), + ]) + ); + assert_eq!( + pairs(MetricKind::ShadowValidation), + owned(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("outcome", "mismatch"), + ]) + ); + assert_eq!( + pairs(MetricKind::Miss), + owned(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("layer", "remote"), + ("reason", "expired"), + ]) + ); + assert_eq!( + pairs(MetricKind::Disabled).last().unwrap(), + &("reason".to_string(), "ramped_down".to_string()) + ); + assert_eq!( + pairs(MetricKind::Compression).last().unwrap(), + &("outcome".to_string(), "compressed".to_string()) + ); + assert_eq!( + pairs(MetricKind::Serialization).last().unwrap(), + &("operation".to_string(), "dump".to_string()) + ); + assert_eq!( + pairs(MetricKind::CompressionDuration).last().unwrap(), + &("operation".to_string(), "compress".to_string()) + ); + let in_fallback_false = Event::Error { + labels: base(), + error: ErrorKind::CacheRead, + in_fallback: false, + }; + assert_eq!( + MetricKind::labels(&in_fallback_false).last().unwrap().1, + "false" + ); + } + + #[test] + fn values_carry_the_documented_units() { + for kind in MetricKind::ALL { + let value = MetricKind::value(&event_of(kind)); + let expected = match kind { + MetricKind::Size | MetricKind::StoredSize => 123.0, + _ if kind.is_counter() => 1.0, + _ => 0.25, + }; + assert_eq!(value, expected, "{}", kind.as_str()); + } + let counters: Vec<&str> = MetricKind::ALL + .iter() + .filter(|k| k.is_counter()) + .map(|k| k.as_str()) + .collect(); + assert_eq!( + counters, + [ + "request", + "miss", + "disabled", + "error", + "invalidation", + "coalesced", + "shadowValidation", + "staleRecovery", + "compression", + ] + ); + } +} diff --git a/rust/src/observe.rs b/rust/src/observe.rs new file mode 100644 index 00000000..2d38e062 --- /dev/null +++ b/rust/src/observe.rs @@ -0,0 +1,579 @@ +//! Backend-neutral diagnostics: metric events and structured log events. + +use std::fmt; +use std::sync::Arc; + +use crate::error::BoxError; +pub use crate::remote::MissReason; + +macro_rules! str_enum { + ($(#[$meta:meta])* $name:ident { $($(#[$vmeta:meta])* $variant:ident => $text:literal),+ $(,)? }) => { + $(#[$meta])* + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub enum $name { $($(#[$vmeta])* $variant),+ } + impl $name { + /// The `snake_case` label value shared with the TypeScript and Go + /// ports and published by the bundled metric exporters. + pub fn as_str(self) -> &'static str { + match self { $($name::$variant => $text),+ } + } + } + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) } + } + }; +} + +str_enum! { + /// The cache layer a diagnostic refers to. + Layer { + /// No layer was reached: caching was disabled for the scope, key + /// construction failed, or the policy could not be resolved. + Noop => "noop", + /// The outermost enabled scope's request memo. + RequestLocal => "request_local", + /// The process-local store. + Local => "local", + /// The remote adapter on the caller-serving path. + Remote => "remote", + /// The remote adapter inside detached shadow work: its reads, fills, + /// serialization, compression and payload sizes. + RemoteShadow => "remote_shadow", + } +} + +str_enum! { + /// Where an in-flight execution was shared. + CoalescingScope { + /// The flight was registered on the outermost enabled scope of the + /// same request. + RequestLocal => "request_local", + /// The flight was registered on the cache instance; separate + /// instances in one process do not share flights. + Process => "process", + } +} + +str_enum! { + /// Why a cache layer was skipped. + DisabledReason { + /// Caching is not enabled for the call's scope; the source runs directly. + Context => "context", + /// The layer has no effective TTL after the runtime overlay, which + /// includes the default of an omitted policy. + PolicyDisabled => "policy_disabled", + /// The merged TTL leaf is not a whole number of seconds within 365 + /// days; only this layer is skipped, with a `config_resolution` error. + InvalidTtl => "invalid_ttl", + /// The merged ramp leaf is not a finite number in `[0, 100]`; only + /// this layer is skipped, with a `config_resolution` error. + InvalidRamp => "invalid_ramp", + /// The layer is configured but this key's stable cohort sample falls + /// outside its ramp. + RampedDown => "ramped_down", + /// The whole invocation's policy could not be resolved (`noop` + /// layer), or the process-local read failed (`local` layer). + ConfigError => "config_error", + } +} + +str_enum! { + /// Stable failure sites. + ErrorKind { + /// The key selector failed or panicked, or the identity could not + /// form a key. + KeyConstruction => "key_construction", + /// The policy provider or resolution failed for the invocation, a + /// layer leaf was invalid, or a recovery or shadow option was rejected. + ConfigResolution => "config_resolution", + /// A process-local read failed, or a remote read failed for a reason + /// other than its deadline. + CacheRead => "cache_read", + /// A remote read did not answer within its read budget. + CacheReadTimeout => "cache_read_timeout", + /// A process-local or remote write failed, or the write stamp left + /// the safe-integer domain. + CacheWrite => "cache_write", + /// A tracked write asked for retention above the one-hour physical + /// cap and was clamped: a configuration signal, not a failure. + TrackedTtlClamped => "tracked_ttl_clamped", + /// The codec failed or panicked while decoding a stored payload. + SerializationLoad => "serialization_load", + /// The codec failed or panicked while encoding a value for the remote layer. + SerializationDump => "serialization_dump", + /// zstd compression failed while preparing a remote write. + Compression => "compression", + /// An explicit invalidation failed, including one made without a + /// remote adapter. + Invalidation => "invalidation", + /// The source failed, panicked or exceeded its deadline; reported + /// with `in_fallback` set. + Fallback => "fallback", + } +} + +str_enum! { + /// Terminal outcomes of sampled shadow validation. + ShadowOutcome { + /// The cached and source values compared equal. + Match => "match", + /// They differed, and a confirmation read found the stored payload + /// unchanged. + Mismatch => "mismatch", + /// They differed, but the stored payload changed or disappeared + /// before confirmation. + Superseded => "superseded", + /// A clean shadow miss was populated. + Filled => "filled", + /// A tracked fill was withheld because its stamp did not exceed the + /// watermark observed by the initial read. + FillFenced => "fill_fenced", + /// Preparing (encoding, compression) or writing a clean-miss fill failed. + FillError => "fill_error", + /// The initial detached remote read failed, including by deadline. + RedisError => "redis_error", + /// The source-of-truth read failed. + SourceError => "source_error", + /// The retained payload could not be decoded for comparison. + DeserializationError => "deserialization_error", + /// The comparator returned an error or panicked. + ComparisonError => "comparison_error", + /// The confirmation remote read failed. + ConfirmationError => "confirmation_error", + /// The shadow deadline expired before a verdict. + Timeout => "timeout", + /// Per-key deduplication or the instance's in-flight cap rejected + /// the job before it started. + Dropped => "dropped", + } +} + +str_enum! { + /// Terminal outcomes of stale-on-error recovery. + RecoveryOutcome { + /// A retained stale frame passed the return-time age check and + /// supplied the result. + Served => "served", + /// No retained frame remained eligible: none was kept, or its age + /// reached the ceiling. + Miss => "miss", + /// Decoding the retained frame failed. + DeserializationError => "deserialization_error", + } +} + +str_enum! { + /// Compression outcomes for writes and reads. + CompressionOutcome { + /// Write: the payload met the threshold and the marked zstd frame was + /// smaller than the escaped raw form. + Compressed => "compressed", + /// Write: the payload was below the threshold and was stored raw. + BelowThreshold => "below_threshold", + /// Write: compression ran but did not shrink the payload; stored raw. + NotSmaller => "not_smaller", + /// Write: the payload exceeds the decompression cap and was stored raw + /// so a read could still accept it; a capacity signal. + WriteOverLimit => "write_over_limit", + /// Read: a marked zstd frame decoded successfully. + Decompressed => "decompressed", + /// Read: zstd rejected a marked payload as malformed or truncated; the + /// stored bytes were handed through untouched. + FallbackRaw => "fallback_raw", + /// Read: decompressing would exceed the cap; the stored bytes were + /// handed through untouched. An integrity signal. + ReadOverLimit => "read_over_limit", + } +} + +str_enum! { + /// Which codec direction a serialization timing measures. + SerializationOperation { + /// Encoding a value for the remote layer. + Dump => "dump", + /// Decoding a stored payload. + Load => "load", + } +} + +str_enum! { + /// Which envelope direction a compression timing measures. + CompressionOperation { + /// zstd compression during a write. + Compress => "compress", + /// zstd decompression during a read. + Decompress => "decompress", + } +} + +/// Labels shared by layer-scoped events. Logical keys are never labels. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Labels { + /// The instance namespace (default `urn`); unrelated to any exporter's + /// metric-name namespace. + pub namespace: Arc, + /// The operation name; `watermark` on invalidation error events. + pub use_case: Arc, + /// The entity type of the identity. + pub key_type: Arc, + /// The layer the event refers to, or [`Layer::Noop`] when none was reached. + pub layer: Layer, +} + +/// Labels of outcome-scoped events (shadow validation and recovery). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OutcomeLabels { + /// The instance namespace (default `urn`); unrelated to any exporter's + /// metric-name namespace. + pub namespace: Arc, + /// The operation name. + pub use_case: Arc, + /// The entity type of the identity. + pub key_type: Arc, +} + +/// One public diagnostic. Counters carry no value; timers report seconds; +/// sizes report bytes. +#[derive(Debug, Clone, PartialEq)] +pub enum Event { + /// One lookup of an enabled cache layer, hit or miss. Emitted before the + /// result for the remote layer and after it for the request-local and + /// process-local layers; a layer whose read failed emits `Error` and + /// `Disabled` instead. + Request { + /// The layer that was looked up. + labels: Labels, + }, + /// A layer lookup found no servable value. + Miss { + /// The layer that missed. + labels: Labels, + /// The bounded cause; request-local and process-local misses are + /// always [`MissReason::ValueAbsent`]. + reason: MissReason, + }, + /// A layer, or the whole call (`noop`), was skipped. + Disabled { + /// The skipped layer, or [`Layer::Noop`] when no layer was reached. + labels: Labels, + /// Why it was skipped. + reason: DisabledReason, + }, + /// A bounded failure site fired. Cache plumbing failures fail open; only + /// source failures reach the caller. + Error { + /// The layer whose work failed. + labels: Labels, + /// The failure site. + error: ErrorKind, + /// `true` when the source (fallback) failed or timed out rather than + /// cache plumbing. + in_fallback: bool, + }, + /// One explicit invalidation attempt, emitted before the remote call and + /// even when no remote adapter is configured. + Invalidation { + /// The namespace whose entity watermark is advanced. + namespace: Arc, + /// The entity type being invalidated. + key_type: Arc, + /// Always [`Layer::Remote`]: watermarks live only in the remote layer. + layer: Layer, + }, + /// A caller joined an execution already in flight for the same logical + /// key instead of starting its own. + Coalesced { + /// The identity of the joined execution. + labels: OutcomeLabels, + /// Whether the flight was registered on the request scope or the instance. + scope: CoalescingScope, + }, + /// One terminal outcome of an admitted or explicitly dropped shadow job. + ShadowValidation { + /// The identity the job validated. + labels: OutcomeLabels, + /// The terminal verdict. + outcome: ShadowOutcome, + }, + /// Age in seconds of the validated value at a match or confirmed mismatch, clamped at zero. + ShadowValueAge { + /// The identity the job validated. + labels: OutcomeLabels, + /// [`ShadowOutcome::Match`] or [`ShadowOutcome::Mismatch`]; no other + /// outcome carries an age. + outcome: ShadowOutcome, + /// Observing wall clock minus the frame's `created_at_ms`, never negative. + seconds: f64, + }, + /// Positive offset in seconds of a frame dated after the observing clock. + FutureTimestampOffset { + /// The layer whose read decoded the frame. + labels: Labels, + /// How far the frame's `created_at_ms` lies ahead of the observing wall clock. + seconds: f64, + }, + /// One classifier-authorized stale-on-error recovery check after a + /// source failure. + StaleRecovery { + /// The identity being recovered. + labels: OutcomeLabels, + /// Whether a retained frame was served. + outcome: RecoveryOutcome, + }, + /// Age of the retained frame at the moment recovery served it; emitted + /// only alongside [`RecoveryOutcome::Served`]. + StaleRecoveryValueAge { + /// The identity being recovered. + labels: OutcomeLabels, + /// Always [`RecoveryOutcome::Served`]. + outcome: RecoveryOutcome, + /// Observing wall clock minus the frame's `created_at_ms`, clamped at zero. + seconds: f64, + }, + /// One envelope decision: on every write while compression is enabled, + /// and on reads whose payload carried a zstd marker. + Compression { + /// The layer performing the write or read. + labels: Labels, + /// Write and read outcomes are disjoint sets; see [`CompressionOutcome`]. + outcome: CompressionOutcome, + }, + /// Lookup latency of one layer, hit or miss; a remote read's value + /// includes any wait up to its deadline. + Get { + /// The layer looked up. + labels: Labels, + /// Elapsed time of the lookup. + seconds: f64, + }, + /// Time until the source settled or its deadline elapsed, once per + /// source invocation that served the caller (shadow re-validation runs + /// the source without it). + Fallback { + /// [`Layer::RequestLocal`] when the request memo was consulted, + /// [`Layer::Remote`] when the remote layer was consulted, otherwise + /// [`Layer::Local`] (even with the local layer disabled), and + /// [`Layer::Noop`] only when the call bypassed caching. + labels: Labels, + /// Elapsed time from invoking the source until it settled or its + /// deadline fired. + seconds: f64, + }, + /// Codec latency of one encode or decode, emitted whether or not the + /// codec succeeded. + Serialization { + /// The layer the payload was written to or read from. + labels: Labels, + /// Encode (`dump`) or decode (`load`). + operation: SerializationOperation, + /// Elapsed time inside the codec. + seconds: f64, + }, + /// Serializer output size of one remote write, before compression and escaping. + Size { + /// The layer being written. + labels: Labels, + /// Length of the codec output. + bytes: u64, + }, + /// Prepared payload size of one remote write, after compression and + /// escaping, before dispatch. + StoredSize { + /// The layer being written. + labels: Labels, + /// Length of the payload handed to the adapter, excluding the + /// ten-byte frame header. + bytes: u64, + }, + /// Size ratio of one write that chose the compressed form. + CompressionRatio { + /// The layer being written. + labels: Labels, + /// Marked compressed bytes divided by serializer output bytes. + ratio: f64, + }, + /// zstd latency: on writes when compression ran (compressed or not + /// smaller), on reads when a marked frame was decoded or rejected. + CompressionDuration { + /// The layer performing the write or read. + labels: Labels, + /// Compress (write) or decompress (read). + operation: CompressionOperation, + /// Elapsed time of the envelope step. + seconds: f64, + }, +} + +/// Receives every public diagnostic. Failures (returned errors or panics) +/// never change a cache, source or maintenance result. +pub trait Observer: Send + Sync + 'static { + /// Receive one event. Called inline on the cache's own path under panic + /// isolation, so it should return quickly and never block. + fn observe(&self, event: &Event); + + /// Whether this observer consumes shadow validation outcomes. + /// + /// Shadow validation is diagnostic work that exists only to be observed, + /// so the cache admits a shadow job only when the observer opts in by + /// returning `true`. The bundled metric exporters opt in. + fn observes_shadow_outcomes(&self) -> bool { + false + } +} + +/// Bounded details of one confirmed shadow mismatch warning. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShadowMismatchDetails { + /// The instance namespace of the mismatching identity. + pub namespace: Arc, + /// The operation name of the mismatching identity. + pub use_case: Arc, + /// The entity type of the mismatching identity. + pub key_type: Arc, + /// The logical key, bounded to 2 KiB of UTF-8. + pub cache_key: String, + /// JSON preview of the cached value, bounded to 8 KiB; absent when unavailable. + pub cached_value_json: Option, + /// JSON preview of the source value, bounded to 8 KiB; absent when unavailable. + pub source_value_json: Option, +} + +/// Severity of a [`LogEvent`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum LogLevel { + /// Verbose diagnostics; no [`LogEvent`] maps here today. + Debug, + /// A diagnostic on a path where the cache itself added no failure: a + /// layer failed open and the call went on to another layer or the + /// source, or an explicit invalidation or a recovery decode failed and + /// the operation reports that failure. + Warn, + /// A whole path is broken: key construction or the process-local store. + Error, +} + +/// Structured log events. All are fail-open diagnostics except the +/// invalidation failure, which is also returned to the caller. +#[derive(Debug)] +pub enum LogEvent { + /// The key selector failed or panicked, or the identity could not form a + /// key; the source ran uncached. [`LogLevel::Error`]. + KeyConstructionFailed(BoxError), + /// The policy provider failed or panicked, or the merged policy was + /// malformed; the source ran uncached. + PolicyResolutionFailed(BoxError), + /// The [`LocalStore`](crate::LocalStore) read failed or panicked; the + /// local layer was skipped for the call. [`LogLevel::Error`]. + LocalReadFailed(BoxError), + /// The [`LocalStore`](crate::LocalStore) write failed or panicked; the + /// value was still returned. + LocalWriteFailed(BoxError), + /// The caller-serving remote read failed or exceeded its read budget. + /// Detached shadow reads do not log. + RemoteReadFailed(BoxError), + /// Preparing or dispatching a caller-serving remote write failed: + /// encoding, compression, the write stamp or the adapter. + RemoteWriteFailed(BoxError), + /// The stale-recovery classifier panicked; recovery was denied. + RecoveryPredicateFailed(BoxError), + /// Decoding the retained stale frame failed during recovery; the source + /// error was returned. Default formatting omits value-bearing error text; + /// custom loggers can inspect the original error here. + RecoveryDecodeFailed(BoxError), + /// A shadow job's clean-miss fill failed while preparing or writing the payload. + ShadowFillFailed(BoxError), + /// A confirmed shadow mismatch, logged only when the resolved policy's + /// `log_mismatches` is true. + ShadowMismatch(ShadowMismatchDetails), + /// An explicit invalidation failed; the same error is returned to the caller. + InvalidationFailed(BoxError), +} + +impl LogEvent { + /// [`LogLevel::Error`] for key construction and process-local read + /// failures, [`LogLevel::Warn`] for everything else. + pub fn level(&self) -> LogLevel { + match self { + LogEvent::KeyConstructionFailed(_) | LogEvent::LocalReadFailed(_) => LogLevel::Error, + _ => LogLevel::Warn, + } + } +} + +impl fmt::Display for LogEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + LogEvent::KeyConstructionFailed(e) => { + write!(f, "Could not construct DialCache key: {e}") + } + LogEvent::PolicyResolutionFailed(e) => { + write!(f, "Could not resolve DialCache key config: {e}") + } + LogEvent::LocalReadFailed(e) => write!(f, "Error getting value from local cache: {e}"), + LogEvent::LocalWriteFailed(e) => write!(f, "Error putting value in local cache: {e}"), + LogEvent::RemoteReadFailed(e) => write!(f, "Error getting value from Redis cache: {e}"), + LogEvent::RemoteWriteFailed(e) => write!(f, "Error putting value in Redis cache: {e}"), + LogEvent::RecoveryPredicateFailed(e) => { + write!( + f, + "DialCache stale recovery predicate failed; recovery was denied: {e}" + ) + } + LogEvent::RecoveryDecodeFailed(e) => { + write!( + f, + "Error using retained Redis value during stale recovery: " + )?; + if let Some(json) = e.downcast_ref::() { + write!( + f, + "JSON {:?} at line {} column {}", + json.classify(), + json.line(), + json.column() + ) + } else { + write!(f, "decoding failed") + } + } + LogEvent::ShadowFillFailed(e) => { + write!(f, "Error populating Redis from DialCache shadow work: {e}") + } + LogEvent::ShadowMismatch(d) => { + write!(f, "DialCache shadow validation mismatch: namespace={} useCase={} keyType={} cacheKey={}", + d.namespace, d.use_case, d.key_type, d.cache_key)?; + if let Some(value) = &d.cached_value_json { + write!(f, " cachedValue={value}")?; + } + if let Some(value) = &d.source_value_json { + write!(f, " sourceValue={value}")?; + } + Ok(()) + } + LogEvent::InvalidationFailed(e) => { + write!(f, "Error writing DialCache invalidation watermark: {e}") + } + } + } +} + +/// Receives structured log events. Panics are isolated from cache results. +pub trait Logger: Send + Sync + 'static { + /// Receive one event. Called inline under panic isolation; a panic here + /// is swallowed and the cache result is unaffected. + fn log(&self, event: &LogEvent); +} + +/// The default logger: the `log` crate facade. +#[derive(Debug, Clone, Copy, Default)] +pub struct LogFacadeLogger; + +impl Logger for LogFacadeLogger { + fn log(&self, event: &LogEvent) { + match event.level() { + LogLevel::Debug => log::debug!(target: "dialcache", "{event}"), + LogLevel::Warn => log::warn!(target: "dialcache", "{event}"), + LogLevel::Error => log::error!(target: "dialcache", "{event}"), + } + } +} diff --git a/rust/src/operation.rs b/rust/src/operation.rs new file mode 100644 index 00000000..4e09a280 --- /dev/null +++ b/rust/src/operation.rs @@ -0,0 +1,266 @@ +//! Operations: the typed description of one cached call and its erased form. + +use std::any::TypeId; +use std::sync::Arc; + +use futures::future::BoxFuture; +use serde::de::DeserializeOwned; +use serde::Serialize; + +use crate::codec::{Codec, JsonCodec, Payload}; +use crate::error::{BoxError, Error}; +use crate::identity::Identity; +use crate::limits::DEFAULT_FALLBACK_TIMEOUT_MS; +use crate::local::StoredValue; +use crate::policy::Policy; +use crate::scope::Scope; + +/// The source deadline of one operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum SourceBudget { + /// The library default, 60,000 ms. + #[default] + Default, + /// No deadline: the caller waits for the source however long it takes. + /// Shadow work still uses the default budget. + Unbounded, + /// A deadline in whole milliseconds, `1..=2_147_483_647`. + Millis(u64), +} + +impl SourceBudget { + pub(crate) fn millis(self) -> Option { + match self { + SourceBudget::Default => Some(DEFAULT_FALLBACK_TIMEOUT_MS), + SourceBudget::Unbounded => None, + SourceBudget::Millis(ms) => Some(ms), + } + } +} + +/// Decides whether a source failure may be answered from the stale value +/// retained by the initial remote read. +pub type RecoveryPredicate = Arc bool + Send + Sync>; +/// Application equality used by shadow validation. +pub type Comparator = Arc bool + Send + Sync>; +/// Produces a bounded textual preview of a value for mismatch warnings. +/// Runs through [`Runtime::spawn_blocking`](crate::Runtime::spawn_blocking) +/// after mismatch confirmation; callbacks may run on a CPU worker thread. +pub type Preview = Arc Option + Send + Sync>; + +/// One inline cached call: its identity, static policy, budget and codecs. +/// +/// Use it directly with +/// [`DialCache::get_or_load`](crate::DialCache::get_or_load) when a stable use +/// case is declared at the call site instead of being registered. +pub struct Operation { + /// The logical identity; an empty namespace inherits the instance's. + pub identity: Identity, + /// Static caching policy, validated before execution. + pub policy: Policy, + /// The source deadline. + pub budget: SourceBudget, + /// Encodes and decodes values for the remote layer. + pub codec: Arc>, + /// Application equality for shadow validation. + pub comparator: Comparator, + /// Per-operation stale-recovery classifier. `None` defers to the + /// instance's, which by default admits only the source deadline error. + pub should_recover: Option, + /// Renders bounded value previews for mismatch warnings; `None` logs none. + pub preview: Option>, +} + +impl Clone for Operation { + fn clone(&self) -> Self { + Self { + identity: self.identity.clone(), + policy: self.policy.clone(), + budget: self.budget, + codec: self.codec.clone(), + comparator: self.comparator.clone(), + should_recover: self.should_recover.clone(), + preview: self.preview.clone(), + } + } +} + +impl std::fmt::Debug for Operation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Operation") + .field("identity", &self.identity) + .field("policy", &self.policy) + .field("budget", &self.budget) + .finish_non_exhaustive() + } +} + +impl Operation { + /// An operation using the JSON codec, `PartialEq` shadow comparison and + /// JSON mismatch previews. + pub fn new(identity: Identity) -> Self { + Operation { + identity, + policy: Policy::default(), + budget: SourceBudget::Default, + codec: Arc::new(JsonCodec), + comparator: Arc::new(|a: &T, b: &T| a == b), + should_recover: None, + preview: Some(Arc::new(crate::preview::json_preview::)), + } + } +} + +impl Operation { + /// An operation with an explicit codec and comparator, for values that + /// are not `serde` JSON values. Mismatch previews are absent unless set. + pub fn with_codec( + identity: Identity, + codec: Arc>, + comparator: impl Fn(&T, &T) -> bool + Send + Sync + 'static, + ) -> Self { + Operation { + identity, + policy: Policy::default(), + budget: SourceBudget::Default, + codec, + comparator: Arc::new(comparator), + should_recover: None, + preview: None, + } + } + + /// Replace the static policy. + pub fn policy(mut self, policy: Policy) -> Self { + self.policy = policy; + self + } + + /// Replace the source deadline. + pub fn budget(mut self, budget: SourceBudget) -> Self { + self.budget = budget; + self + } + + /// Set the stale-recovery classifier for this operation, overriding the + /// instance's. + pub fn should_recover( + mut self, + predicate: impl Fn(&Error) -> bool + Send + Sync + 'static, + ) -> Self { + self.should_recover = Some(Arc::new(predicate)); + self + } + + /// Set the mismatch-warning preview; `None` from the closure omits the value. + pub fn preview( + mut self, + preview: impl Fn(&T) -> Option + Send + Sync + 'static, + ) -> Self { + self.preview = Some(Arc::new(preview)); + self + } + + pub(crate) fn erase(self) -> (Identity, Arc) { + let codec = self.codec; + let comparator = self.comparator; + let preview = self.preview; + let metadata = OperationMetadata { + value_type: TypeId::of::(), + policy: self.policy, + budget: self.budget, + codec: Arc::new(TypedCodec { codec }), + compare: Arc::new(move |a: &StoredValue, b: &StoredValue| { + let (Some(a), Some(b)) = (a.downcast_ref::(), b.downcast_ref::()) else { + return Err("shadow comparison received a value of another type".into()); + }; + Ok(comparator(a, b)) + }), + should_recover: self.should_recover, + preview: preview.map(|preview| { + Arc::new(move |value: &StoredValue| { + value.downcast_ref::().and_then(|v| preview(v)) + }) as Arc Option + Send + Sync> + }), + }; + (self.identity, Arc::new(metadata)) + } +} + +/// Source loader over erased values. +pub(crate) type ErasedLoad = + Arc BoxFuture<'static, Result> + Send + Sync>; +pub(crate) type ErasedCompare = + Arc Result + Send + Sync>; +pub(crate) type ErasedPreview = Arc Option + Send + Sync>; +pub(crate) type IdentityProvider = Arc Result + Send + Sync>; + +/// Immutable adapters and policy, shared by every registered invocation. +pub(crate) struct OperationMetadata { + pub value_type: TypeId, + pub policy: Policy, + pub budget: SourceBudget, + pub codec: Arc, + pub compare: ErasedCompare, + pub should_recover: Option, + pub preview: Option, +} + +/// The type-erased operation the engine executes. +pub(crate) struct ErasedOperation { + pub identity: Identity, + pub identity_provider: Option, + pub metadata: Arc, + pub load: ErasedLoad, +} + +/// A codec over erased values. +pub(crate) trait ErasedCodec: Send + Sync { + fn encode(&self, value: StoredValue) -> BoxFuture<'_, Result>; + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result>; +} + +struct TypedCodec { + codec: Arc>, +} + +impl ErasedCodec for TypedCodec { + fn encode(&self, value: StoredValue) -> BoxFuture<'_, Result> { + match Arc::downcast::(value) { + Ok(value) => self.codec.encode_owned(value), + Err(_) => Box::pin(std::future::ready(Err( + "codec received a value of another type".into(), + ))), + } + } + + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result> { + let decoding = self.codec.decode(payload); + Box::pin(async move { decoding.await.map(|value| Arc::new(value) as StoredValue) }) + } +} + +/// Erase a typed source closure. +pub(crate) fn erase_load(load: F) -> ErasedLoad +where + T: Send + Sync + 'static, + F: Fn(Scope) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + Send + 'static, +{ + Arc::new(move |scope: Scope| { + let future = load(scope); + Box::pin(async move { future.await.map(|value| Arc::new(value) as StoredValue) }) + }) +} + +/// Downcast an erased value to its typed form. +pub(crate) fn downcast_value( + value: StoredValue, +) -> Result, Error> { + Arc::downcast::(value).map_err(|_| { + Error::Config(crate::error::ConfigError::invalid(format!( + "cached value is not a {}", + std::any::type_name::() + ))) + }) +} diff --git a/rust/src/policy.rs b/rust/src/policy.rs new file mode 100644 index 00000000..78c890a7 --- /dev/null +++ b/rust/src/policy.rs @@ -0,0 +1,1607 @@ +//! Static operation policy, sparse runtime overlays and per-invocation resolution. + +use serde_json::{Map, Value}; + +use crate::limits::{DEFAULT_REMOTE_READ_TIMEOUT_MS, MAX_CACHE_TTL_SEC, MAX_DEADLINE_MS}; +use crate::observe::DisabledReason; + +/// Per-use-case shadow validation policy. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ShadowPolicy { + /// Independent stable cohort percentage in `[0, 100]`. Omitted and zero disable shadow work. + pub ramp: Option, + /// Emit one warning per confirmed mismatch. Defaults to false. + pub log_mismatches: Option, +} + +/// Static caching policy of one operation. Omitted leaves use library +/// defaults or are overridden by a runtime overlay. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Policy { + /// Memoize values for the outermost enabled scope. Defaults to false. + pub request_local: Option, + /// Share one execution across concurrent same-key callers. Defaults to true. + pub coalesce: Option, + /// Process-local TTL in whole seconds, `1..=MAX_CACHE_TTL_SEC`. Omitted disables the layer. + pub local_ttl_sec: Option, + /// Remote TTL in whole seconds. Omitted disables the layer. + pub remote_ttl_sec: Option, + /// Local serving cohort percentage; a configured TTL implies 100. + pub local_ramp: Option, + /// Remote serving cohort percentage; a configured TTL implies 100. + pub remote_ramp: Option, + /// Exclusive stale-recovery age ceiling in seconds; must exceed the remote TTL. Zero disables. + pub stale_on_error_max_age_sec: Option, + /// Remote read budget in milliseconds, `1..=MAX_DEADLINE_MS`. + pub remote_read_timeout_ms: Option, + /// Shadow validation cohort and warning policy. Omitted means no shadow work. + pub shadow: Option, +} + +impl Policy { + /// Both serving layers enabled with the same TTL and full ramps. + pub fn enabled(ttl_sec: u64) -> Self { + Policy { + local_ttl_sec: Some(ttl_sec), + remote_ttl_sec: Some(ttl_sec), + local_ramp: Some(100.0), + remote_ramp: Some(100.0), + ..Policy::default() + } + } + + /// The explicit kill switch: request memoization, recovery and shadow + /// work off, both serving layers ramped to zero. As an overlay it disables + /// every inherited path instead of relying on omission. + pub fn disabled() -> Self { + Policy { + request_local: Some(false), + stale_on_error_max_age_sec: Some(0), + local_ramp: Some(0.0), + remote_ramp: Some(0.0), + shadow: Some(ShadowPolicy { + ramp: Some(0.0), + log_mismatches: Some(false), + }), + ..Policy::default() + } + } + + /// Set whether values are memoized for the outermost enabled scope. + pub fn request_local(mut self, enabled: bool) -> Self { + self.request_local = Some(enabled); + self + } + + /// Set whether concurrent same-key callers share one execution. + pub fn coalesce(mut self, enabled: bool) -> Self { + self.coalesce = Some(enabled); + self + } + + /// Set the process-local TTL in whole seconds, enabling the layer with + /// an implied full ramp. + pub fn local_ttl_sec(mut self, ttl_sec: u64) -> Self { + self.local_ttl_sec = Some(ttl_sec); + self + } + + /// Set the remote TTL in whole seconds, enabling the layer with an + /// implied full ramp. + pub fn remote_ttl_sec(mut self, ttl_sec: u64) -> Self { + self.remote_ttl_sec = Some(ttl_sec); + self + } + + /// Set the process-local serving cohort percentage, `0.0..=100.0`. + pub fn local_ramp(mut self, ramp: f64) -> Self { + self.local_ramp = Some(ramp); + self + } + + /// Set the remote serving cohort percentage, `0.0..=100.0`. + pub fn remote_ramp(mut self, ramp: f64) -> Self { + self.remote_ramp = Some(ramp); + self + } + + /// Set the stale-recovery age ceiling in seconds. Zero disables + /// recovery; a positive value must exceed the remote TTL. + pub fn stale_on_error_max_age_sec(mut self, seconds: u64) -> Self { + self.stale_on_error_max_age_sec = Some(seconds); + self + } + + /// Set the remote read budget in milliseconds, overriding the instance default. + pub fn remote_read_timeout_ms(mut self, ms: u64) -> Self { + self.remote_read_timeout_ms = Some(ms); + self + } + + /// Set the shadow validation policy. + pub fn shadow(mut self, shadow: ShadowPolicy) -> Self { + self.shadow = Some(shadow); + self + } + + /// Parse the JSON-shaped TypeScript configuration + /// (`ttlSec`, `ramp`, `shadow`, `requestLocal`, `coalesce`, + /// `staleOnErrorMaxAgeSec`, `remoteReadTimeoutMs`). `null` or absent means an empty policy. + /// + /// A present leaf holding `null` is an invalid supplied value, not an + /// omission: JSON has no way to spell JavaScript's `undefined`, so every + /// member of an object counts as present. + pub fn from_json(value: &Value) -> Result { + let mut policy = Policy::default(); + if value.is_null() { + return Ok(policy); + } + let config = policy_object(value, "defaultConfig")?; + reject_shadow_ramp(config)?; + if let Some(layers) = config.get("ttlSec") { + let layers = policy_object(layers, "ttlSec")?; + for (layer, slot) in [ + ("local", &mut policy.local_ttl_sec), + ("remote", &mut policy.remote_ttl_sec), + ] { + if let Some(leaf) = layers.get(layer) { + let seconds = ttl_sec(leaf) + .ok_or_else(|| PolicyError(format!("invalid static ttlSec.{layer}")))?; + *slot = Some(seconds); + } + } + } + if let Some(layers) = config.get("ramp") { + let layers = policy_object(layers, "ramp")?; + for (layer, slot) in [ + ("local", &mut policy.local_ramp), + ("remote", &mut policy.remote_ramp), + ] { + if let Some(leaf) = layers.get(layer) { + let ramp = finite_range(leaf, 0.0, 100.0, false) + .ok_or_else(|| PolicyError(format!("invalid static ramp.{layer}")))?; + *slot = Some(ramp); + } + } + } + for (field, slot) in [ + ("requestLocal", &mut policy.request_local), + ("coalesce", &mut policy.coalesce), + ] { + if let Some(leaf) = config.get(field) { + let flag = leaf + .as_bool() + .ok_or_else(|| PolicyError(format!("{field} must be boolean")))?; + *slot = Some(flag); + } + } + if let Some(leaf) = config.get("staleOnErrorMaxAgeSec") { + let seconds = finite_range(leaf, 0.0, MAX_CACHE_TTL_SEC as f64, true) + .ok_or_else(|| PolicyError("invalid static staleOnErrorMaxAgeSec".to_string()))?; + policy.stale_on_error_max_age_sec = Some(seconds as u64); + } + if let Some(leaf) = config.get("remoteReadTimeoutMs") { + let ms = finite_range(leaf, 1.0, MAX_DEADLINE_MS as f64, true) + .ok_or_else(|| PolicyError("invalid remoteReadTimeoutMs".to_string()))?; + policy.remote_read_timeout_ms = Some(ms as u64); + } + if let Some(shadow) = config.get("shadow") { + let shadow = policy_object(shadow, "shadow")?; + let mut parsed = ShadowPolicy::default(); + if let Some(leaf) = shadow.get("ramp") { + let ramp = finite_range(leaf, 0.0, 100.0, false) + .ok_or_else(|| PolicyError("invalid static shadow.ramp".to_string()))?; + parsed.ramp = Some(ramp); + } + if let Some(leaf) = shadow.get("logMismatches") { + let flag = leaf.as_bool().ok_or_else(|| { + PolicyError("shadow.logMismatches must be boolean".to_string()) + })?; + parsed.log_mismatches = Some(flag); + } + policy.shadow = Some(parsed); + } + policy.validate()?; + Ok(policy) + } + + /// The JSON-shaped form of this policy, with omitted leaves absent. + /// + /// `ttlSec` and `ramp` are always objects (possibly empty). All other + /// leaves, including `requestLocal` and `coalesce`, appear only when set, + /// so converting this policy to a runtime overlay preserves inheritance. + pub fn to_json(&self) -> Value { + let mut ttl = Map::new(); + if let Some(seconds) = self.local_ttl_sec { + ttl.insert("local".to_string(), Value::from(seconds)); + } + if let Some(seconds) = self.remote_ttl_sec { + ttl.insert("remote".to_string(), Value::from(seconds)); + } + let mut ramp = Map::new(); + if let Some(value) = self.local_ramp { + ramp.insert("local".to_string(), ramp_json(value)); + } + if let Some(value) = self.remote_ramp { + ramp.insert("remote".to_string(), ramp_json(value)); + } + let mut object = Map::new(); + object.insert("ttlSec".to_string(), Value::Object(ttl)); + object.insert("ramp".to_string(), Value::Object(ramp)); + if let Some(enabled) = self.request_local { + object.insert("requestLocal".to_string(), Value::Bool(enabled)); + } + if let Some(enabled) = self.coalesce { + object.insert("coalesce".to_string(), Value::Bool(enabled)); + } + if let Some(seconds) = self.stale_on_error_max_age_sec { + object.insert("staleOnErrorMaxAgeSec".to_string(), Value::from(seconds)); + } + if let Some(ms) = self.remote_read_timeout_ms { + object.insert("remoteReadTimeoutMs".to_string(), Value::from(ms)); + } + if let Some(shadow) = &self.shadow { + let mut leaves = Map::new(); + if let Some(value) = shadow.ramp { + leaves.insert("ramp".to_string(), ramp_json(value)); + } + if let Some(flag) = shadow.log_mismatches { + leaves.insert("logMismatches".to_string(), Value::Bool(flag)); + } + object.insert("shadow".to_string(), Value::Object(leaves)); + } + Value::Object(object) + } + + /// Reject statically invalid leaves. + pub fn validate(&self) -> Result<(), PolicyError> { + for ttl in [self.local_ttl_sec, self.remote_ttl_sec] + .into_iter() + .flatten() + { + if !(1..=MAX_CACHE_TTL_SEC).contains(&ttl) { + return Err(PolicyError( + "static TTL must be whole seconds within 365 days".to_string(), + )); + } + } + for ramp in [self.local_ramp, self.remote_ramp].into_iter().flatten() { + if !ramp_in_domain(ramp) { + return Err(PolicyError( + "static ramp must be between zero and 100".to_string(), + )); + } + } + if let Some(age) = self.stale_on_error_max_age_sec { + let exceeds_remote = match self.remote_ttl_sec { + Some(remote) if remote > 0 => age > remote, + _ => false, + }; + if age > MAX_CACHE_TTL_SEC || (age > 0 && !exceeds_remote) { + return Err(PolicyError( + "static recovery age must exceed a positive remote TTL".to_string(), + )); + } + } + if let Some(ms) = self.remote_read_timeout_ms { + if !(1..=MAX_DEADLINE_MS).contains(&ms) { + return Err(PolicyError("invalid remote read deadline".to_string())); + } + } + if let Some(ramp) = self.shadow.as_ref().and_then(|shadow| shadow.ramp) { + if !ramp_in_domain(ramp) { + return Err(PolicyError( + "static shadow ramp must be between zero and 100".to_string(), + )); + } + } + Ok(()) + } +} + +/// A sparse runtime overlay returned by a [`PolicyProvider`](crate::PolicyProvider). +/// +/// Each present leaf replaces the operation leaf; omitted leaves inherit. +/// Leaves may hold invalid values on purpose: their consequences are defined +/// by the portable contract (an invalid TTL or ramp disables only that +/// layer, an invalid flag or read deadline bypasses caching for the call). +#[derive(Debug, Clone, PartialEq)] +pub struct RuntimePolicy(pub Value); + +impl RuntimePolicy { + /// Wrap a JSON overlay as received from a configuration source. Nothing + /// is validated here; resolution judges every leaf per call. + pub fn from_json(value: Value) -> Self { + RuntimePolicy(value) + } +} + +impl From for RuntimePolicy { + fn from(policy: Policy) -> Self { + RuntimePolicy(policy.to_json()) + } +} + +impl From for RuntimePolicy { + fn from(value: Value) -> Self { + RuntimePolicy(value) + } +} + +/// Instance defaults consulted during resolution. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PolicyDefaults { + /// Instance remote read budget in milliseconds, used when neither the + /// operation nor the overlay sets one. Zero selects the library default + /// of 50 ms. + pub remote_read_timeout_ms: u64, +} + +impl Default for PolicyDefaults { + fn default() -> Self { + PolicyDefaults { + remote_read_timeout_ms: DEFAULT_REMOTE_READ_TIMEOUT_MS, + } + } +} + +/// Why a whole invocation's policy could not be resolved. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{0}")] +pub struct PolicyError(pub String); + +/// One serving layer after resolution. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ResolvedLayer { + /// Whether the layer serves this key: configured and admitted by its ramp. + pub enabled: bool, + /// Why the layer is disabled, when it is. + pub reason: Option, + /// A valid TTL and ramp remain available when the ramp excluded this key. + pub configured: bool, + /// The effective TTL in milliseconds; zero when not configured. + pub ttl_ms: u64, + /// The effective cohort percentage; zero when not configured. + pub ramp: f64, +} + +impl ResolvedLayer { + fn disabled(reason: DisabledReason) -> Self { + ResolvedLayer { + enabled: false, + reason: Some(reason), + configured: false, + ttl_ms: 0, + ramp: 0.0, + } + } +} + +/// Shadow policy after resolution. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct ResolvedShadow { + /// Cohort selection only; admission still needs an eligible path, hook and capacity. + pub enabled: bool, + /// The shadow cohort percentage; zero when omitted or invalid. + pub ramp: f64, + /// Whether a confirmed mismatch logs a warning; `false` when the flag + /// was omitted or malformed. + pub log_mismatches: bool, + /// The shadow ramp leaf was present but invalid: shadow work is off and a + /// `config_resolution` error is emitted on the remote layer whenever a + /// shadow job would otherwise have been scheduled (a remote hit or a + /// ramped-down remote). + pub config_error: bool, + /// Recorded only if a job is admitted. + pub logging_config_error: bool, +} + +/// The captured policy of one enabled invocation. +#[derive(Debug, Clone, PartialEq)] +pub struct ResolvedPolicy { + /// Whether the call memoizes in the outermost enabled scope. + pub request_local: bool, + /// Whether the call joins or leads a single flight for its key. + pub coalesce: bool, + /// The process-local layer. + pub local: ResolvedLayer, + /// The remote layer. + pub remote: ResolvedLayer, + /// The effective remote read budget in milliseconds: overlay, then + /// operation, then instance default. + pub remote_read_timeout_ms: u64, + /// Zero when recovery is off. + pub stale_on_error_max_age_ms: u64, + /// The recovery age leaf was present but invalid (not whole seconds, or + /// not above the remote TTL): recovery is off and a `config_resolution` + /// error is emitted on the remote layer. + pub stale_on_error_config_error: bool, + /// The shadow validation policy. + pub shadow: ResolvedShadow, +} + +/// Merge a static policy with a sparse overlay once, for one logical key. +/// +/// A malformed container, boolean or read deadline fails the whole +/// invocation; TTL or ramp errors disable only that layer; an invalid +/// recovery or shadow option preserves ordinary serving. +pub fn resolve_policy( + base: &Policy, + overlay: Option<&RuntimePolicy>, + logical_key: &str, + defaults: PolicyDefaults, +) -> Result { + let mut remote_read_timeout_ms = defaults.remote_read_timeout_ms; + if remote_read_timeout_ms == 0 { + remote_read_timeout_ms = DEFAULT_REMOTE_READ_TIMEOUT_MS; + } + if remote_read_timeout_ms > MAX_DEADLINE_MS { + return Err(PolicyError("invalid instance read deadline".to_string())); + } + base.validate()?; + let mut merged = match base.to_json() { + Value::Object(object) => object, + _ => unreachable!("Policy::to_json always produces an object"), + }; + // Defaults belong to resolution, not the sparse policy representation. + // Present overlay values (including invalid nulls) still replace them. + merged.entry("requestLocal").or_insert(Value::Bool(false)); + merged.entry("coalesce").or_insert(Value::Bool(true)); + if let Some(RuntimePolicy(overlay)) = overlay { + if !overlay.is_null() { + merge_overlay(&mut merged, overlay)?; + } + } + + let request_local = merged + .get("requestLocal") + .and_then(Value::as_bool) + .ok_or_else(|| PolicyError("runtime requestLocal must be boolean".to_string()))?; + let coalesce = merged + .get("coalesce") + .and_then(Value::as_bool) + .ok_or_else(|| PolicyError("runtime coalesce must be boolean".to_string()))?; + if let Some(leaf) = merged.get("remoteReadTimeoutMs") { + let ms = finite_range(leaf, 1.0, MAX_DEADLINE_MS as f64, true) + .ok_or_else(|| PolicyError("invalid runtime remoteReadTimeoutMs".to_string()))?; + remote_read_timeout_ms = ms as u64; + } + + let empty = Map::new(); + let ttls = merged + .get("ttlSec") + .and_then(Value::as_object) + .unwrap_or(&empty); + let ramps = merged + .get("ramp") + .and_then(Value::as_object) + .unwrap_or(&empty); + let local = resolve_layer(ttls, ramps, logical_key, "local"); + let remote = resolve_layer(ttls, ramps, logical_key, "remote"); + + let mut stale_on_error_max_age_ms = 0; + let mut stale_on_error_config_error = false; + if let Some(age) = merged.get("staleOnErrorMaxAgeSec") { + let numeric_zero = policy_number(age) == Some(0.0); + if !remote.configured { + stale_on_error_config_error = + remote.reason == Some(DisabledReason::PolicyDisabled) && !numeric_zero; + } else if !numeric_zero { + match ttl_sec(age).map(|seconds| seconds * 1_000) { + Some(ms) if ms > remote.ttl_ms => stale_on_error_max_age_ms = ms, + _ => stale_on_error_config_error = true, + } + } + } + + let mut shadow = ResolvedShadow::default(); + if let Some(leaves) = merged.get("shadow").and_then(Value::as_object) { + if let Some(leaf) = leaves.get("ramp") { + match finite_range(leaf, 0.0, 100.0, false) { + Some(ramp) => { + shadow.ramp = ramp; + shadow.enabled = admits(ramp, logical_key, "shadow"); + } + None => shadow.config_error = true, + } + } + if let Some(leaf) = leaves.get("logMismatches") { + match leaf.as_bool() { + Some(flag) => shadow.log_mismatches = flag, + None => { + shadow.log_mismatches = false; + shadow.logging_config_error = true; + } + } + } + } + + Ok(ResolvedPolicy { + request_local, + coalesce, + local, + remote, + remote_read_timeout_ms, + stale_on_error_max_age_ms, + stale_on_error_config_error, + shadow, + }) +} + +/// Apply a sparse runtime overlay to the static policy's JSON form. Every +/// present member counts, including `null`, so an explicit null leaf replaces +/// the operation leaf and is then judged as an invalid supplied value. +fn merge_overlay(merged: &mut Map, overlay: &Value) -> Result<(), PolicyError> { + let overlay = policy_object(overlay, "runtime config")?; + reject_shadow_ramp(overlay)?; + for (name, leaves) in [ + ("ttlSec", &["local", "remote"][..]), + ("ramp", &["local", "remote"][..]), + ("shadow", &["ramp", "logMismatches"][..]), + ] { + let Some(incoming) = overlay.get(name) else { + continue; + }; + let incoming = policy_object(incoming, name)?; + let mut output = match merged.get(name) { + Some(Value::Object(existing)) => existing.clone(), + _ => Map::new(), + }; + for leaf in leaves { + if let Some(value) = incoming.get(*leaf) { + output.insert((*leaf).to_string(), value.clone()); + } + } + merged.insert(name.to_string(), Value::Object(output)); + } + for name in [ + "requestLocal", + "coalesce", + "staleOnErrorMaxAgeSec", + "remoteReadTimeoutMs", + ] { + if let Some(value) = overlay.get(name) { + merged.insert(name.to_string(), value.clone()); + } + } + Ok(()) +} + +fn resolve_layer( + ttls: &Map, + ramps: &Map, + logical_key: &str, + layer: &str, +) -> ResolvedLayer { + let Some(ttl) = ttls.get(layer) else { + return ResolvedLayer::disabled(DisabledReason::PolicyDisabled); + }; + let Some(ttl_ms) = ttl_sec(ttl).map(|seconds| seconds * 1_000) else { + return ResolvedLayer::disabled(DisabledReason::InvalidTtl); + }; + let ramp = match ramps.get(layer) { + None => 100.0, + Some(leaf) => match finite_range(leaf, 0.0, 100.0, false) { + Some(ramp) => ramp, + None => return ResolvedLayer::disabled(DisabledReason::InvalidRamp), + }, + }; + let enabled = admits(ramp, logical_key, layer); + ResolvedLayer { + enabled, + reason: if enabled { + None + } else { + Some(DisabledReason::RampedDown) + }, + configured: true, + ttl_ms, + ramp, + } +} + +/// Cohort admission: a full ramp admits every key, a zero ramp none, and a +/// partial ramp admits keys whose stable sample is strictly below it. +fn admits(ramp: f64, logical_key: &str, discriminator: &str) -> bool { + ramp >= 100.0 || (ramp > 0.0 && crate::identity::cohort(logical_key, discriminator) < ramp) +} + +fn ramp_in_domain(ramp: f64) -> bool { + ramp.is_finite() && (0.0..=100.0).contains(&ramp) +} + +/// Spell a ramp the way JavaScript and Go's `encoding/json` do: an integral +/// value has no fraction (`100`, not `100.0`), so a static policy's JSON form +/// compares equal to the same policy parsed from text. +fn ramp_json(ramp: f64) -> Value { + if ramp_in_domain(ramp) && ramp.trunc() == ramp { + Value::from(ramp as u64) + } else { + Value::from(ramp) + } +} + +/// The numeric domain: JSON numbers only. Strings and booleans never count. +fn policy_number(value: &Value) -> Option { + value.as_number().and_then(serde_json::Number::as_f64) +} + +fn finite_range(value: &Value, min: f64, max: f64, integer: bool) -> Option { + let n = policy_number(value)?; + (n.is_finite() && n >= min && n <= max && (!integer || n.trunc() == n)).then_some(n) +} + +/// A TTL-domain leaf in whole seconds. +fn ttl_sec(value: &Value) -> Option { + finite_range(value, 1.0, MAX_CACHE_TTL_SEC as f64, true).map(|n| n as u64) +} + +fn policy_object<'a>(value: &'a Value, name: &str) -> Result<&'a Map, PolicyError> { + value + .as_object() + .ok_or_else(|| PolicyError(format!("DialCache {name} must be an object"))) +} + +fn reject_shadow_ramp(object: &Map) -> Result<(), PolicyError> { + if object.contains_key("shadowRamp") { + return Err(PolicyError( + "shadowRamp was replaced by shadow.ramp".to_string(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const KEY: &str = "policy:item:one#lookup"; + const MAX_TTL: u64 = MAX_CACHE_TTL_SEC; + + fn resolve(base: &Policy, overlay: Option) -> Result { + let overlay = overlay.map(RuntimePolicy); + resolve_policy(base, overlay.as_ref(), KEY, PolicyDefaults::default()) + } + + fn resolved(base: &Policy, overlay: Option) -> ResolvedPolicy { + resolve(base, overlay).expect("policy resolves") + } + + // Port of TestPolicySparseResolutionAndSnapshots. + #[test] + fn sparse_resolution_and_inheritance() { + let base = Policy::from_json(&json!({ + "requestLocal": true, "coalesce": false, + "ttlSec": {"local": 1.0, "remote": 2.0}, + "staleOnErrorMaxAgeSec": 5.0, "remoteReadTimeoutMs": 30.0, + "shadow": {"ramp": 100.0, "logMismatches": true}, + })) + .unwrap(); + let overlay = RuntimePolicy(json!({ + "ttlSec": {"remote": 4.0}, + "ramp": {"local": 0.0}, + "shadow": {"logMismatches": false}, + })); + let r = resolve_policy( + &base, + Some(&overlay), + KEY, + PolicyDefaults { + remote_read_timeout_ms: 20, + }, + ) + .unwrap(); + assert!(r.request_local); + assert!(!r.coalesce); + assert!(!r.local.enabled); + assert_eq!(r.local.reason, Some(DisabledReason::RampedDown)); + assert!(r.local.configured); + assert_eq!(r.local.ttl_ms, 1_000); + assert_eq!(r.local.ramp, 0.0); + assert!(r.remote.enabled); + assert_eq!(r.remote.reason, None); + assert_eq!(r.remote.ttl_ms, 4_000); + assert_eq!(r.remote.ramp, 100.0); + assert_eq!(r.stale_on_error_max_age_ms, 5_000); + assert!(!r.stale_on_error_config_error); + assert_eq!(r.remote_read_timeout_ms, 30); + assert!(r.shadow.enabled); + assert_eq!(r.shadow.ramp, 100.0); + assert!(!r.shadow.log_mismatches); + assert!(!r.shadow.config_error); + assert!(!r.shadow.logging_config_error); + + // A clone is an independent snapshot of every leaf. + let snapshot = base.clone(); + let mut mutated = base.clone(); + mutated.remote_read_timeout_ms = Some(99); + mutated.shadow.as_mut().unwrap().ramp = Some(0.0); + assert_eq!(snapshot.remote_read_timeout_ms, Some(30)); + assert_eq!(snapshot.shadow.as_ref().unwrap().ramp, Some(100.0)); + + // A null provider reply inherits the whole operation policy. + for overlay in [None, Some(Value::Null)] { + let inherit = resolve_policy( + &snapshot, + overlay.map(RuntimePolicy).as_ref(), + KEY, + PolicyDefaults::default(), + ) + .unwrap(); + assert!(inherit.local.enabled); + assert!(inherit.remote.enabled); + assert_eq!(inherit.remote_read_timeout_ms, 30); + assert!(inherit.shadow.enabled); + assert!(inherit.shadow.log_mismatches); + } + } + + // Port of TestPolicyStaticValidation. + #[test] + fn static_validation_rejects_malformed_configs() { + let invalid = [ + json!(true), + json!([]), + json!("policy"), + json!(1), + json!({"shadowRamp": 1}), + json!({"ttlSec": null}), + json!({"ramp": null}), + json!({"shadow": null}), + json!({"ttlSec": []}), + json!({"ttlSec": 60}), + json!({"shadow": true}), + json!({"requestLocal": null}), + json!({"coalesce": null}), + json!({"coalesce": "false"}), + json!({"requestLocal": 1}), + json!({"ttlSec": {"local": null}}), + json!({"ttlSec": {"local": 0}}), + json!({"ttlSec": {"local": -1}}), + json!({"ttlSec": {"local": 1.5}}), + json!({"ttlSec": {"local": "60"}}), + json!({"ttlSec": {"remote": 31_536_001}}), + json!({"ramp": {"remote": null}}), + json!({"ramp": {"remote": "nan"}}), + json!({"ramp": {"remote": -0.5}}), + json!({"ramp": {"local": 100.5}}), + json!({"ramp": {"local": true}}), + json!({"remoteReadTimeoutMs": null}), + json!({"remoteReadTimeoutMs": 0}), + json!({"remoteReadTimeoutMs": 1.5}), + json!({"remoteReadTimeoutMs": 2_147_483_648u64}), + json!({"staleOnErrorMaxAgeSec": null}), + json!({"staleOnErrorMaxAgeSec": -1}), + json!({"staleOnErrorMaxAgeSec": 2.5}), + json!({"staleOnErrorMaxAgeSec": 2}), + json!({"ttlSec": {"remote": 2}, "staleOnErrorMaxAgeSec": 2}), + json!({"ttlSec": {"remote": 2}, "staleOnErrorMaxAgeSec": 1}), + json!({"ttlSec": {"local": 2}, "staleOnErrorMaxAgeSec": 3}), + json!({"ttlSec": {"remote": 2}, "staleOnErrorMaxAgeSec": 31_536_001}), + json!({"shadow": {"ramp": null}}), + json!({"shadow": {"ramp": 101}}), + json!({"shadow": {"ramp": "50"}}), + json!({"shadow": {"logMismatches": null}}), + json!({"shadow": {"logMismatches": 1}}), + ]; + for config in invalid { + assert!( + Policy::from_json(&config).is_err(), + "accepted invalid static config: {config}" + ); + } + + let accepted = Policy::from_json(&json!({ + "ttlSec": {"remote": 31_536_000}, + "remoteReadTimeoutMs": 2_147_483_647u64, + })) + .unwrap(); + assert_eq!(accepted.remote_ttl_sec, Some(MAX_TTL)); + assert_eq!(accepted.remote_read_timeout_ms, Some(MAX_DEADLINE_MS)); + assert_eq!(accepted.local_ttl_sec, None); + + // Zero recovery age is valid without a remote TTL; a positive one needs to exceed it. + assert!(Policy::from_json(&json!({"staleOnErrorMaxAgeSec": 0})).is_ok()); + assert!( + Policy::from_json(&json!({"ttlSec": {"remote": 2}, "staleOnErrorMaxAgeSec": 3})) + .is_ok() + ); + // Integer-valued floats and unsigned integers share one numeric domain. + let floats = + Policy::from_json(&json!({"ttlSec": {"local": 60.0}, "ramp": {"local": 50}})).unwrap(); + assert_eq!(floats.local_ttl_sec, Some(60)); + assert_eq!(floats.local_ramp, Some(50.0)); + // Ramp boundaries are inclusive. + assert!(Policy::from_json( + &json!({"ramp": {"local": 0, "remote": 100}, "shadow": {"ramp": 0}}) + ) + .is_ok()); + + // Direct validation mirrors the same domains without the parser. + assert!(Policy::default().local_ttl_sec(0).validate().is_err()); + assert!(Policy::default() + .local_ttl_sec(MAX_TTL + 1) + .validate() + .is_err()); + assert!(Policy::default().remote_ttl_sec(MAX_TTL).validate().is_ok()); + assert!(Policy::default().local_ramp(f64::NAN).validate().is_err()); + assert!(Policy::default() + .remote_ramp(f64::INFINITY) + .validate() + .is_err()); + assert!(Policy::default().remote_ramp(100.0001).validate().is_err()); + assert!(Policy::default().local_ramp(-0.1).validate().is_err()); + assert!(Policy::default() + .remote_read_timeout_ms(0) + .validate() + .is_err()); + assert!(Policy::default() + .remote_read_timeout_ms(MAX_DEADLINE_MS + 1) + .validate() + .is_err()); + assert!(Policy::default() + .remote_read_timeout_ms(MAX_DEADLINE_MS) + .validate() + .is_ok()); + assert!(Policy::default() + .stale_on_error_max_age_sec(1) + .validate() + .is_err()); + assert!(Policy::default() + .stale_on_error_max_age_sec(0) + .validate() + .is_ok()); + assert!(Policy::default() + .remote_ttl_sec(2) + .stale_on_error_max_age_sec(2) + .validate() + .is_err()); + assert!(Policy::default() + .remote_ttl_sec(2) + .stale_on_error_max_age_sec(3) + .validate() + .is_ok()); + assert!(Policy::default() + .remote_ttl_sec(2) + .stale_on_error_max_age_sec(MAX_TTL + 1) + .validate() + .is_err()); + let nan_shadow = ShadowPolicy { + ramp: Some(f64::NAN), + log_mismatches: None, + }; + assert!(Policy::default().shadow(nan_shadow).validate().is_err()); + let high_shadow = ShadowPolicy { + ramp: Some(100.5), + log_mismatches: None, + }; + assert!(Policy::default().shadow(high_shadow).validate().is_err()); + let flag_only = ShadowPolicy { + ramp: None, + log_mismatches: Some(true), + }; + assert!(Policy::default().shadow(flag_only).validate().is_ok()); + assert!(Policy::enabled(60).validate().is_ok()); + assert!(Policy::disabled().validate().is_ok()); + } + + #[test] + fn from_json_treats_absent_and_null_wholes_as_empty() { + assert_eq!(Policy::from_json(&Value::Null).unwrap(), Policy::default()); + assert_eq!(Policy::from_json(&json!({})).unwrap(), Policy::default()); + // Unknown members are ignored, as in the TypeScript constructor. + assert_eq!( + Policy::from_json(&json!({"unknown": 1})).unwrap(), + Policy::default() + ); + // Empty containers are valid and leave every leaf omitted. + let empty = Policy::from_json(&json!({"ttlSec": {}, "ramp": {}, "shadow": {}})).unwrap(); + assert_eq!(empty.local_ttl_sec, None); + assert_eq!(empty.shadow, Some(ShadowPolicy::default())); + } + + #[test] + fn from_json_round_trips_through_to_json() { + let source = json!({ + "ttlSec": {"local": 5, "remote": 60}, + "ramp": {"local": 12.5, "remote": 100}, + "requestLocal": true, + "coalesce": false, + "staleOnErrorMaxAgeSec": 120, + "remoteReadTimeoutMs": 25, + "shadow": {"ramp": 3.5, "logMismatches": true}, + }); + let policy = Policy::from_json(&source).unwrap(); + assert_eq!( + policy, + Policy { + request_local: Some(true), + coalesce: Some(false), + local_ttl_sec: Some(5), + remote_ttl_sec: Some(60), + local_ramp: Some(12.5), + remote_ramp: Some(100.0), + stale_on_error_max_age_sec: Some(120), + remote_read_timeout_ms: Some(25), + shadow: Some(ShadowPolicy { + ramp: Some(3.5), + log_mismatches: Some(true) + }), + } + ); + assert_eq!(policy.to_json(), source); + assert_eq!(Policy::from_json(&policy.to_json()).unwrap(), policy); + } + + #[test] + fn to_json_matches_the_static_policy_map_shape() { + // Empty policy: layer maps present but empty, all other leaves absent. + assert_eq!( + Policy::default().to_json(), + json!({"ttlSec": {}, "ramp": {}}) + ); + // Only present leaves appear inside the layer maps; integral ramps spell without a fraction. + assert_eq!( + Policy::default() + .local_ttl_sec(7) + .remote_ramp(40.0) + .to_json(), + json!({"ttlSec": {"local": 7}, "ramp": {"remote": 40}}) + ); + assert_eq!( + Policy::default().local_ramp(12.5).to_json(), + json!({"ttlSec": {}, "ramp": {"local": 12.5}}) + ); + // An explicit flag value is emitted even when it equals the default. + assert_eq!( + Policy::default() + .request_local(false) + .coalesce(true) + .to_json(), + json!({"ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true}) + ); + // A present but empty shadow policy is an empty object. + assert_eq!( + Policy::default().shadow(ShadowPolicy::default()).to_json(), + json!({"ttlSec": {}, "ramp": {}, "shadow": {}}) + ); + assert_eq!( + Policy::default() + .shadow(ShadowPolicy { + ramp: None, + log_mismatches: Some(false) + }) + .to_json(), + json!({ + "ttlSec": {}, "ramp": {}, + "shadow": {"logMismatches": false}, + }) + ); + // Zero recovery age is a present leaf, not an omission. + assert_eq!( + Policy::default() + .stale_on_error_max_age_sec(0) + .remote_read_timeout_ms(75) + .to_json(), + json!({ + "ttlSec": {}, "ramp": {}, + "staleOnErrorMaxAgeSec": 0, "remoteReadTimeoutMs": 75, + }) + ); + assert_eq!( + Policy::enabled(60).to_json(), + json!({ + "ttlSec": {"local": 60, "remote": 60}, + "ramp": {"local": 100, "remote": 100}, + }) + ); + assert_eq!( + Policy::disabled().to_json(), + json!({ + "ttlSec": {}, + "ramp": {"local": 0, "remote": 0}, + "requestLocal": false, + "staleOnErrorMaxAgeSec": 0, + "shadow": {"ramp": 0, "logMismatches": false}, + }) + ); + } + + #[test] + fn enabled_and_disabled_presets_resolve_as_documented() { + let enabled = resolved(&Policy::enabled(60), None); + assert!(!enabled.request_local); + assert!(enabled.coalesce); + assert!(enabled.local.enabled && enabled.remote.enabled); + assert_eq!( + (enabled.local.ttl_ms, enabled.remote.ttl_ms), + (60_000, 60_000) + ); + assert_eq!((enabled.local.ramp, enabled.remote.ramp), (100.0, 100.0)); + assert_eq!( + enabled.remote_read_timeout_ms, + DEFAULT_REMOTE_READ_TIMEOUT_MS + ); + assert_eq!(enabled.stale_on_error_max_age_ms, 0); + assert!(!enabled.stale_on_error_config_error); + assert_eq!(enabled.shadow, ResolvedShadow::default()); + + // Standing alone, the kill switch has no TTLs: both layers are simply not configured, + // and its zero recovery age is not a configuration error. + let alone = resolved(&Policy::disabled(), None); + assert_eq!(alone.local.reason, Some(DisabledReason::PolicyDisabled)); + assert_eq!(alone.remote.reason, Some(DisabledReason::PolicyDisabled)); + assert!(!alone.stale_on_error_config_error); + assert!(!alone.shadow.enabled && !alone.shadow.config_error); + assert!(!alone.shadow.log_mismatches && !alone.shadow.logging_config_error); + + // As an overlay it ramps every inherited path to zero instead of relying on omission. + let base = Policy::enabled(60) + .request_local(true) + .stale_on_error_max_age_sec(120) + .shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: Some(true), + }); + let killed = resolved(&base, Some(RuntimePolicy::from(Policy::disabled()).0)); + assert!(!killed.request_local); + assert!(killed.coalesce); + for layer in [killed.local, killed.remote] { + assert!(!layer.enabled); + assert_eq!(layer.reason, Some(DisabledReason::RampedDown)); + assert!(layer.configured); + assert_eq!(layer.ttl_ms, 60_000); + assert_eq!(layer.ramp, 0.0); + } + assert_eq!(killed.stale_on_error_max_age_ms, 0); + assert!(!killed.stale_on_error_config_error); + assert_eq!(killed.shadow, ResolvedShadow::default()); + } + + // Port of TestRuntimePolicyFailureScopes. + #[test] + fn runtime_failure_scopes() { + let base = Policy::default() + .request_local(true) + .local_ttl_sec(1) + .remote_ttl_sec(2); + for overlay in [ + json!(false), + json!([]), + json!("config"), + json!(0), + json!({"shadowRamp": 5}), + json!({"ttlSec": null}), + json!({"ttlSec": []}), + json!({"ttlSec": 5}), + json!({"ramp": null}), + json!({"shadow": null}), + json!({"shadow": "off"}), + json!({"requestLocal": null}), + json!({"requestLocal": "true"}), + json!({"coalesce": 1}), + json!({"coalesce": null}), + json!({"remoteReadTimeoutMs": 0}), + json!({"remoteReadTimeoutMs": null}), + json!({"remoteReadTimeoutMs": 1.5}), + json!({"remoteReadTimeoutMs": "30"}), + json!({"remoteReadTimeoutMs": 2_147_483_648u64}), + ] { + assert!( + resolve(&base, Some(overlay.clone())).is_err(), + "accepted invalid invocation policy: {overlay}" + ); + } + + struct Case { + overlay: Value, + local: Option, + remote: Option, + recovery_error: bool, + } + let invalid_ttl = Some(DisabledReason::InvalidTtl); + let invalid_ramp = Some(DisabledReason::InvalidRamp); + let cases = [ + Case { + overlay: json!({"ttlSec": {"local": null}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"local": "1"}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"local": 0}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"local": 1.5}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"local": 31_536_001}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"remote": true}}), + local: None, + remote: invalid_ramp, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"remote": null}}), + local: None, + remote: invalid_ramp, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"remote": 100.1}}), + local: None, + remote: invalid_ramp, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"local": -1}}), + local: invalid_ramp, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": null}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 1}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 2}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": "3"}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 2.5}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 0}), + local: None, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 0.0}), + local: None, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 3}), + local: None, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"remote": -1}, "staleOnErrorMaxAgeSec": -1}), + local: None, + remote: invalid_ttl, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"remote": "bad"}, "staleOnErrorMaxAgeSec": 5}), + local: None, + remote: invalid_ramp, + recovery_error: false, + }, + ]; + for case in cases { + let r = resolved(&base, Some(case.overlay.clone())); + assert!(r.request_local, "{}", case.overlay); + assert!(r.coalesce, "{}", case.overlay); + assert_eq!( + r.local.reason, case.local, + "local reason for {}", + case.overlay + ); + assert_eq!( + r.remote.reason, case.remote, + "remote reason for {}", + case.overlay + ); + assert_eq!(r.local.enabled, case.local.is_none(), "{}", case.overlay); + assert_eq!(r.remote.enabled, case.remote.is_none(), "{}", case.overlay); + assert_eq!( + r.stale_on_error_config_error, case.recovery_error, + "recovery for {}", + case.overlay + ); + if case.recovery_error { + assert_eq!(r.stale_on_error_max_age_ms, 0, "{}", case.overlay); + } + } + // A valid runtime age above the remote TTL enables recovery. + assert_eq!( + resolved(&base, Some(json!({"staleOnErrorMaxAgeSec": 3}))).stale_on_error_max_age_ms, + 3_000 + ); + + // Recovery configured without any remote TTL is a diagnostic error on a disabled layer. + let missing = resolved( + &Policy::default(), + Some(json!({"staleOnErrorMaxAgeSec": 3})), + ); + assert!(missing.stale_on_error_config_error); + assert_eq!(missing.remote.reason, Some(DisabledReason::PolicyDisabled)); + assert!(!missing.remote.configured); + assert_eq!(missing.stale_on_error_max_age_ms, 0); + // ... unless it is numeric zero. + let zero = resolved( + &Policy::default(), + Some(json!({"staleOnErrorMaxAgeSec": 0})), + ); + assert!(!zero.stale_on_error_config_error); + // A non-numeric leaf on a disabled remote layer is still an error. + let text = resolved( + &Policy::default(), + Some(json!({"staleOnErrorMaxAgeSec": "0"})), + ); + assert!(text.stale_on_error_config_error); + } + + // Port of TestRecoveryRetentionAndShadowDiagnosticsRemainIndependent. + #[test] + fn recovery_retention_and_shadow_diagnostics_remain_independent() { + let base = Policy::default() + .remote_ttl_sec(1) + .remote_ramp(0.0) + .stale_on_error_max_age_sec(86_400); + let r = resolved( + &base, + Some(json!({"shadow": {"ramp": 100, "logMismatches": "invalid"}})), + ); + assert!(!r.remote.enabled); + assert!(r.remote.configured); + assert_eq!(r.remote.reason, Some(DisabledReason::RampedDown)); + assert_eq!(r.stale_on_error_max_age_ms, 86_400_000); + assert!(!r.stale_on_error_config_error); + assert!(r.shadow.enabled); + assert_eq!(r.shadow.ramp, 100.0); + assert!(!r.shadow.config_error); + assert!(r.shadow.logging_config_error); + assert!(!r.shadow.log_mismatches); + + let r = resolved(&base, Some(json!({"shadow": {"ramp": null}}))); + assert!(r.shadow.config_error); + assert!(!r.shadow.enabled); + assert_eq!(r.shadow.ramp, 0.0); + assert!(r.remote.configured); + assert_eq!(r.stale_on_error_max_age_ms, 86_400_000); + + for invalid in [json!("50"), json!(-1), json!(100.5), json!(true)] { + let r = resolved(&base, Some(json!({"shadow": {"ramp": invalid}}))); + assert!(r.shadow.config_error, "{invalid}"); + assert!(!r.shadow.enabled, "{invalid}"); + } + // Zero shadow ramp is valid and silently off. + let off = resolved(&base, Some(json!({"shadow": {"ramp": 0}}))); + assert!(!off.shadow.enabled && !off.shadow.config_error); + // A null logging flag is malformed and disables warnings without touching the cohort. + let logging = resolved( + &base, + Some(json!({"shadow": {"ramp": 100, "logMismatches": null}})), + ); + assert!( + logging.shadow.enabled + && logging.shadow.logging_config_error + && !logging.shadow.log_mismatches + ); + } + + #[test] + fn overlay_semantics_follow_the_portable_table() { + let base = Policy::enabled(10) + .request_local(true) + .coalesce(false) + .stale_on_error_max_age_sec(30) + .remote_read_timeout_ms(40) + .shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: Some(true), + }); + + // Omitted leaves inherit the operation leaf; an empty overlay changes nothing. + let inherited = resolved(&base, None); + assert_eq!(resolved(&base, Some(json!({}))), inherited); + assert_eq!( + resolved(&base, Some(json!({"ttlSec": {}, "ramp": {}, "shadow": {}}))), + inherited + ); + assert_eq!(resolved(&base, Some(Value::Null)), inherited); + assert!(inherited.request_local && !inherited.coalesce); + assert_eq!(inherited.remote_read_timeout_ms, 40); + assert_eq!(inherited.stale_on_error_max_age_ms, 30_000); + assert!(inherited.shadow.enabled && inherited.shadow.log_mismatches); + + // A present leaf replaces the operation leaf, sparsely. + let patched = resolved( + &base, + Some(json!({"ttlSec": {"local": 3}, "ramp": {"remote": 0}})), + ); + assert_eq!(patched.local.ttl_ms, 3_000); + assert!(patched.local.enabled); + assert_eq!(patched.remote.ttl_ms, 10_000); + assert_eq!(patched.remote.reason, Some(DisabledReason::RampedDown)); + assert!(patched.remote.configured); + // Recovery still measures against the merged remote TTL. + assert_eq!(patched.stale_on_error_max_age_ms, 30_000); + + // An explicit null leaf is an invalid supplied value that disables only that layer. + let nulled = resolved(&base, Some(json!({"ttlSec": {"remote": null}}))); + assert!(nulled.local.enabled); + assert_eq!(nulled.remote.reason, Some(DisabledReason::InvalidTtl)); + assert!(!nulled.remote.configured); + // The pre-existing recovery age on a layer disabled by an invalid TTL is not a config error. + assert!(!nulled.stale_on_error_config_error); + assert_eq!(nulled.stale_on_error_max_age_ms, 0); + + // An invalid flag fails the whole invocation. + assert!(resolve(&base, Some(json!({"coalesce": "yes"}))).is_err()); + assert!(resolve(&base, Some(json!({"coalesce": null}))).is_err()); + assert!(resolve(&base, Some(json!({"requestLocal": 0}))).is_err()); + // ... as does an invalid runtime read deadline. + assert!(resolve(&base, Some(json!({"remoteReadTimeoutMs": -5}))).is_err()); + // A valid runtime flag or deadline wins over the operation. + let flipped = resolved( + &base, + Some(json!({"coalesce": true, "requestLocal": false, "remoteReadTimeoutMs": 7})), + ); + assert!(flipped.coalesce && !flipped.request_local); + assert_eq!(flipped.remote_read_timeout_ms, 7); + + // An invalid optional recovery age preserves remote serving with a diagnostic error. + let recovery = resolved(&base, Some(json!({"staleOnErrorMaxAgeSec": 10}))); + assert!(recovery.remote.enabled); + assert_eq!(recovery.stale_on_error_max_age_ms, 0); + assert!(recovery.stale_on_error_config_error); + let recovery = resolved(&base, Some(json!({"staleOnErrorMaxAgeSec": "30"}))); + assert!(recovery.remote.enabled && recovery.stale_on_error_config_error); + // Zero disables inherited recovery silently. + let off = resolved(&base, Some(json!({"staleOnErrorMaxAgeSec": 0}))); + assert!(off.remote.enabled); + assert_eq!(off.stale_on_error_max_age_ms, 0); + assert!(!off.stale_on_error_config_error); + // Raising the remote TTL above the inherited age invalidates the inherited recovery. + let raised = resolved(&base, Some(json!({"ttlSec": {"remote": 30}}))); + assert!(raised.remote.enabled && raised.stale_on_error_config_error); + assert_eq!(raised.stale_on_error_max_age_ms, 0); + + // Shadow leaves merge independently of serving leaves. + let shadow = resolved(&base, Some(json!({"shadow": {"logMismatches": false}}))); + assert!(shadow.shadow.enabled && !shadow.shadow.log_mismatches); + let shadow = resolved(&base, Some(json!({"shadow": {"ramp": 0}}))); + assert!( + !shadow.shadow.enabled && shadow.shadow.log_mismatches && !shadow.shadow.config_error + ); + assert!(shadow.local.enabled && shadow.remote.enabled); + } + + #[test] + fn runtime_leaves_can_configure_paths_the_operation_omitted() { + // Runtime TTLs enable layers the operation left off, with an implied full ramp. + let r = resolved( + &Policy::default(), + Some(json!({"ttlSec": {"local": 5, "remote": 6}})), + ); + assert!(r.local.enabled && r.remote.enabled); + assert_eq!((r.local.ramp, r.remote.ramp), (100.0, 100.0)); + assert_eq!((r.local.ttl_ms, r.remote.ttl_ms), (5_000, 6_000)); + // A ramp without a TTL leaves the layer policy-disabled. + let r = resolved(&Policy::default(), Some(json!({"ramp": {"local": 100}}))); + assert_eq!(r.local.reason, Some(DisabledReason::PolicyDisabled)); + assert!(!r.local.configured); + // Runtime shadow and recovery leaves attach to a runtime remote layer. + let r = resolved( + &Policy::default(), + Some( + json!({"ttlSec": {"remote": 6}, "staleOnErrorMaxAgeSec": 7, "shadow": {"ramp": 100}}), + ), + ); + assert_eq!(r.stale_on_error_max_age_ms, 7_000); + assert!(r.shadow.enabled && !r.shadow.log_mismatches); + // A runtime static-policy overlay (via From) behaves like its JSON form. + let overlay = RuntimePolicy::from(Policy::default().local_ttl_sec(9)); + let r = resolve_policy( + &Policy::default(), + Some(&overlay), + KEY, + PolicyDefaults::default(), + ) + .unwrap(); + assert!(r.local.enabled); + assert_eq!(r.local.ttl_ms, 9_000); + assert_eq!(r.remote.reason, Some(DisabledReason::PolicyDisabled)); + } + + #[test] + fn typed_runtime_overlays_preserve_omitted_flags() { + let base = Policy::default().request_local(true).coalesce(false); + for overlay in [Policy::default(), Policy::default().local_ttl_sec(9)] { + let r = resolve_policy( + &base, + Some(&RuntimePolicy::from(overlay)), + KEY, + PolicyDefaults::default(), + ) + .unwrap(); + assert!(r.request_local, "an omitted flag must inherit the baseline"); + assert!(!r.coalesce, "an omitted flag must inherit the baseline"); + } + + let explicit = resolved( + &base, + Some(RuntimePolicy::from(Policy::default().request_local(false).coalesce(true)).0), + ); + assert!(!explicit.request_local); + assert!(explicit.coalesce); + + let disabled = resolved(&base, Some(RuntimePolicy::from(Policy::disabled()).0)); + assert!(!disabled.request_local); + assert!( + !disabled.coalesce, + "the kill switch leaves coalescing unset" + ); + } + + #[test] + fn json_round_trip_preserves_omitted_and_explicit_flags() { + for request_local in [None, Some(false), Some(true)] { + for coalesce in [None, Some(false), Some(true)] { + let policy = Policy { + request_local, + coalesce, + ..Policy::default() + }; + assert_eq!(Policy::from_json(&policy.to_json()).unwrap(), policy); + } + } + } + + #[test] + fn instance_defaults_and_deadline_precedence() { + let base = Policy::enabled(1); + // Zero instance default means the library default. + let r = resolve_policy( + &base, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: 0, + }, + ) + .unwrap(); + assert_eq!(r.remote_read_timeout_ms, DEFAULT_REMOTE_READ_TIMEOUT_MS); + // The instance default applies when neither operation nor runtime supplies a budget. + let r = resolve_policy( + &base, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: 20, + }, + ) + .unwrap(); + assert_eq!(r.remote_read_timeout_ms, 20); + // Operation beats instance; runtime beats operation. + let op = base.clone().remote_read_timeout_ms(30); + let r = resolve_policy( + &op, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: 20, + }, + ) + .unwrap(); + assert_eq!(r.remote_read_timeout_ms, 30); + let overlay = RuntimePolicy(json!({"remoteReadTimeoutMs": 40})); + let r = resolve_policy( + &op, + Some(&overlay), + KEY, + PolicyDefaults { + remote_read_timeout_ms: 20, + }, + ) + .unwrap(); + assert_eq!(r.remote_read_timeout_ms, 40); + // An out-of-domain instance default fails resolution before anything else. + let err = resolve_policy( + &base, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: MAX_DEADLINE_MS + 1, + }, + ); + assert_eq!( + err.unwrap_err(), + PolicyError("invalid instance read deadline".to_string()) + ); + assert!(resolve_policy( + &base, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: MAX_DEADLINE_MS + } + ) + .is_ok()); + // A statically invalid base fails resolution even with a valid overlay. + let invalid = Policy::default().local_ttl_sec(0); + assert!(resolve(&invalid, Some(json!({"ttlSec": {"local": 5}}))).is_err()); + } + + #[test] + fn error_messages_name_the_offending_container() { + assert_eq!( + Policy::from_json(&json!([])).unwrap_err(), + PolicyError("DialCache defaultConfig must be an object".to_string()) + ); + assert_eq!( + Policy::from_json(&json!({"ttlSec": null})).unwrap_err(), + PolicyError("DialCache ttlSec must be an object".to_string()) + ); + assert_eq!( + Policy::from_json(&json!({"shadowRamp": 1})).unwrap_err(), + PolicyError("shadowRamp was replaced by shadow.ramp".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!(1))).unwrap_err(), + PolicyError("DialCache runtime config must be an object".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"shadow": 1}))).unwrap_err(), + PolicyError("DialCache shadow must be an object".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"shadowRamp": null}))).unwrap_err(), + PolicyError("shadowRamp was replaced by shadow.ramp".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"coalesce": null}))).unwrap_err(), + PolicyError("runtime coalesce must be boolean".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"requestLocal": "x"}))).unwrap_err(), + PolicyError("runtime requestLocal must be boolean".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"remoteReadTimeoutMs": 0}))).unwrap_err(), + PolicyError("invalid runtime remoteReadTimeoutMs".to_string()) + ); + } +} diff --git a/rust/src/preview.rs b/rust/src/preview.rs new file mode 100644 index 00000000..93efe84c --- /dev/null +++ b/rust/src/preview.rs @@ -0,0 +1,150 @@ +//! Bounded previews for mismatch warnings. + +use std::io::{self, Write}; + +use serde::Serialize; + +/// Maximum UTF-8 bytes of a logged cache key. +pub const SHADOW_LOG_KEY_MAX_BYTES: usize = 2 * 1024; +/// Maximum UTF-8 bytes of a logged value preview. +pub const SHADOW_LOG_VALUE_MAX_BYTES: usize = 8 * 1024; +/// Marker appended to a clamped preview. +pub const SHADOW_LOG_TRUNCATION_MARKER: &str = "...[truncated]"; + +/// Retain only the logged prefix, but consume the complete serialization so +/// errors beyond the cap still suppress the preview. +struct JsonPrefix { + bytes: Vec, + truncated: bool, +} + +impl JsonPrefix { + fn new() -> Self { + Self { + bytes: Vec::with_capacity(SHADOW_LOG_VALUE_MAX_BYTES), + truncated: false, + } + } + + fn finish(mut self) -> String { + if self.truncated { + self.bytes + .truncate(SHADOW_LOG_VALUE_MAX_BYTES - SHADOW_LOG_TRUNCATION_MARKER.len()); + } + // The prefix can end inside a multibyte character. JSON serialization + // produces valid UTF-8, so only that trailing character can be partial. + if let Err(error) = std::str::from_utf8(&self.bytes) { + self.bytes.truncate(error.valid_up_to()); + } + let mut text = String::from_utf8(self.bytes).expect("valid JSON prefix"); + if self.truncated { + text.push_str(SHADOW_LOG_TRUNCATION_MARKER); + } + text + } +} + +impl Write for JsonPrefix { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let retained = bytes + .len() + .min(SHADOW_LOG_VALUE_MAX_BYTES - self.bytes.len()); + self.bytes.extend_from_slice(&bytes[..retained]); + self.truncated |= retained < bytes.len(); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub(crate) fn json_preview(value: &T) -> Option { + let mut prefix = JsonPrefix::new(); + serde_json::to_writer(&mut prefix, value).ok()?; + Some(prefix.finish()) +} + +/// Clamp text to `max_bytes` of UTF-8, ending a clamped result with the +/// truncation marker on a character boundary. +pub fn clamp_utf8(value: &str, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value.to_string(); + } + let mut limit = max_bytes.saturating_sub(SHADOW_LOG_TRUNCATION_MARKER.len()); + while limit > 0 && !value.is_char_boundary(limit) { + limit -= 1; + } + let mut result = String::with_capacity(limit + SHADOW_LOG_TRUNCATION_MARKER.len()); + result.push_str(&value[..limit]); + result.push_str(SHADOW_LOG_TRUNCATION_MARKER); + result +} + +/// Clamp a logical key to [`SHADOW_LOG_KEY_MAX_BYTES`]. +pub fn preview_key(key: &str) -> String { + clamp_utf8(key, SHADOW_LOG_KEY_MAX_BYTES) +} + +/// Clamp a value's JSON preview to [`SHADOW_LOG_VALUE_MAX_BYTES`]. +pub fn preview_value(json: &str) -> String { + clamp_utf8(json, SHADOW_LOG_VALUE_MAX_BYTES) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::ser::{Error, SerializeSeq}; + + #[test] + fn clamps_on_character_boundaries() { + let text = "é".repeat(2000); + let clamped = clamp_utf8(&text, 100); + assert!(clamped.len() <= 100); + assert!(clamped.ends_with(SHADOW_LOG_TRUNCATION_MARKER)); + assert_eq!(clamp_utf8("short", 100), "short"); + } + + #[test] + fn streamed_json_preserves_the_existing_preview_prefix() { + for value in [ + "short".to_owned(), + "x".repeat(SHADOW_LOG_VALUE_MAX_BYTES - 2), + "x".repeat(SHADOW_LOG_VALUE_MAX_BYTES - 1), + "é🙂".repeat(SHADOW_LOG_VALUE_MAX_BYTES), + "\n\t\"\\".repeat(SHADOW_LOG_VALUE_MAX_BYTES), + ] { + assert_eq!( + json_preview(&value).unwrap(), + preview_value(&serde_json::to_string(&value).unwrap()) + ); + } + } + + struct LargeSequence { + fail_at_end: bool, + } + + impl Serialize for LargeSequence { + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(Some(10_000))?; + for _ in 0..10_000 { + sequence.serialize_element(&"large element".repeat(100))?; + } + if self.fail_at_end { + return Err(S::Error::custom("failure after the logged prefix")); + } + sequence.end() + } + } + + #[test] + fn large_json_construction_keeps_a_capped_buffer_and_detects_late_errors() { + let mut prefix = JsonPrefix::new(); + serde_json::to_writer(&mut prefix, &LargeSequence { fail_at_end: false }).unwrap(); + assert_eq!(prefix.bytes.len(), SHADOW_LOG_VALUE_MAX_BYTES); + assert_eq!(prefix.bytes.capacity(), SHADOW_LOG_VALUE_MAX_BYTES); + assert!(prefix.finish().ends_with(SHADOW_LOG_TRUNCATION_MARKER)); + assert!(json_preview(&LargeSequence { fail_at_end: true }).is_none()); + } +} diff --git a/rust/src/prometheus.rs b/rust/src/prometheus.rs new file mode 100644 index 00000000..f103a12d --- /dev/null +++ b/rust/src/prometheus.rs @@ -0,0 +1,414 @@ +//! Prometheus metric exporter (feature `prometheus`). +//! +//! [`PrometheusObserver`] registers the nineteen DialCache collectors on a +//! [`Registry`] under the names, help text, label sets and histogram buckets +//! of the TypeScript adapter, and feeds every [`Event`] to them. +//! +//! # Sharing one registry +//! +//! One observer owns one set of collectors. Instances that export to the same +//! registry share the observer: it is `Clone`, and every clone feeds the same +//! series. The `prometheus` crate reports a duplicate registration without a +//! handle to the existing collector, so a second `new` for the same registry +//! and prefix is a [`PrometheusError::Conflict`], as is an externally +//! registered collector under one of the DialCache names. A failed +//! constructor unregisters every collector it had registered, so no series is +//! left behind; the `prometheus` crate nevertheless remembers each such name +//! with the DialCache schema for the registry's lifetime, so only a collector +//! with the same schema can take that name later. + +use std::fmt; +use std::sync::Arc; + +use ::prometheus::core::Collector; +use ::prometheus::{HistogramOpts, HistogramVec, IntCounterVec, Opts, Registry}; + +use crate::metrics::MetricKind; +use crate::observe::{Event, Observer}; + +const TIMER_BUCKETS: &[f64] = &[ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, +]; +const SIZE_BUCKETS: &[f64] = &[ + 100.0, + 1_000.0, + 10_000.0, + 100_000.0, + 1_000_000.0, + 10_000_000.0, +]; +const RATIO_BUCKETS: &[f64] = &[0.05, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9, 1.0]; +/// Value ages span seconds to the 365-day TTL ceiling: 1s..15m, then 1h, 3h, 12h, 1d, 3d, 7d. +const VALUE_AGE_BUCKETS: &[f64] = &[ + 1.0, 5.0, 15.0, 60.0, 300.0, 900.0, 3_600.0, 10_800.0, 43_200.0, 86_400.0, 259_200.0, 604_800.0, +]; +const FUTURE_TIMESTAMP_OFFSET_BUCKETS: &[f64] = &[ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 15.0, 60.0, 300.0, 900.0, 3_600.0, + 10_800.0, 43_200.0, +]; + +/// The wire schema of one collector: what a scrape exposes. +#[derive(Debug, Clone, PartialEq)] +pub struct CollectorSchema { + /// The event kind the collector receives. + pub kind: MetricKind, + /// The fully qualified metric name, prefix included. + pub name: String, + /// The Prometheus help text, spelled as in the TypeScript adapter. + pub help: &'static str, + /// Label names in wire order. + pub labels: &'static [&'static str], + /// Histogram bucket upper bounds; empty for counters. + pub buckets: &'static [f64], +} + +impl CollectorSchema { + /// `true` for the counter vectors; every other collector is a histogram. + pub fn is_counter(&self) -> bool { + self.kind.is_counter() + } +} + +/// The nineteen collectors DialCache registers, in registration order, with +/// their names under `prefix`. +pub fn schemas(prefix: &str) -> Vec { + let schema = |kind: MetricKind, + suffix: &str, + help: &'static str, + buckets: &'static [f64]| + -> CollectorSchema { + CollectorSchema { + kind, + name: format!("{prefix}dialcache_{suffix}"), + help, + labels: kind.label_names(), + buckets, + } + }; + vec![ + schema( + MetricKind::Disabled, + "disabled_counter", + "Requests where DialCache skipped a cache layer.", + &[], + ), + schema( + MetricKind::Miss, + "miss_counter", + "DialCache cache misses.", + &[], + ), + schema( + MetricKind::Request, + "request_counter", + "Total DialCache cache-layer requests.", + &[], + ), + schema( + MetricKind::Error, + "error_counter", + "Errors during DialCache cache operations or fallback execution.", + &[], + ), + schema( + MetricKind::Invalidation, + "invalidation_counter", + "DialCache invalidation calls by key type and layer.", + &[], + ), + schema( + MetricKind::Coalesced, + "coalesced_counter", + "DialCache requests coalesced onto in-flight work by sharing scope.", + &[], + ), + schema( + MetricKind::ShadowValidation, + "shadow_validation_counter", + "Sampled DialCache Redis shadow-validation outcomes.", + &[], + ), + schema( + MetricKind::ShadowValueAge, + "shadow_value_age_histogram", + "Age in seconds of the validated Redis value at DialCache shadow verdict time.", + VALUE_AGE_BUCKETS, + ), + schema( + MetricKind::FutureTimestampOffset, + "future_timestamp_offset_histogram", + "Positive offset in seconds of Redis frames dated after the observing DialCache process clock.", + FUTURE_TIMESTAMP_OFFSET_BUCKETS, + ), + schema( + MetricKind::StaleRecovery, + "stale_recovery_counter", + "DialCache stale-on-error Redis recovery outcomes.", + &[], + ), + schema( + MetricKind::StaleRecoveryValueAge, + "stale_recovery_value_age_histogram", + "Age in seconds of Redis values served by DialCache stale-on-error recovery.", + VALUE_AGE_BUCKETS, + ), + schema( + MetricKind::Compression, + "compression_counter", + "DialCache Redis payload compression and decompression outcomes.", + &[], + ), + schema( + MetricKind::Get, + "get_timer", + "DialCache cache get latency in seconds.", + TIMER_BUCKETS, + ), + schema( + MetricKind::Fallback, + "fallback_timer", + "Time DialCache waited for the fallback function in seconds.", + TIMER_BUCKETS, + ), + schema( + MetricKind::Serialization, + "serialization_timer", + "DialCache serialization latency in seconds.", + TIMER_BUCKETS, + ), + schema( + MetricKind::Size, + "size_histogram", + "Serialized DialCache value sizes in bytes.", + SIZE_BUCKETS, + ), + schema( + MetricKind::StoredSize, + "stored_size_histogram", + "Stored DialCache payload sizes in bytes, after compression and escaping.", + SIZE_BUCKETS, + ), + schema( + MetricKind::CompressionRatio, + "compression_ratio_histogram", + "Compressed-to-original DialCache payload size ratio for compressed writes.", + RATIO_BUCKETS, + ), + schema( + MetricKind::CompressionDuration, + "compression_timer", + "DialCache payload compression and decompression latency in seconds.", + TIMER_BUCKETS, + ), + ] +} + +/// Construction failures of a [`PrometheusObserver`]. +#[derive(Debug)] +pub enum PrometheusError { + /// A collector with this name is already registered: by another observer + /// (clone that observer instead of constructing a second one), by someone + /// else, or with another schema. Use a unique prefix or another registry. + /// The collectors registered before the collision were unregistered + /// again; their names stay bound to the DialCache schema in that registry. + Conflict { + /// The fully qualified collector name that collided. + name: String, + /// The registry's rejection. + source: ::prometheus::Error, + }, + /// The `prometheus` crate rejected a collector definition, which only a + /// prefix that is not a valid metric name fragment can cause. + InvalidCollector { + /// The fully qualified collector name that was rejected. + name: String, + /// The crate's rejection. + source: ::prometheus::Error, + }, +} + +impl fmt::Display for PrometheusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PrometheusError::Conflict { name, source } => write!( + f, + "Prometheus collector {name:?} already exists with an incompatible or externally \ + owned schema; use a unique prefix or registry: {source}" + ), + PrometheusError::InvalidCollector { name, source } => { + write!(f, "invalid Prometheus collector {name:?}: {source}") + } + } + } +} + +impl std::error::Error for PrometheusError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + PrometheusError::Conflict { source, .. } + | PrometheusError::InvalidCollector { source, .. } => Some(source), + } + } +} + +#[derive(Clone)] +enum Vector { + Counter(IntCounterVec), + Histogram(HistogramVec), +} + +impl Vector { + fn boxed(&self) -> Box { + match self { + Vector::Counter(vector) => Box::new(vector.clone()), + Vector::Histogram(vector) => Box::new(vector.clone()), + } + } +} + +/// The collectors of one (registry, prefix) pair, indexed by [`MetricKind::index`]. +struct Group { + vectors: Vec, +} + +impl Group { + fn build(prefix: &str) -> Result { + let mut vectors: Vec> = vec![None; MetricKind::ALL.len()]; + for schema in schemas(prefix) { + let vector = if schema.is_counter() { + IntCounterVec::new(Opts::new(&schema.name, schema.help), schema.labels) + .map(Vector::Counter) + } else { + HistogramVec::new( + HistogramOpts::new(&schema.name, schema.help).buckets(schema.buckets.to_vec()), + schema.labels, + ) + .map(Vector::Histogram) + } + .map_err(|source| PrometheusError::InvalidCollector { + name: schema.name.clone(), + source, + })?; + vectors[schema.kind.index()] = Some(vector); + } + Ok(Group { + vectors: vectors + .into_iter() + .map(|vector| vector.expect("every kind has a schema")) + .collect(), + }) + } + + /// Register every collector on `registry`; roll back on the first failure. + fn register(&self, registry: &Registry, prefix: &str) -> Result<(), PrometheusError> { + let mut registered: Vec<&Vector> = Vec::new(); + for schema in schemas(prefix) { + let vector = &self.vectors[schema.kind.index()]; + if let Err(source) = registry.register(vector.boxed()) { + for done in registered { + let _ = registry.unregister(done.boxed()); + } + return Err(PrometheusError::Conflict { + name: schema.name, + source, + }); + } + registered.push(vector); + } + Ok(()) + } +} + +/// Publishes DialCache diagnostics to Prometheus collectors. +/// +/// Clones share the collectors; hand one observer to every instance that +/// exports to the same registry. +#[derive(Clone)] +pub struct PrometheusObserver { + group: Arc, + prefix: String, +} + +impl fmt::Debug for PrometheusObserver { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PrometheusObserver") + .field("prefix", &self.prefix) + .finish_non_exhaustive() + } +} + +impl PrometheusObserver { + /// Register the DialCache collectors named `dialcache_*` on + /// `registry`. Registering the same names twice on one registry is a + /// [`PrometheusError::Conflict`]: clone the observer instead. + pub fn new(registry: &Registry, prefix: &str) -> Result { + let group = Arc::new(Group::build(prefix)?); + group.register(registry, prefix)?; + Ok(PrometheusObserver { + group, + prefix: prefix.to_string(), + }) + } + + /// The wire schemas of this observer's collectors. + pub fn schemas(&self) -> Vec { + schemas(&self.prefix) + } +} + +impl Observer for PrometheusObserver { + fn observe(&self, event: &Event) { + let kind = MetricKind::of(event); + let labels = MetricKind::labels(event); + let values: Vec<&str> = labels.iter().map(|(_, value)| value.as_str()).collect(); + let result = match &self.group.vectors[kind.index()] { + Vector::Counter(vector) => vector + .get_metric_with_label_values(&values) + .map(|metric| metric.inc()), + Vector::Histogram(vector) => vector + .get_metric_with_label_values(&values) + .map(|metric| metric.observe(MetricKind::value(event))), + }; + if let Err(error) = result { + log::warn!( + target: "dialcache", + "Dropped DialCache {} metric with labels {:?}: {error}", + kind.as_str(), + labels + ); + } + } + + fn observes_shadow_outcomes(&self) -> bool { + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schemas_cover_every_kind_once_with_matching_label_names() { + let all = schemas("p_"); + assert_eq!(all.len(), MetricKind::ALL.len()); + let mut seen = std::collections::HashSet::new(); + for schema in &all { + assert!(seen.insert(schema.kind), "{:?} twice", schema.kind); + assert!(schema.name.starts_with("p_dialcache_")); + assert_eq!(schema.labels, schema.kind.label_names()); + assert_eq!(schema.buckets.is_empty(), schema.is_counter()); + } + } + + #[test] + fn invalid_prefix_is_an_error_not_a_panic() { + let registry = Registry::new(); + let error = PrometheusObserver::new(®istry, "bad prefix ").unwrap_err(); + assert!( + matches!(error, PrometheusError::InvalidCollector { .. }), + "{error}" + ); + assert!(registry.gather().is_empty()); + } +} diff --git a/rust/src/protocol/envelope.rs b/rust/src/protocol/envelope.rs new file mode 100644 index 00000000..7aefdd05 --- /dev/null +++ b/rust/src/protocol/envelope.rs @@ -0,0 +1,584 @@ +//! Payload envelopes: escaping and zstd compression. +//! +//! Byte 0 of a binary payload written by this version or later selects the +//! envelope: `0x00` escapes raw serializer output whose own first byte would +//! collide, `0x01` marks a zstd frame of UTF-8 text and `0x02` a zstd frame of +//! binary bytes. Raw output is escaped on every write, and every read +//! interprets the envelope, so disabling compression never strands entries. +//! See "Envelope selection and codec environment" in `formal/PROTOCOL.md`. + +use crate::codec::Payload; +use crate::limits::{DEFAULT_COMPRESSION_THRESHOLD_BYTES, DEFAULT_ZSTD_LEVEL, MAX_SAFE_INTEGER}; +use crate::observe::CompressionOutcome; + +use super::frame::ProtocolError; +use super::text::replacement_utf8; + +/// Envelope byte `0x00`: the rest is raw serializer output whose own first +/// byte would have collided with a marker. +pub const MARKER_ESCAPED_RAW: u8 = 0x00; +/// Envelope byte `0x01`: the rest is one zstd frame of UTF-8 text. +pub const MARKER_ZSTD_UTF8: u8 = 0x01; +/// Envelope byte `0x02`: the rest is one zstd frame of binary bytes. +pub const MARKER_ZSTD_BINARY: u8 = 0x02; + +const MIN_ZSTD_LEVEL: i32 = 1; +const MAX_ZSTD_LEVEL: i32 = 22; + +/// Write-side compression policy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CompressionConfig { + /// Payloads of at least this many serialized bytes are compressed. + pub threshold_bytes: usize, + /// zstd level, `1..=22`. + pub level: i32, +} + +impl Default for CompressionConfig { + fn default() -> Self { + CompressionConfig { + threshold_bytes: DEFAULT_COMPRESSION_THRESHOLD_BYTES, + level: DEFAULT_ZSTD_LEVEL, + } + } +} + +impl CompressionConfig { + /// The threshold must be a positive safe integer and the level `1..=22`. + pub fn validate(&self) -> Result<(), ProtocolError> { + if self.threshold_bytes < 1 || self.threshold_bytes as u64 > MAX_SAFE_INTEGER { + return Err(ProtocolError::Compression( + "threshold_bytes must be a positive safe integer".to_string(), + )); + } + if !(MIN_ZSTD_LEVEL..=MAX_ZSTD_LEVEL).contains(&self.level) { + return Err(ProtocolError::Compression( + "level must be an integer between 1 and 22".to_string(), + )); + } + Ok(()) + } +} + +/// What [`compress_payload`] chose to store, and why. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompressionWriteResult { + /// The bytes to store: marked zstd output, or the escaped raw form. + pub payload: Payload, + /// One of the write outcomes: compressed, below threshold, not smaller + /// or over limit. + pub outcome: CompressionOutcome, + /// Length of the serializer output after text replacement, before escaping. + pub original_bytes: usize, + /// Length of `payload`. + pub stored_bytes: usize, +} + +/// What [`decompress_payload`] produced from a stored payload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompressionReadResult { + /// The decoded payload; the input with its `0x00` escape prefix removed + /// when it was escaped raw output; or the input unchanged when it passed + /// through untouched or zstd rejected it. + pub payload: Payload, + /// `None` when no zstd envelope was involved: the payload passed through + /// untouched or only had its escape prefix removed. + pub outcome: Option, +} + +fn needs_raw_escape(payload: &Payload) -> bool { + payload.binary + && payload + .bytes + .first() + .is_some_and(|first| *first <= MARKER_ZSTD_BINARY) +} + +/// Prefix raw binary output whose first byte would collide with the envelope. +pub fn escape_raw_payload(payload: Payload) -> Payload { + if needs_raw_escape(&payload) { + let mut escaped = Vec::with_capacity(payload.bytes.len() + 1); + escaped.push(MARKER_ESCAPED_RAW); + escaped.extend_from_slice(&payload.bytes); + return Payload::binary(escaped); + } + payload +} + +/// Compress when the payload meets the threshold and the marked result is +/// strictly smaller than the escaped raw form. +/// +/// Text payloads receive replacement UTF-8 conversion first, so the measured +/// size is the stored text's. The threshold comparison precedes the output +/// cap: a value the read side would refuse is stored raw (`write_over_limit`) +/// and subject to Redis's own value limit instead. `original_bytes` is the +/// payload's own length and `stored_bytes` the length actually written. +pub fn compress_payload( + payload: Payload, + config: &CompressionConfig, + max_decompressed_bytes: usize, +) -> Result { + config.validate()?; + let payload = if payload.binary { + payload + } else { + Payload { + bytes: replacement_utf8(&payload.bytes), + binary: false, + } + }; + let original_bytes = payload.len(); + let outcome = if original_bytes < config.threshold_bytes { + CompressionOutcome::BelowThreshold + } else if original_bytes > max_decompressed_bytes { + CompressionOutcome::WriteOverLimit + } else { + let compressed = zstd_compress(&payload.bytes, config.level)?; + let raw_bytes = original_bytes + usize::from(needs_raw_escape(&payload)); + if compressed.len() + 1 < raw_bytes { + let mut marked = Vec::with_capacity(compressed.len() + 1); + marked.push(if payload.binary { + MARKER_ZSTD_BINARY + } else { + MARKER_ZSTD_UTF8 + }); + marked.extend_from_slice(&compressed); + return Ok(CompressionWriteResult { + stored_bytes: marked.len(), + payload: Payload::binary(marked), + outcome: CompressionOutcome::Compressed, + original_bytes, + }); + } + CompressionOutcome::NotSmaller + }; + let raw = escape_raw_payload(payload); + Ok(CompressionWriteResult { + stored_bytes: raw.len(), + payload: raw, + outcome, + original_bytes, + }) +} + +/// Reverse of [`compress_payload`], applied to every read. +/// +/// Text payloads and empty binary payloads pass through. An escape prefix is +/// removed only when the next byte is one the writer escapes; other `0x00` +/// leads and unknown markers pass through unchanged. A marked payload decodes +/// only its first zstd frame and ignores trailing bytes, like Node's +/// synchronous decoder. Malformed or truncated zstd data keeps the original +/// marked bytes with `fallback_raw`; output beyond `max_decompressed_bytes` +/// (or a frame demanding more decoder memory than allowed) keeps them with +/// `read_over_limit`. Successful `0x01` output receives replacement UTF-8 +/// conversion; `0x02` output is exact bytes. The input is never mutated, so +/// repeated loads of a retained payload are independent. +pub fn decompress_payload( + payload: Payload, + max_decompressed_bytes: usize, +) -> CompressionReadResult { + let passthrough = |payload| CompressionReadResult { + payload, + outcome: None, + }; + if !payload.binary { + return passthrough(payload); + } + let Some(&marker) = payload.bytes.first() else { + return passthrough(payload); + }; + match marker { + MARKER_ESCAPED_RAW => { + if payload + .bytes + .get(1) + .is_some_and(|next| *next <= MARKER_ZSTD_BINARY) + { + return passthrough(Payload::binary(&payload.bytes[1..])); + } + passthrough(payload) + } + MARKER_ZSTD_UTF8 | MARKER_ZSTD_BINARY => { + match decode_first_zstd_frame(&payload.bytes[1..], max_decompressed_bytes) { + Ok(decoded) => { + let decoded = if marker == MARKER_ZSTD_UTF8 { + Payload { + bytes: replacement_utf8(&decoded), + binary: false, + } + } else { + Payload::binary(decoded) + }; + CompressionReadResult { + payload: decoded, + outcome: Some(CompressionOutcome::Decompressed), + } + } + Err(outcome) => CompressionReadResult { + payload, + outcome: Some(outcome), + }, + } + } + _ => passthrough(payload), + } +} + +/// One zstd frame of `bytes` at `level`, without a checksum, as the writer +/// stores it (single-shot, so the frame header carries the content size). +fn zstd_compress(bytes: &[u8], level: i32) -> Result, ProtocolError> { + let codec = + |code: usize| ProtocolError::Compression(zstd_safe::get_error_name(code).to_string()); + let mut context = zstd_safe::CCtx::create(); + context + .set_parameter(zstd_safe::CParameter::CompressionLevel(level)) + .map_err(codec)?; + context + .set_parameter(zstd_safe::CParameter::ChecksumFlag(false)) + .map_err(codec)?; + let mut out = vec![0u8; zstd_safe::compress_bound(bytes.len())]; + let written = context.compress2(&mut out[..], bytes).map_err(codec)?; + out.truncate(written); + Ok(out) +} + +/// Decode only the first zstd frame of `body`, ignoring any trailer, with the +/// output cap enforced while streaming so a small stored frame can never force +/// a large allocation. Skippable frames decode to empty output. +fn decode_first_zstd_frame( + body: &[u8], + max_decompressed_bytes: usize, +) -> Result, CompressionOutcome> { + // Rejects truncated and malformed headers or block sequences up front, + // like Go's firstZstdFrame; only a complete first frame is decoded. + let frame_len = + zstd_safe::find_frame_compressed_size(body).map_err(|_| CompressionOutcome::FallbackRaw)?; + let frame = body + .get(..frame_len) + .ok_or(CompressionOutcome::FallbackRaw)?; + + let mut context = zstd_safe::DCtx::create(); + let mut out = Vec::new(); + let mut input = zstd_safe::InBuffer::around(frame); + // Each step writes at most one chunk; a cap smaller than a chunk still + // only needs one chunk to observe the overflow. + let chunk_len = zstd_safe::DCtx::out_size().max(1); + let mut chunk = vec![0u8; chunk_len]; + loop { + let mut output = zstd_safe::OutBuffer::around(&mut chunk[..]); + let consumed_before = input.pos(); + let hint = context + .decompress_stream(&mut output, &mut input) + .map_err(classify_decoder_error)?; + let produced = output.as_slice(); + if out.len() + produced.len() > max_decompressed_bytes { + return Err(CompressionOutcome::ReadOverLimit); + } + out.extend_from_slice(produced); + if hint == 0 { + return Ok(out); + } + if input.pos() == frame.len() && produced.is_empty() { + // The verified frame ended without the decoder finishing it. + return Err(CompressionOutcome::FallbackRaw); + } + if input.pos() == consumed_before && produced.is_empty() { + return Err(CompressionOutcome::FallbackRaw); + } + } +} + +/// zstd errors that mean the frame exceeds the decoder's resource limits are +/// read-limit outcomes; anything else falls back to the raw bytes. +fn classify_decoder_error(code: usize) -> CompressionOutcome { + let name = zstd_safe::get_error_name(code); + if name.contains("too much memory") || name.contains("Destination buffer is too small") { + CompressionOutcome::ReadOverLimit + } else { + CompressionOutcome::FallbackRaw + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::limits::MAX_DECOMPRESSED_BYTES; + + fn text(len: usize) -> Payload { + Payload::text("x".repeat(len)) + } + + fn compressed(payload: Payload) -> CompressionWriteResult { + let result = compress_payload( + payload, + &CompressionConfig { + threshold_bytes: 1, + level: 3, + }, + MAX_DECOMPRESSED_BYTES, + ) + .unwrap(); + assert_eq!(result.outcome, CompressionOutcome::Compressed); + result + } + + #[test] + fn config_validation() { + assert_eq!(CompressionConfig::default().validate(), Ok(())); + assert!(CompressionConfig { + threshold_bytes: 0, + level: 3 + } + .validate() + .is_err()); + assert!(CompressionConfig { + threshold_bytes: usize::MAX, + level: 3 + } + .validate() + .is_err()); + assert!(CompressionConfig { + threshold_bytes: 1, + level: 0 + } + .validate() + .is_err()); + assert!(CompressionConfig { + threshold_bytes: 1, + level: 23 + } + .validate() + .is_err()); + assert_eq!( + CompressionConfig { + threshold_bytes: 1, + level: 22 + } + .validate(), + Ok(()) + ); + let invalid = compress_payload( + text(1), + &CompressionConfig { + threshold_bytes: 1, + level: 23, + }, + MAX_DECOMPRESSED_BYTES, + ); + assert!(matches!(invalid, Err(ProtocolError::Compression(_)))); + } + + #[test] + fn escaping_and_passthrough() { + assert_eq!( + escape_raw_payload(Payload::binary(vec![0, 9])).bytes, + [0, 0, 9] + ); + assert_eq!(escape_raw_payload(Payload::binary(vec![2])).bytes, [0, 2]); + assert_eq!(escape_raw_payload(Payload::binary(vec![3])).bytes, [3]); + assert_eq!( + escape_raw_payload(Payload::binary(Vec::new())).bytes, + Vec::::new() + ); + assert_eq!(escape_raw_payload(Payload::text("\u{0}")).bytes, [0]); + // Only a prefix the writer produces is stripped; legacy 0x00 leads stay. + assert_eq!( + decompress_payload(Payload::binary(vec![0, 1, 7]), 10), + CompressionReadResult { + payload: Payload::binary(vec![1, 7]), + outcome: None + } + ); + for untouched in [ + Payload::binary(vec![0, 3]), + Payload::binary(vec![0]), + Payload::binary(vec![3, 1]), + Payload::binary(Vec::new()), + Payload::text("\u{1}x"), + ] { + assert_eq!( + decompress_payload(untouched.clone(), 10), + CompressionReadResult { + payload: untouched, + outcome: None + } + ); + } + } + + /// Port of go/codec_test.go TestCompressionLimitsAndIndependentLoads. + #[test] + fn compression_limits_and_independent_loads() { + let raw = text(4096); + let written = compressed(raw.clone()); + let too_large = decompress_payload(written.payload.clone(), 4095); + assert_eq!(too_large.outcome, Some(CompressionOutcome::ReadOverLimit)); + assert_eq!(too_large.payload, written.payload); + let exact = decompress_payload(written.payload.clone(), 4096); + assert_eq!( + exact, + CompressionReadResult { + payload: raw.clone(), + outcome: Some(CompressionOutcome::Decompressed) + } + ); + let refused = compress_payload( + raw.clone(), + &CompressionConfig { + threshold_bytes: 1, + level: 3, + }, + 4095, + ) + .unwrap(); + assert_eq!(refused.outcome, CompressionOutcome::WriteOverLimit); + assert_eq!(refused.payload, raw); + assert_eq!((refused.original_bytes, refused.stored_bytes), (4096, 4096)); + // Repeated loads of a retained input are independent. + let mut first = decompress_payload(written.payload.clone(), MAX_DECOMPRESSED_BYTES); + first.payload.bytes[0] = b'y'; + assert_eq!( + decompress_payload(written.payload.clone(), MAX_DECOMPRESSED_BYTES).payload, + raw + ); + let escaped = escape_raw_payload(Payload::binary(vec![1, 2, 3])); + let mut first = decompress_payload(escaped.clone(), MAX_DECOMPRESSED_BYTES); + first.payload.bytes[0] = 9; + assert_eq!( + decompress_payload(escaped, MAX_DECOMPRESSED_BYTES) + .payload + .bytes, + [1, 2, 3] + ); + // A zero cap admits only empty output. + assert_eq!( + decompress_payload(written.payload.clone(), 0).outcome, + Some(CompressionOutcome::ReadOverLimit) + ); + } + + /// Port of go/codec_test.go TestZstdFirstFrameAndMalformedBodies. + #[test] + fn first_frame_and_malformed_bodies() { + let source = Payload::text("first".repeat(100)); + let written = compressed(source.clone()); + for trailer in [&b"CRC!"[..], &written.payload.bytes[1..]] { + let mut marked = written.payload.bytes.clone(); + marked.extend_from_slice(trailer); + let result = decompress_payload(Payload::binary(marked), MAX_DECOMPRESSED_BYTES); + assert_eq!( + result, + CompressionReadResult { + payload: source.clone(), + outcome: Some(CompressionOutcome::Decompressed) + } + ); + } + let half = written.payload.bytes[..written.payload.bytes.len() / 2].to_vec(); + for marked in [ + vec![1], + vec![2], + half, + vec![1, 0x28, 0xB5, 0x2F, 0xFD], + vec![2, 0xFF, 0xFF], + ] { + let got = decompress_payload(Payload::binary(marked.clone()), MAX_DECOMPRESSED_BYTES); + assert_eq!( + got, + CompressionReadResult { + payload: Payload::binary(marked), + outcome: Some(CompressionOutcome::FallbackRaw) + } + ); + } + } + + #[test] + fn skippable_frames_decode_to_empty_output() { + // Magic 0x184D2A50, 4-byte little-endian size, then that many bytes. + let skippable = [ + 0x01, 0x50, 0x2A, 0x4D, 0x18, 0x02, 0x00, 0x00, 0x00, 0xAA, 0xBB, 0xCC, + ]; + let got = decompress_payload(Payload::binary(skippable.to_vec()), MAX_DECOMPRESSED_BYTES); + assert_eq!( + got, + CompressionReadResult { + payload: Payload::text(""), + outcome: Some(CompressionOutcome::Decompressed) + } + ); + // Zero cap with empty output is still a successful decode. + assert_eq!( + decompress_payload(Payload::binary(skippable.to_vec()), 0).outcome, + Some(CompressionOutcome::Decompressed) + ); + let truncated = skippable[..8].to_vec(); + assert_eq!( + decompress_payload(Payload::binary(truncated), MAX_DECOMPRESSED_BYTES).outcome, + Some(CompressionOutcome::FallbackRaw) + ); + } + + #[test] + fn selection_rule_and_types() { + let config = CompressionConfig { + threshold_bytes: 4, + level: 3, + }; + let below = compress_payload(text(3), &config, MAX_DECOMPRESSED_BYTES).unwrap(); + assert_eq!( + (below.outcome, below.original_bytes, below.stored_bytes), + (CompressionOutcome::BelowThreshold, 3, 3) + ); + let incompressible = compress_payload( + Payload::binary(vec![1, 2, 3, 4]), + &config, + MAX_DECOMPRESSED_BYTES, + ) + .unwrap(); + assert_eq!(incompressible.outcome, CompressionOutcome::NotSmaller); + assert_eq!(incompressible.payload.bytes, [0, 1, 2, 3, 4]); + assert_eq!( + (incompressible.original_bytes, incompressible.stored_bytes), + (4, 5) + ); + let binary = compressed(Payload::binary(vec![7; 500])); + assert_eq!(binary.payload.bytes[0], MARKER_ZSTD_BINARY); + assert!(binary.payload.binary && binary.stored_bytes < 500); + assert_eq!( + decompress_payload(binary.payload, MAX_DECOMPRESSED_BYTES).payload, + Payload::binary(vec![7; 500]) + ); + let text_result = compressed(text(500)); + assert_eq!(text_result.payload.bytes[0], MARKER_ZSTD_UTF8); + assert_eq!( + decompress_payload(text_result.payload, MAX_DECOMPRESSED_BYTES).payload, + text(500) + ); + // Ill-formed text is replaced before measuring and compressing. + let ill_formed = Payload { + bytes: vec![0xE2, 0x82, b'A'], + binary: false, + }; + let stored = compress_payload( + ill_formed, + &CompressionConfig { + threshold_bytes: 100, + level: 3, + }, + MAX_DECOMPRESSED_BYTES, + ) + .unwrap(); + assert_eq!(stored.payload, Payload::text("\u{FFFD}A")); + assert_eq!((stored.original_bytes, stored.stored_bytes), (4, 4)); + // Decoded 0x01 text is replaced; 0x02 bytes are exact. + let mut marked_text = + compressed(Payload::binary([0xE2, 0x82, b'A', 0xFF].repeat(50))).payload; + marked_text.bytes[0] = MARKER_ZSTD_UTF8; + let decoded = decompress_payload(marked_text, MAX_DECOMPRESSED_BYTES); + assert_eq!( + decoded.payload, + Payload::text("\u{FFFD}A\u{FFFD}".repeat(50)) + ); + } +} diff --git a/rust/src/protocol/frame.rs b/rust/src/protocol/frame.rs new file mode 100644 index 00000000..911ef507 --- /dev/null +++ b/rust/src/protocol/frame.rs @@ -0,0 +1,558 @@ +//! Version-1 frames and semantic read results. +//! +//! A frame is `[0x01] ++ big-endian u64 createdAtMs ++ [0x00 text | 0x01 binary] +//! ++ payload`. [`decode_frame`] applies the protocol's classification +//! precedence to one raw read; [`normalize_read_result`] is the trust boundary +//! above wire decoding that every semantic adapter result passes through. + +use crate::codec::Payload; +use crate::limits::{MAX_SAFE_INTEGER, MAX_SUPPORTED_DURATION_MS}; +use crate::remote::{Frame, MissReason, ReadResult}; + +use super::text::replacement_utf8; + +/// Wire-level failures that are errors rather than misses. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ProtocolError { + /// A frame's encoding tag is neither text (`0x00`) nor binary (`0x01`). + #[error("Invalid DialCache Redis payload encoding")] + PayloadEncoding, + /// The server accepted a command but replied outside the protocol; the + /// text says how, for example a watermark that is not a digit string. + #[error("Invalid DialCache Redis reply: {0}")] + InvalidReply(String), + /// A stamp is not a nonnegative safe integer, or an invalidation would + /// push its watermark past one. + #[error("DialCache timestamp must be a nonnegative safe integer")] + InvalidTimestamp, + /// A TTL is not positive, or a TTL or future buffer exceeds 365 days. + #[error("DialCache cache TTL must be positive and no greater than 365 days")] + InvalidDuration, + /// zstd or its configuration failed on the write side; the text is + /// zstd's error name or the rejected option. + #[error("DialCache compression failed: {0}")] + Compression(String), +} + +const FRAME_VERSION: u8 = 1; +const FRAME_HEADER_LEN: usize = 10; +const ENCODING_TEXT: u8 = 0; +const ENCODING_BINARY: u8 = 1; + +/// Encode a frame: version byte, big-endian uint64 stamp, encoding tag, payload. +/// Text payloads receive replacement UTF-8 conversion first. +pub fn encode_frame(frame: &Frame) -> Result, ProtocolError> { + if frame.created_at_ms > MAX_SAFE_INTEGER { + return Err(ProtocolError::InvalidTimestamp); + } + let mut out = Vec::with_capacity(FRAME_HEADER_LEN + frame.payload.len()); + out.push(FRAME_VERSION); + out.extend_from_slice(&frame.created_at_ms.to_be_bytes()); + if frame.payload.binary { + out.push(ENCODING_BINARY); + out.extend_from_slice(&frame.payload.bytes); + } else { + out.push(ENCODING_TEXT); + out.extend_from_slice(&replacement_utf8(&frame.payload.bytes)); + } + Ok(out) +} + +/// Parse a watermark bulk string. `None` input is absent (a valid zero +/// baseline); `Some(Err)` is malformed; `Some(Ok(value))` is a valid stamp. +/// +/// Only ASCII digits are accepted. Leading zeros are ignored even when the +/// literal would overflow a machine integer; the remaining digits must fit +/// the safe-integer domain. +pub fn parse_watermark(raw: Option<&[u8]>) -> Option> { + let raw = raw?; + Some(parse_watermark_digits(raw)) +} + +fn parse_watermark_digits(raw: &[u8]) -> Result { + let malformed = + || ProtocolError::InvalidReply("watermark is not a nonnegative safe integer".to_string()); + if raw.is_empty() || !raw.iter().all(u8::is_ascii_digit) { + return Err(malformed()); + } + let mut value: u64 = 0; + for digit in raw.iter().skip_while(|byte| **byte == b'0') { + value = value + .checked_mul(10) + .and_then(|scaled| scaled.checked_add(u64::from(digit - b'0'))) + .ok_or_else(malformed)?; + } + if value > MAX_SAFE_INTEGER { + return Err(malformed()); + } + Ok(value) +} + +/// Classify one raw read with the protocol's precedence: absent value, +/// unsupported frame, tracked marker validity and zero stamp, fence, +/// payload encoding, then a hit. +/// +/// `watermark` is consulted only when `tracked`. A miss carries the parsed +/// fence whenever the marker was valid (including the absent baseline, which +/// carries none); a malformed marker yields an unclassified miss without one. +/// The stamp is returned with full `u64` precision; the safe-integer check +/// belongs to [`normalize_read_result`]. +pub fn decode_frame( + raw: Option<&[u8]>, + tracked: bool, + watermark: Option<&[u8]>, +) -> Result { + let (fence, valid_marker) = if tracked { + match parse_watermark(watermark) { + None => (None, true), + Some(Ok(value)) => (Some(value), true), + Some(Err(_)) => (None, false), + } + } else { + (None, true) + }; + let miss = |reason: MissReason| { + Ok(ReadResult::Miss { + reason, + observed_watermark_ms: fence, + }) + }; + + let Some(raw) = raw else { + return miss(MissReason::ValueAbsent); + }; + if raw.len() < FRAME_HEADER_LEN || raw[0] != FRAME_VERSION { + return miss(MissReason::Unclassified); + } + let stamp = u64::from_be_bytes( + raw[1..9] + .try_into() + .expect("frame header holds eight stamp bytes"), + ); + if tracked && (!valid_marker || stamp == 0) { + return miss(MissReason::Unclassified); + } + if let (true, Some(fence)) = (tracked, fence) { + if stamp <= fence { + return miss(MissReason::WatermarkFenced); + } + } + let payload = match raw[9] { + ENCODING_TEXT => Payload { + bytes: replacement_utf8(&raw[FRAME_HEADER_LEN..]), + binary: false, + }, + ENCODING_BINARY => Payload::binary(&raw[FRAME_HEADER_LEN..]), + _ => return Err(ProtocolError::PayloadEncoding), + }; + Ok(ReadResult::Hit(Frame { + created_at_ms: stamp, + payload, + })) +} + +/// Interpret an untrusted JSON-shaped semantic reply from a custom adapter. +/// Unknown shapes become unclassified misses; frame-shaped objects ignore +/// stray miss metadata. +/// +/// Only `"kind": "miss"` selects the miss branch. Its reason is parsed from +/// `"reason"` (unknown strings become unclassified) and its +/// `observedWatermarkMs` is kept only when it is a JSON number in the +/// timestamp domain. Any other object is frame-shaped: `createdAtMs` must be a +/// number in the timestamp domain, and a JSON string `payload` is text while +/// anything else is an empty text payload. The result still needs +/// [`normalize_read_result`] for the tracked-key fence rules. +pub fn read_result_from_untrusted_json(value: &serde_json::Value) -> ReadResult { + let Some(object) = value.as_object() else { + return ReadResult::miss(MissReason::Unclassified); + }; + if object.get("kind").and_then(serde_json::Value::as_str) == Some("miss") { + let reason = object + .get("reason") + .and_then(serde_json::Value::as_str) + .and_then(MissReason::parse) + .unwrap_or(MissReason::Unclassified); + let observed_watermark_ms = object + .get("observedWatermarkMs") + .and_then(serde_json::Value::as_f64) + .and_then(|number| validate_timestamp_ms(number).ok()); + return ReadResult::Miss { + reason, + observed_watermark_ms, + }; + } + let Some(created_at_ms) = object + .get("createdAtMs") + .and_then(serde_json::Value::as_f64) + .and_then(|number| validate_timestamp_ms(number).ok()) + else { + return ReadResult::miss(MissReason::Unclassified); + }; + let payload = match object.get("payload").and_then(serde_json::Value::as_str) { + Some(text) => Payload::text(text), + None => Payload::default(), + }; + ReadResult::Hit(Frame { + created_at_ms, + payload, + }) +} + +/// The core trust boundary above wire decoding: drop fences on untracked +/// keys or outside the safe domain, and demote a fenced claim without a fence. +pub fn normalize_read_result(result: ReadResult, tracked: bool) -> ReadResult { + match result { + ReadResult::Miss { + reason, + observed_watermark_ms, + } => { + let observed_watermark_ms = + observed_watermark_ms.filter(|fence| tracked && *fence <= MAX_SAFE_INTEGER); + let reason = match reason { + MissReason::WatermarkFenced if observed_watermark_ms.is_none() => { + MissReason::Unclassified + } + other => other, + }; + ReadResult::Miss { + reason, + observed_watermark_ms, + } + } + ReadResult::Hit(frame) if frame.created_at_ms > MAX_SAFE_INTEGER => { + ReadResult::miss(MissReason::Unclassified) + } + hit @ ReadResult::Hit(_) => hit, + } +} + +/// A nonnegative safe-integer timestamp. +pub fn validate_timestamp_ms(value: f64) -> Result { + if !value.is_finite() + || value < 0.0 + || value.trunc() != value + || value > MAX_SAFE_INTEGER as f64 + { + return Err(ProtocolError::InvalidTimestamp); + } + Ok(value as u64) +} + +/// Round a write TTL up to whole milliseconds and check the 365-day domain. +pub fn ceil_supported_cache_ttl_ms(value: f64) -> Result { + let ceiled = value.ceil(); + if !ceiled.is_finite() || ceiled <= 0.0 || ceiled > MAX_SUPPORTED_DURATION_MS as f64 { + return Err(ProtocolError::InvalidDuration); + } + Ok(ceiled as u64) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn text_frame(stamp: u64, text: &str) -> Vec { + encode_frame(&Frame { + created_at_ms: stamp, + payload: Payload::text(text), + }) + .expect("encodes") + } + + #[test] + fn encode_layout_and_domain() { + assert_eq!(text_frame(1, "A"), [1, 0, 0, 0, 0, 0, 0, 0, 1, 0, b'A']); + let binary = Frame { + created_at_ms: 0x0102, + payload: Payload::binary(vec![0xFF]), + }; + assert_eq!( + encode_frame(&binary).unwrap(), + [1, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0xFF] + ); + let unsafe_stamp = Frame { + created_at_ms: MAX_SAFE_INTEGER + 1, + payload: Payload::text(""), + }; + assert_eq!( + encode_frame(&unsafe_stamp), + Err(ProtocolError::InvalidTimestamp) + ); + // Ill-formed text is replaced at encode time; binary bytes stay exact. + let ill_formed = Frame { + created_at_ms: 1, + payload: Payload { + bytes: vec![0xE2, 0x82], + binary: false, + }, + }; + assert_eq!( + &encode_frame(&ill_formed).unwrap()[10..], + [0xEF, 0xBF, 0xBD] + ); + } + + #[test] + fn watermark_parsing() { + assert_eq!(parse_watermark(None), None); + assert_eq!(parse_watermark(Some(b"0")), Some(Ok(0))); + assert_eq!( + parse_watermark(Some(b"000000000000000000000000000042")), + Some(Ok(42)) + ); + assert_eq!( + parse_watermark(Some(b"9007199254740991")), + Some(Ok(MAX_SAFE_INTEGER)) + ); + for malformed in [ + &b""[..], + b"9007199254740992", + b"99999999999999999999999", + b"-1", + b"1.0", + b" 1", + b"1a", + "١".as_bytes(), + ] { + assert!( + matches!( + parse_watermark(Some(malformed)), + Some(Err(ProtocolError::InvalidReply(_))) + ), + "{malformed:?}" + ); + } + } + + #[test] + fn decode_precedence() { + let frame = text_frame(1000, "cached"); + let fenced = decode_frame(Some(&frame), true, Some(b"1000")).unwrap(); + assert_eq!( + fenced, + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(1000) + } + ); + let hit = decode_frame(Some(&frame), true, Some(b"999")).unwrap(); + assert_eq!( + hit, + ReadResult::Hit(Frame { + created_at_ms: 1000, + payload: Payload::text("cached") + }) + ); + // Untracked reads never consult the marker. + assert_eq!( + decode_frame(Some(&frame), false, Some(b"garbage")).unwrap(), + hit + ); + // Absent value carries a valid fence; a malformed marker drops it. + assert_eq!( + decode_frame(None, true, Some(b"7")).unwrap(), + ReadResult::Miss { + reason: MissReason::ValueAbsent, + observed_watermark_ms: Some(7) + } + ); + assert_eq!( + decode_frame(None, true, Some(b"x")).unwrap(), + ReadResult::miss(MissReason::ValueAbsent) + ); + // Short or unversioned frames are unclassified before the marker is examined. + assert_eq!( + decode_frame(Some(&frame[..9]), true, Some(b"5")).unwrap(), + ReadResult::Miss { + reason: MissReason::Unclassified, + observed_watermark_ms: Some(5) + } + ); + assert_eq!( + decode_frame(Some(b""), false, None).unwrap(), + ReadResult::miss(MissReason::Unclassified) + ); + // Zero stamps and malformed markers are unclassified only on tracked keys. + let zero = text_frame(0, "z"); + assert_eq!( + decode_frame(Some(&zero), true, None).unwrap(), + ReadResult::miss(MissReason::Unclassified) + ); + assert!(matches!( + decode_frame(Some(&zero), false, None).unwrap(), + ReadResult::Hit(_) + )); + assert_eq!( + decode_frame(Some(&frame), true, Some(b"")).unwrap(), + ReadResult::miss(MissReason::Unclassified) + ); + // The fence check precedes the encoding tag; a fenced bad tag is a miss. + let mut bad_tag = frame.clone(); + bad_tag[9] = 2; + assert_eq!( + decode_frame(Some(&bad_tag), false, None), + Err(ProtocolError::PayloadEncoding) + ); + assert_eq!( + decode_frame(Some(&bad_tag), true, Some(b"1000")).unwrap(), + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(1000) + } + ); + // Full u64 stamps survive decoding; normalization demotes them. + let mut huge = text_frame(1, "h"); + huge[1..9].copy_from_slice(&u64::MAX.to_be_bytes()); + let decoded = decode_frame(Some(&huge), false, None).unwrap(); + assert!(matches!( + decoded, + ReadResult::Hit(Frame { + created_at_ms: u64::MAX, + .. + }) + )); + assert_eq!( + normalize_read_result(decoded, false), + ReadResult::miss(MissReason::Unclassified) + ); + } + + #[test] + fn untrusted_json_shapes() { + assert_eq!( + read_result_from_untrusted_json(&json!(null)), + ReadResult::miss(MissReason::Unclassified) + ); + assert_eq!( + read_result_from_untrusted_json(&json!([1])), + ReadResult::miss(MissReason::Unclassified) + ); + assert_eq!( + read_result_from_untrusted_json( + &json!({"kind": "miss", "reason": "watermark_fenced", "observedWatermarkMs": 5}) + ), + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(5) + } + ); + assert_eq!( + read_result_from_untrusted_json( + &json!({"kind": "miss", "reason": "bogus", "observedWatermarkMs": "5"}) + ), + ReadResult::miss(MissReason::Unclassified) + ); + assert_eq!( + read_result_from_untrusted_json( + &json!({"kind": "miss", "reason": "expired", "observedWatermarkMs": 1.5}) + ), + ReadResult::miss(MissReason::Expired) + ); + // Frame-shaped objects ignore stray miss metadata and non-string payloads. + assert_eq!( + read_result_from_untrusted_json( + &json!({"kind": "hit", "createdAtMs": 3, "payload": "p", "reason": "expired"}) + ), + ReadResult::Hit(Frame { + created_at_ms: 3, + payload: Payload::text("p") + }) + ); + assert_eq!( + read_result_from_untrusted_json(&json!({"createdAtMs": 3, "payload": [1, 2]})), + ReadResult::Hit(Frame { + created_at_ms: 3, + payload: Payload::default() + }) + ); + for bad in [ + json!({"createdAtMs": -1}), + json!({"createdAtMs": "3"}), + json!({"payload": "p"}), + json!({"createdAtMs": 1.5}), + ] { + assert_eq!( + read_result_from_untrusted_json(&bad), + ReadResult::miss(MissReason::Unclassified), + "{bad}" + ); + } + } + + #[test] + fn normalization_rules() { + let fenced = ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(9), + }; + assert_eq!(normalize_read_result(fenced.clone(), true), fenced); + assert_eq!( + normalize_read_result(fenced, false), + ReadResult::miss(MissReason::Unclassified) + ); + let unsafe_fence = ReadResult::Miss { + reason: MissReason::Expired, + observed_watermark_ms: Some(MAX_SAFE_INTEGER + 1), + }; + assert_eq!( + normalize_read_result(unsafe_fence, true), + ReadResult::miss(MissReason::Expired) + ); + let hit = ReadResult::Hit(Frame { + created_at_ms: MAX_SAFE_INTEGER, + payload: Payload::text("ok"), + }); + assert_eq!(normalize_read_result(hit.clone(), true), hit); + } + + #[test] + fn timestamp_and_duration_domains() { + assert_eq!(validate_timestamp_ms(0.0), Ok(0)); + assert_eq!( + validate_timestamp_ms(MAX_SAFE_INTEGER as f64), + Ok(MAX_SAFE_INTEGER) + ); + for bad in [ + -1.0, + 1.5, + 9007199254740992.0, + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + -0.5, + ] { + assert_eq!( + validate_timestamp_ms(bad), + Err(ProtocolError::InvalidTimestamp), + "{bad}" + ); + } + assert_eq!(validate_timestamp_ms(-0.0), Ok(0)); + assert_eq!(ceil_supported_cache_ttl_ms(0.1), Ok(1)); + assert_eq!(ceil_supported_cache_ttl_ms(1.1), Ok(2)); + assert_eq!( + ceil_supported_cache_ttl_ms(31535999999.999), + Ok(MAX_SUPPORTED_DURATION_MS) + ); + assert_eq!( + ceil_supported_cache_ttl_ms(MAX_SUPPORTED_DURATION_MS as f64), + Ok(MAX_SUPPORTED_DURATION_MS) + ); + for bad in [ + 0.0, + -1.0, + 31536000000.001, + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + ] { + assert_eq!( + ceil_supported_cache_ttl_ms(bad), + Err(ProtocolError::InvalidDuration), + "{bad}" + ); + } + } +} diff --git a/rust/src/protocol/mod.rs b/rust/src/protocol/mod.rs new file mode 100644 index 00000000..68f47367 --- /dev/null +++ b/rust/src/protocol/mod.rs @@ -0,0 +1,17 @@ +//! The portable wire protocol: frames, envelopes, text conversion and +//! duration/timestamp domains (W04–W08 in `formal/CONTRACTS.md`). + +mod envelope; +mod frame; +mod text; + +pub use envelope::{ + compress_payload, decompress_payload, escape_raw_payload, CompressionConfig, + CompressionReadResult, CompressionWriteResult, MARKER_ESCAPED_RAW, MARKER_ZSTD_BINARY, + MARKER_ZSTD_UTF8, +}; +pub use frame::{ + ceil_supported_cache_ttl_ms, decode_frame, encode_frame, normalize_read_result, + parse_watermark, read_result_from_untrusted_json, validate_timestamp_ms, ProtocolError, +}; +pub use text::replacement_utf8; diff --git a/rust/src/protocol/text.rs b/rust/src/protocol/text.rs new file mode 100644 index 00000000..8338e1b7 --- /dev/null +++ b/rust/src/protocol/text.rs @@ -0,0 +1,68 @@ +//! Text payload conversion. +//! +//! Text frame payloads and decompressed `0x01` envelopes may hold any bytes. +//! `formal/PROTOCOL.md` ("Text payload domain") fixes how they become text: +//! the WHATWG UTF-8 decoder with replacement error handling and without BOM +//! removal, so every maximal ill-formed subpart becomes exactly one U+FFFD. + +/// Decode bytes as UTF-8 with one U+FFFD per maximal ill-formed subpart and +/// no BOM removal (the WHATWG decoder with replacement error handling). +/// +/// The result is always valid UTF-8. Well-formed input is returned unchanged, +/// so valid text costs one validation pass and one copy. +pub fn replacement_utf8(bytes: &[u8]) -> Vec { + String::from_utf8_lossy(bytes).into_owned().into_bytes() +} + +#[cfg(test)] +mod tests { + use super::replacement_utf8; + + fn decoded(bytes: &[u8]) -> String { + String::from_utf8(replacement_utf8(bytes)).expect("replacement output is UTF-8") + } + + #[test] + fn protocol_replacement_table() { + // The representative outcomes fixed by formal/PROTOCOL.md. + assert_eq!(decoded(&[0x22, 0xFF, 0x22]), "\"\u{FFFD}\""); + assert_eq!(decoded(&[0xE2, 0x82]), "\u{FFFD}"); + assert_eq!(decoded(&[0xE2, 0x82, 0x41]), "\u{FFFD}A"); + assert_eq!(decoded(&[0xED, 0xA0, 0x80]), "\u{FFFD}\u{FFFD}\u{FFFD}"); + assert_eq!( + decoded(&[0xF4, 0x90, 0x80, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}" + ); + assert_eq!(decoded(&[0xEF, 0xBB, 0xBF, 0x61]), "\u{FEFF}a"); + } + + #[test] + fn well_formed_input_is_unchanged() { + let text = "\u{80}\u{7FF}\u{800}\u{D7FF}\u{E000}\u{FFFF}\u{10000}\u{10FFFF}\0"; + assert_eq!(replacement_utf8(text.as_bytes()), text.as_bytes()); + assert!(replacement_utf8(b"").is_empty()); + } + + #[test] + fn isolated_continuations_and_overlong_forms_replace_separately() { + assert_eq!(decoded(&[0x80, 0xBF]), "\u{FFFD}\u{FFFD}"); + assert_eq!( + decoded(&[0xC0, 0xAF, 0xE0, 0x80, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}" + ); + // A lead that reaches end of input after accepted continuations. + assert_eq!(decoded(&[0xF0, 0x9F, 0x98]), "\u{FFFD}"); + // The narrowed second byte after F0 rejects 8F; 8F is then a lone continuation. + assert_eq!( + decoded(&[0xF0, 0x8F, 0x80, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}" + ); + // A complete prefix interrupted by a new lead keeps the second sequence intact. + assert_eq!(decoded(&[0xE2, 0x82, 0xC3, 0xA9]), "\u{FFFD}\u{E9}"); + // Leads beyond F4 and FE/FF are single-byte subparts. + assert_eq!( + decoded(&[0xF5, 0x80, 0xFE, 0xFF, 0x41]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}A" + ); + } +} diff --git a/rust/src/read_snapshot_tests.rs b/rust/src/read_snapshot_tests.rs new file mode 100644 index 00000000..6af64831 --- /dev/null +++ b/rust/src/read_snapshot_tests.rs @@ -0,0 +1,137 @@ +//! Native allocation and codec-ownership boundaries for acquired remote frames. +use super::*; +use crate::operation::{downcast_value, erase_load, Operation}; +use crate::testing::{TestExecutor, WALL_EPOCH_MS}; +use crate::{Codec, DialCache, InvalidateRequest, Payload, Policy, Remote}; +use futures::channel::oneshot; +use parking_lot::Mutex; + +struct OneRead(Mutex>); +impl Remote for OneRead { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(std::future::ready(Ok(ReadResult::Hit( + self.0.lock().take().unwrap(), + )))) + } + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + panic!("this test only reads") + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + panic!("this test only reads") + } +} +struct GatedCodec { + gate: Mutex>>, + input: Settled, +} +impl Codec for GatedCodec { + fn encode(&self, _: &u64) -> BoxFuture<'_, Result> { + panic!("this test only decodes") + } + fn decode(&self, mut payload: Payload) -> BoxFuture<'_, Result> { + let gate = self.gate.lock().take().unwrap(); + Box::pin(async move { + self.input.settle(payload.bytes.as_ptr() as usize); + payload.bytes[0] = 99; + gate.await?; + Ok(payload.len() as u64) + }) + } +} +fn hit(result: RawRead) -> Arc { + match result.unwrap() { + ReadSnapshot::Hit(frame) => frame, + ReadSnapshot::Miss { .. } => panic!("expected a hit"), + } +} + +#[test] +fn acquired_frames_share_storage_while_a_gated_codec_owns_an_independent_input() { + for ready_first in [false, true] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let payload = Payload::binary(vec![42; 128 * 1024]); + let original_buffer = payload.bytes.as_ptr() as usize; + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote(OneRead(Mutex::new(Some(Frame { + created_at_ms: WALL_EPOCH_MS as u64, + payload, + })))) + .build() + .unwrap(); + let (release, gate) = oneshot::channel(); + let codec = Arc::new(GatedCodec { + gate: Mutex::new(Some(gate)), + input: Settled::new(), + }); + let (identity, metadata) = Operation::with_codec( + Identity::new("thing", "one", "ReadSnapshot"), + codec.clone(), + |a, b| a == b, + ) + .policy(Policy::default().remote_ttl_sec(60)) + .erase(); + let keys = identity.keys().unwrap(); + let policy = resolve_policy( + &metadata.policy, + None, + &keys.logical, + PolicyDefaults { + remote_read_timeout_ms: 100, + }, + ) + .unwrap(); + let request = cache.enable_guard(); + let execution = Arc::new(Execution { + core: cache.core.clone(), + scope: request.scope().clone(), + op: Arc::new(ErasedOperation { + identity: identity.clone(), + identity_provider: None, + metadata, + load: erase_load(|_| async { Ok(0u64) }), + }), + labels: crate::engine::outcome_labels(&identity), + identity, + keys, + policy, + }); + let (bounded, raw) = execution.raw_read(); + if ready_first { + executor.drain(); + } + // The two schedules cover the deadline cell's ready peek and pending wait. + let frame = hit(executor.block_on(bounded)); + let observed = hit(raw.peek().unwrap()); + assert!(Arc::ptr_eq(&frame, &observed)); + assert_eq!(frame.payload.bytes.as_ptr() as usize, original_buffer); + let weak = Arc::downgrade(&frame); + let (done, receive) = oneshot::channel(); + let decoding = execution.clone(); + let decoding_frame = frame.clone(); + executor.spawn(async move { + let result = decoding.decode(&decoding_frame, Layer::Remote, None).await; + done.send(result).unwrap(); + }); + executor.drain(); + assert_ne!(codec.input.peek().expect("codec started"), original_buffer); + assert!( + frame.payload.bytes.iter().all(|byte| *byte == 42), + "codec mutation leaked into the retained snapshot" + ); + let cleanup = hit(executor.block_on(async move { raw.wait().await })); + assert!( + Arc::ptr_eq(&frame, &cleanup), + "cleanup must observe the shared allocation" + ); + release.send(()).unwrap(); + let value = executor.block_on(async move { receive.await.unwrap().unwrap() }); + assert_eq!(*downcast_value::(value).unwrap(), 128 * 1024); + drop((frame, observed, cleanup)); + assert!( + weak.upgrade().is_none(), + "completed observers retained the frame" + ); + } +} diff --git a/rust/src/redis.rs b/rust/src/redis.rs new file mode 100644 index 00000000..2cc724a2 --- /dev/null +++ b/rust/src/redis.rs @@ -0,0 +1,906 @@ +//! Production [`Remote`] adapter over the [`redis`] crate. +//! +//! [`RedisAdapter`] borrows a connected, caller-owned connection handle: a +//! [`ConnectionManager`] for standalone or Sentinel-managed servers, a +//! [`ClusterConnection`] for Redis Cluster, or a plain +//! [`MultiplexedConnection`]. It never connects, reconnects, drains or closes +//! the handle and never changes its options. The caller remains responsible +//! for: +//! +//! - **Finite budgets.** Configure connection, response and retry limits on +//! the handle (`ConnectionManagerConfig::set_response_timeout`, +//! `ClusterClientBuilder::response_timeout`, ...). The cache bounds its own +//! wait for reads and honors cooperative cancellation, but cancellation +//! after dispatch cannot prove a command did not execute. +//! - **Primaries for tracked reads.** On a cluster handle the adapter routes +//! every tracked `MGET` explicitly to the slot primary, so replica lag can +//! never hide an invalidation watermark even when the caller enabled +//! `read_from_replicas`. A standalone or Sentinel-managed handle must point +//! at the primary itself; the adapter cannot tell a replica apart. +//! - **Watermark retention.** Invalidation stores watermarks with a +//! retention of at least two hours that only widens. Configure eviction so +//! watermark keys are not evicted before tracked values age out; an +//! evicted watermark resurrects fenced values. +//! +//! Wire behavior matches the TypeScript and Go adapters exactly: an +//! untracked read is one `GET`; a tracked read is one `MGET value watermark` +//! on the primary; a write is exactly one native `SET key frame PX ttl`; an +//! invalidation is `EVALSHA` of [`INVALIDATION_SCRIPT`] with one `EVAL` +//! retry carrying identical arguments after any `EVALSHA` failure. Accepted +//! but invalid replies are [`RedisProtocolError`]s and are never retried. + +use std::fmt; + +use futures::future::{self, BoxFuture, Either, FutureExt}; +use redis::aio::{ConnectionLike, ConnectionManager, MultiplexedConnection}; +use redis::cluster_async::ClusterConnection; +use redis::cluster_routing::{Route, RoutingInfo, SingleNodeRoutingInfo, Slot, SlotAddr}; +use redis::{Cmd, RedisResult, Value}; +use sha1::{Digest, Sha1}; + +use crate::error::BoxError; +use crate::limits::{MAX_SAFE_INTEGER, MAX_SUPPORTED_DURATION_MS}; +use crate::protocol::{decode_frame, encode_frame, ProtocolError}; +use crate::remote::{ + InvalidateRequest, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest, +}; + +/// The version-1 wire invalidation transition, byte-identical to the Go +/// `InvalidationScript` (a unit test pins this against `go/redis_adapter.go`). +/// TypeScript's `INVALIDATE_CACHE_SCRIPT` is the same Lua with different +/// whitespace and a comment, so its `EVALSHA` digest differs; every port +/// self-heals through the `EVAL` fallback, so mixed-language clusters need no +/// shared script cache. +/// +/// Redis Lua numbers exactly represent the accepted safe-integer domain. +/// Invalid arguments return before `GET`/repair; wrong-type keys alone are +/// repairable read failures. +pub const INVALIDATION_SCRIPT: &str = r#"local function parse_safe_integer(raw) + if not string.match(raw, "^%d+$") then return nil end + local value = tonumber(raw) + if not value or value > 9007199254740991 then return nil end + return value +end +local future_buffer_ms = parse_safe_integer(ARGV[1]) +if not future_buffer_ms or future_buffer_ms < 0 or future_buffer_ms > 31536000000 then + return redis.error_reply("ERR invalid DialCache future buffer") +end +local invalidated_at_ms = parse_safe_integer(ARGV[2]) +if not invalidated_at_ms or invalidated_at_ms > 9007199254740991 - future_buffer_ms then + return redis.error_reply("ERR invalid DialCache invalidatedAtMs") +end +local proposed_watermark = invalidated_at_ms + future_buffer_ms +local raw_watermark = redis.pcall("GET", KEYS[1]) +if type(raw_watermark) == "table" and raw_watermark.err then + if not string.match(raw_watermark.err, "^WRONGTYPE ") then return raw_watermark end + raw_watermark = false +end +local current_watermark = 0 +if raw_watermark then + local parsed_watermark = parse_safe_integer(raw_watermark) + if parsed_watermark then current_watermark = parsed_watermark end +end +local watermark = math.max(current_watermark, proposed_watermark) +local current_ttl_ms = -2 +if raw_watermark then current_ttl_ms = redis.call("PTTL", KEYS[1]) end +local desired_ttl_ms = math.max(7200000, watermark - invalidated_at_ms + 3600000 + 60000) +if current_ttl_ms > desired_ttl_ms then desired_ttl_ms = current_ttl_ms end +local encoded_watermark = string.format("%.0f", watermark) +if current_ttl_ms == -1 then + redis.call("SET", KEYS[1], encoded_watermark) +else + redis.call("SET", KEYS[1], encoded_watermark, "PX", desired_ttl_ms) +end +return 1"#; + +/// Lowercase hex SHA-1 of [`INVALIDATION_SCRIPT`], the `EVALSHA` argument. +pub fn invalidation_script_sha1() -> String { + hex::encode(Sha1::digest(INVALIDATION_SCRIPT.as_bytes())) +} + +/// A reply the server accepted but that violates the DialCache wire protocol. +/// +/// Such replies are never retried: the command executed and the adapter has +/// no way to tell what state the server holds. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("Invalid DialCache Redis reply: {message}")] +pub struct RedisProtocolError { + /// What the reply violated: an unexpected reply type, a tracked read + /// without exactly two bulk values, a `SET` reply other than `OK`, or an + /// invalidation reply other than the integer `1`. + pub message: String, +} + +impl RedisProtocolError { + fn new(message: impl Into) -> Self { + RedisProtocolError { + message: message.into(), + } + } +} + +/// A read stopped waiting because its [`ReadContext`] was cancelled. +/// +/// The command may still execute on the server; the cache's own deadline is +/// authoritative and this error only ends the adapter's wait. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("DialCache Redis read cancelled before a reply arrived")] +pub struct RedisReadCancelled; + +/// A cloneable, task-shareable handle that runs one Redis command. +/// +/// Implemented for [`ConnectionManager`], [`MultiplexedConnection`] and +/// [`ClusterConnection`]. Test doubles implement it to observe dispatched +/// arguments without a server. Every method borrows `&self`; implementations +/// clone the underlying handle per command, which the `redis` crate's handles +/// make cheap. +pub trait RedisConnection: Send + Sync + 'static { + /// Run one command with the handle's default routing. + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult>; + + /// Run one command that reads `key`, on the node that owns the key's + /// slot as a primary. Non-cluster handles have exactly one node and use + /// their default routing. + fn run_on_primary(&self, key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult>; +} + +/// Lift a RESP3 error reply carried as a value into the error channel so +/// every caller sees one failure shape. +fn settle(reply: RedisResult) -> RedisResult { + match reply { + Ok(Value::ServerError(error)) => Err(error.into()), + other => other, + } +} + +fn run_cloned(connection: &C, cmd: Cmd) -> BoxFuture<'_, RedisResult> +where + C: ConnectionLike + Clone + Send + Sync, +{ + let mut connection = connection.clone(); + async move { settle(connection.req_packed_command(&cmd).await) }.boxed() +} + +impl RedisConnection for ConnectionManager { + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } + + fn run_on_primary(&self, _key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } +} + +impl RedisConnection for MultiplexedConnection { + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } + + fn run_on_primary(&self, _key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } +} + +impl RedisConnection for ClusterConnection { + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } + + /// Selecting the slot primary is explicit here because a stale replica + /// could hide an invalidation fence when the client enabled replica reads. + fn run_on_primary(&self, key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + let route = Route::with_slot(Slot::for_key(key), SlotAddr::Master); + let routing = RoutingInfo::SingleNode(SingleNodeRoutingInfo::SpecificNode(route)); + let mut connection = self.clone(); + async move { settle(connection.route_command(cmd, routing).await) }.boxed() + } +} + +/// [`Remote`] over a caller-owned `redis` crate connection handle. +/// +/// See the [module documentation](self) for the caller's responsibilities. +#[derive(Clone)] +pub struct RedisAdapter { + connection: C, +} + +impl fmt::Debug for RedisAdapter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RedisAdapter").finish_non_exhaustive() + } +} + +impl RedisAdapter { + /// Borrow `connection`. The adapter never connects or closes it. + pub fn new(connection: C) -> Self { + RedisAdapter { connection } + } + + /// The borrowed connection handle. + pub fn connection(&self) -> &C { + &self.connection + } + + /// The script's raw decimal boundary for out-of-band tooling and + /// conformance replay. Validation inside Lua precedes every mutation. + /// + /// `EVALSHA` runs first; any failure is retried exactly once as `EVAL` + /// with identical arguments, and a failed retry surfaces unmodified. The + /// transition is idempotent (the watermark only advances and its + /// retention only widens), so a duplicate run after an ambiguous failure + /// is harmless. A successful reply must be the integer `1`. + pub fn invalidate_decimal<'a>( + &'a self, + watermark_key: &'a str, + future_buffer_ms: &'a str, + invalidated_at_ms: &'a str, + ) -> BoxFuture<'a, Result<(), BoxError>> { + async move { + let script_command = |program: &str, script: &str| { + let mut cmd = redis::cmd(program); + cmd.arg(script) + .arg(1) + .arg(watermark_key) + .arg(future_buffer_ms) + .arg(invalidated_at_ms); + cmd + }; + let evalsha = script_command("EVALSHA", &invalidation_script_sha1()); + let reply = match self.connection.run(evalsha).await { + Ok(reply) => reply, + Err(_) => { + let eval = script_command("EVAL", INVALIDATION_SCRIPT); + self.connection.run(eval).await? + } + }; + validate_invalidation_reply(&reply)?; + Ok(()) + } + .boxed() + } +} + +/// One bulk-string reply position: `None` is absent. +type Bulk = Option>; + +/// One bulk-string reply position: `Nil` is absent, string replies are +/// content, anything else is a protocol violation. +fn bulk(reply: Value) -> Result { + match reply { + Value::Nil => Ok(None), + Value::BulkString(bytes) => Ok(Some(bytes)), + Value::SimpleString(text) => Ok(Some(text.into_bytes())), + other => Err(RedisProtocolError::new(format!( + "expected Redis bulk string, got {}", + value_kind(&other) + ))), + } +} + +/// A tracked read must return exactly two bulk values. +fn tracked_reply(reply: Value) -> Result<(Bulk, Bulk), RedisProtocolError> { + let Value::Array(items) = reply else { + return Err(RedisProtocolError::new( + "tracked read must return exactly two bulk values", + )); + }; + let [value, watermark]: [Value; 2] = items + .try_into() + .map_err(|_| RedisProtocolError::new("tracked read must return exactly two bulk values"))?; + Ok((bulk(value)?, bulk(watermark)?)) +} + +/// `SET` must answer `OK`. +pub fn validate_set_reply(reply: &Value) -> Result<(), RedisProtocolError> { + let ok = match reply { + Value::Okay => true, + Value::SimpleString(text) => text == "OK", + Value::BulkString(bytes) => bytes == b"OK", + _ => false, + }; + if ok { + Ok(()) + } else { + Err(RedisProtocolError::new( + "invalid Redis SET reply; expected OK", + )) + } +} + +/// The invalidation script must answer the integer `1`. +pub fn validate_invalidation_reply(reply: &Value) -> Result<(), RedisProtocolError> { + if matches!(reply, Value::Int(1)) { + Ok(()) + } else { + Err(RedisProtocolError::new( + "invalid Redis invalidation reply; expected integer 1", + )) + } +} + +fn value_kind(value: &Value) -> &'static str { + match value { + Value::Nil => "nil", + Value::Int(_) => "integer", + Value::BulkString(_) => "bulk string", + Value::Array(_) => "array", + Value::SimpleString(_) => "simple string", + Value::Okay => "OK", + Value::Map(_) => "map", + Value::Attribute { .. } => "attribute", + Value::Set(_) => "set", + Value::Double(_) => "double", + Value::Boolean(_) => "boolean", + Value::VerbatimString { .. } => "verbatim string", + Value::BigNumber(_) => "big number", + Value::Push { .. } => "push", + Value::ServerError(_) => "server error", + _ => "unknown", + } +} + +impl Remote for RedisAdapter { + fn read( + &self, + request: ReadRequest, + context: ReadContext, + ) -> BoxFuture<'_, Result> { + async move { + if context.cancel.is_cancelled() { + return Err(Box::new(RedisReadCancelled) as BoxError); + } + let tracked = request.watermark_key.is_some(); + let command = match &request.watermark_key { + None => { + let mut cmd = redis::cmd("GET"); + cmd.arg(&request.value_key); + self.connection.run(cmd) + } + Some(watermark_key) => { + let mut cmd = redis::cmd("MGET"); + cmd.arg(&request.value_key).arg(watermark_key); + self.connection.run_on_primary(&request.value_key, cmd) + } + }; + let reply = match future::select(command, context.cancel.cancelled()).await { + Either::Left((reply, _)) => reply?, + Either::Right(((), _)) => return Err(Box::new(RedisReadCancelled) as BoxError), + }; + let (value, watermark) = if tracked { + tracked_reply(reply)? + } else { + (bulk(reply)?, None) + }; + Ok(decode_frame( + value.as_deref(), + tracked, + watermark.as_deref(), + )?) + } + .boxed() + } + + /// Exactly one native `SET` of the complete frame. It neither reads nor + /// modifies the entity watermark; the frame's stamp is stored exactly. + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + async move { + if request.ttl_ms == 0 || request.ttl_ms > MAX_SUPPORTED_DURATION_MS { + return Err(ProtocolError::InvalidDuration.into()); + } + let raw = encode_frame(&request.frame)?; + let mut cmd = redis::cmd("SET"); + cmd.arg(&request.value_key) + .arg(raw) + .arg("PX") + .arg(request.ttl_ms.to_string()); + let reply = self.connection.run(cmd).await?; + validate_set_reply(&reply)?; + Ok(()) + } + .boxed() + } + + /// The caller's clock sample stays stable through the `EVAL` retry, so + /// one logical invalidation proposes one watermark. + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + async move { + if request.future_buffer_ms > MAX_SUPPORTED_DURATION_MS { + return Err(ProtocolError::InvalidDuration.into()); + } + if request.invalidated_at_ms > MAX_SAFE_INTEGER - request.future_buffer_ms { + return Err(ProtocolError::InvalidTimestamp.into()); + } + self.invalidate_decimal( + &request.watermark_key, + &request.future_buffer_ms.to_string(), + &request.invalidated_at_ms.to_string(), + ) + .await + } + .boxed() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cancel::CancelToken; + use crate::codec::Payload; + use crate::remote::{Frame, MissReason}; + use parking_lot::Mutex; + use std::collections::VecDeque; + use std::path::Path; + use std::sync::Arc; + use std::time::Duration; + + /// One recorded dispatch: whether it was pinned to a primary, and its + /// arguments including the command name. + #[derive(Debug, Clone, PartialEq, Eq)] + struct Call { + primary: bool, + args: Vec>, + } + + /// Replays scripted replies in order. A dispatch beyond the script fails + /// with a client error so a test that expected no dispatch fails loudly; + /// [`Scripted::hanging`] makes it wait forever instead. + #[derive(Default)] + struct Scripted { + calls: Mutex>, + replies: Mutex>>, + hang_when_exhausted: bool, + } + + impl Scripted { + fn with(replies: Vec>) -> Arc { + Arc::new(Scripted { + calls: Mutex::new(Vec::new()), + replies: Mutex::new(replies.into_iter().collect()), + hang_when_exhausted: false, + }) + } + + /// A connection whose replies never arrive, for cancellation tests. + fn hanging() -> Arc { + Arc::new(Scripted { + calls: Mutex::new(Vec::new()), + replies: Mutex::new(VecDeque::new()), + hang_when_exhausted: true, + }) + } + + fn calls(&self) -> Vec { + self.calls.lock().clone() + } + + fn dispatch(&self, primary: bool, cmd: &Cmd) -> BoxFuture<'static, RedisResult> { + let args = cmd + .args_iter() + .map(|arg| match arg { + redis::Arg::Simple(bytes) => bytes.to_vec(), + _ => b"".to_vec(), + }) + .collect(); + self.calls.lock().push(Call { primary, args }); + match self.replies.lock().pop_front() { + Some(reply) => future::ready(reply).boxed(), + // A dispatch beyond the script is a test failure, not a hang, + // unless the test asked for a reply that never arrives. + None if self.hang_when_exhausted => future::pending().boxed(), + None => future::ready(Err(redis::RedisError::from(( + redis::ErrorKind::Io, + "unexpected dispatch beyond the scripted replies", + )))) + .boxed(), + } + } + } + + impl RedisConnection for Arc { + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + self.dispatch(false, &cmd) + } + + fn run_on_primary(&self, _key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + self.dispatch(true, &cmd) + } + } + + fn failure() -> RedisResult { + Err(redis::RedisError::from(( + redis::ErrorKind::Io, + "ambiguous response loss", + ))) + } + + fn strings(args: &[Vec]) -> Vec { + args.iter() + .map(|arg| String::from_utf8_lossy(arg).into_owned()) + .collect() + } + + fn block_on(future: impl std::future::Future) -> T { + futures::executor::block_on(future) + } + + fn tracked(value_key: &str, watermark_key: &str) -> ReadRequest { + ReadRequest { + value_key: value_key.to_string(), + watermark_key: Some(watermark_key.to_string()), + } + } + + fn context() -> ReadContext { + ReadContext { + timeout_ms: 50, + cancel: CancelToken::new(), + } + } + + fn go_invalidation_script() -> String { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../go/redis_adapter.go"); + let source = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + let start = source + .find("const InvalidationScript = `") + .expect("Go declares InvalidationScript") + + "const InvalidationScript = `".len(); + let end = source[start..] + .find('`') + .expect("Go raw string literal closes"); + source[start..start + end].to_string() + } + + #[test] + fn invalidation_script_matches_go_byte_for_byte() { + assert_eq!(INVALIDATION_SCRIPT, go_invalidation_script()); + } + + #[test] + fn invalidation_script_sha1_matches_independent_digests() { + let sha = invalidation_script_sha1(); + // Pinned from `shasum -a 1` over the Go constant. TypeScript's + // INVALIDATE_CACHE_SCRIPT_SHA1 (e90603d37a1a3d7a49ae5417bea0d595b94ddec6) + // hashes the same Lua with different formatting. + assert_eq!(sha, "a6c1c661884bd7f535a13c79135a40b1edd2e216"); + assert_eq!(sha, redis::Script::new(INVALIDATION_SCRIPT).get_hash()); + } + + #[test] + fn write_is_one_set_with_px_and_validates_ok() { + let frame = Frame { + created_at_ms: 1_700_000_000_000, + payload: Payload::binary(vec![0, 1, 255]), + }; + let connection = Scripted::with(vec![Ok(Value::Okay)]); + let adapter = RedisAdapter::new(connection.clone()); + block_on(adapter.write(WriteRequest { + value_key: "value".into(), + frame: frame.clone(), + ttl_ms: 2, + })) + .expect("SET succeeds"); + let calls = connection.calls(); + assert_eq!(calls.len(), 1); + assert!(!calls[0].primary); + assert_eq!( + calls[0].args, + vec![ + b"SET".to_vec(), + b"value".to_vec(), + encode_frame(&frame).unwrap(), + b"PX".to_vec(), + b"2".to_vec(), + ] + ); + + for ttl_ms in [0, MAX_SUPPORTED_DURATION_MS + 1] { + let error = block_on(adapter.write(WriteRequest { + value_key: "value".into(), + frame: frame.clone(), + ttl_ms, + })) + .expect_err("invalid TTL rejected"); + assert!(error.is::(), "{error}"); + } + assert_eq!(connection.calls().len(), 1, "invalid input dispatched"); + + for reply in [ + Value::Nil, + Value::SimpleString("ok".into()), + Value::Boolean(true), + Value::Int(1), + ] { + assert!(validate_set_reply(&reply).is_err(), "{reply:?}"); + } + for reply in [ + Value::Okay, + Value::SimpleString("OK".into()), + Value::BulkString(b"OK".to_vec()), + ] { + assert!(validate_set_reply(&reply).is_ok(), "{reply:?}"); + } + } + + #[test] + fn write_reports_protocol_error_on_bad_reply() { + let connection = Scripted::with(vec![Ok(Value::Int(1))]); + let adapter = RedisAdapter::new(connection); + let error = block_on(adapter.write(WriteRequest { + value_key: "value".into(), + frame: Frame { + created_at_ms: 1, + payload: Payload::text("1"), + }, + ttl_ms: 1, + })) + .expect_err("bad reply rejected"); + assert!(error.is::(), "{error}"); + } + + #[test] + fn invalidate_retries_evalsha_once_as_eval_with_identical_arguments() { + let connection = Scripted::with(vec![failure(), Ok(Value::Int(1))]); + let adapter = RedisAdapter::new(connection.clone()); + block_on(adapter.invalidate(InvalidateRequest { + watermark_key: "watermark".into(), + invalidated_at_ms: 1_700_000_000_000, + future_buffer_ms: 100, + })) + .expect("EVAL retry succeeds"); + let calls = connection.calls(); + assert_eq!(calls.len(), 2, "{calls:?}"); + let first = strings(&calls[0].args); + let second = strings(&calls[1].args); + assert_eq!(first[0], "EVALSHA"); + assert_eq!(first[1], invalidation_script_sha1()); + assert_eq!(second[0], "EVAL"); + assert_eq!(second[1], INVALIDATION_SCRIPT); + assert_eq!(first[2..], second[2..], "retry changed logical arguments"); + assert_eq!( + first[2..], + ["1", "watermark", "100", "1700000000000"].map(str::to_string) + ); + } + + #[test] + fn invalidate_failed_retry_surfaces_and_stops() { + let connection = Scripted::with(vec![failure(), failure()]); + let adapter = RedisAdapter::new(connection.clone()); + let error = block_on(adapter.invalidate_decimal("watermark", "0", "1")) + .expect_err("second failure surfaces"); + assert!(error.is::(), "{error}"); + assert_eq!(connection.calls().len(), 2); + } + + #[test] + fn invalidate_accepted_invalid_reply_is_not_retried() { + let connection = Scripted::with(vec![Ok(Value::BulkString(b"1".to_vec()))]); + let adapter = RedisAdapter::new(connection.clone()); + let error = block_on(adapter.invalidate(InvalidateRequest { + watermark_key: "watermark".into(), + invalidated_at_ms: 1, + future_buffer_ms: 0, + })) + .expect_err("string reply rejected"); + assert!(error.is::(), "{error}"); + assert_eq!( + connection.calls().len(), + 1, + "invalid accepted reply retried" + ); + + for reply in [ + Value::Nil, + Value::BulkString(Vec::new()), + Value::Int(0), + Value::Double(1.0), + Value::BulkString(b"1".to_vec()), + Value::SimpleString("1".into()), + Value::Boolean(true), + Value::Okay, + ] { + assert!(validate_invalidation_reply(&reply).is_err(), "{reply:?}"); + } + assert!(validate_invalidation_reply(&Value::Int(1)).is_ok()); + } + + #[test] + fn invalidate_rejects_out_of_domain_arguments_before_dispatch() { + let connection = Scripted::with(vec![]); + let adapter = RedisAdapter::new(connection.clone()); + for (invalidated_at_ms, future_buffer_ms) in [ + (0, MAX_SUPPORTED_DURATION_MS + 1), + (MAX_SAFE_INTEGER, 1), + (MAX_SAFE_INTEGER + 1, 0), + ] { + let error = block_on(adapter.invalidate(InvalidateRequest { + watermark_key: "watermark".into(), + invalidated_at_ms, + future_buffer_ms, + })) + .expect_err("invalid domain rejected"); + assert!(error.is::(), "{error}"); + } + assert!(connection.calls().is_empty()); + } + + #[test] + fn tracked_read_is_one_primary_mget_classified_by_the_protocol() { + let raw = encode_frame(&Frame { + created_at_ms: 2, + payload: Payload::text("1"), + }) + .unwrap(); + let bulk = |bytes: &[u8]| Value::BulkString(bytes.to_vec()); + struct Case { + reply: Value, + expected: Result, + } + let hit = || { + Ok(ReadResult::Hit(Frame { + created_at_ms: 2, + payload: Payload::text("1"), + })) + }; + let miss = |reason, observed| { + Ok(ReadResult::Miss { + reason, + observed_watermark_ms: observed, + }) + }; + let cases = [ + Case { + reply: Value::Array(vec![bulk(&raw), bulk(b"1")]), + expected: hit(), + }, + Case { + reply: Value::Array(vec![bulk(&raw), bulk(b"2")]), + expected: miss(MissReason::WatermarkFenced, Some(2)), + }, + Case { + reply: Value::Array(vec![Value::Nil, bulk(b"5")]), + expected: miss(MissReason::ValueAbsent, Some(5)), + }, + Case { + reply: Value::Array(vec![bulk(&raw), Value::Nil]), + expected: hit(), + }, + Case { + reply: Value::Array(vec![bulk(&raw), bulk(b"bad")]), + expected: miss(MissReason::Unclassified, None), + }, + Case { + reply: Value::Array(vec![bulk(&raw)]), + expected: Err(()), + }, + Case { + reply: Value::Array(vec![Value::Int(12), Value::Nil]), + expected: Err(()), + }, + Case { + reply: Value::Array(vec![bulk(&raw), Value::Boolean(false)]), + expected: Err(()), + }, + Case { + reply: bulk(&raw), + expected: Err(()), + }, + ]; + for case in cases { + let connection = Scripted::with(vec![Ok(case.reply.clone())]); + let adapter = RedisAdapter::new(connection.clone()); + let got = block_on(adapter.read(tracked("value", "watermark"), context())); + match (&case.expected, &got) { + (Ok(expected), Ok(got)) => assert_eq!(got, expected, "{:?}", case.reply), + (Err(()), Err(error)) => { + assert!( + error.is::(), + "{:?}: {error}", + case.reply + ) + } + _ => panic!("reply {:?}: {got:?}", case.reply), + } + assert_eq!( + connection.calls(), + vec![Call { + primary: true, + args: vec![b"MGET".to_vec(), b"value".to_vec(), b"watermark".to_vec()], + }] + ); + } + } + + #[test] + fn untracked_read_is_one_get() { + let raw = encode_frame(&Frame { + created_at_ms: 7, + payload: Payload::binary(vec![0, 255]), + }) + .unwrap(); + let connection = Scripted::with(vec![Ok(Value::Nil), Ok(Value::BulkString(raw))]); + let adapter = RedisAdapter::new(connection.clone()); + let request = ReadRequest { + value_key: "value".into(), + watermark_key: None, + }; + assert_eq!( + block_on(adapter.read(request.clone(), context())).unwrap(), + ReadResult::miss(MissReason::ValueAbsent) + ); + assert_eq!( + block_on(adapter.read(request, context())).unwrap(), + ReadResult::Hit(Frame { + created_at_ms: 7, + payload: Payload::binary(vec![0, 255]), + }) + ); + let calls = connection.calls(); + assert_eq!(calls.len(), 2); + for call in calls { + assert_eq!( + call, + Call { + primary: false, + args: vec![b"GET".to_vec(), b"value".to_vec()], + } + ); + } + } + + #[test] + fn read_payload_encoding_error_is_an_error_not_a_miss() { + let mut raw = encode_frame(&Frame { + created_at_ms: 2, + payload: Payload::text("1"), + }) + .unwrap(); + raw[9] = 7; + let connection = Scripted::with(vec![Ok(Value::BulkString(raw))]); + let adapter = RedisAdapter::new(connection); + let error = block_on(adapter.read( + ReadRequest { + value_key: "value".into(), + watermark_key: None, + }, + context(), + )) + .expect_err("unknown encoding tag is an error"); + assert_eq!( + error.downcast_ref::(), + Some(&ProtocolError::PayloadEncoding) + ); + } + + #[test] + fn read_honors_cancellation_before_and_during_the_wait() { + let connection = Scripted::hanging(); + let adapter = RedisAdapter::new(connection.clone()); + let cancelled = ReadContext { + timeout_ms: 50, + cancel: CancelToken::new(), + }; + cancelled.cancel.cancel(); + let error = block_on(adapter.read(tracked("value", "watermark"), cancelled)) + .expect_err("pre-cancelled read never dispatches"); + assert!(error.is::(), "{error}"); + assert!(connection.calls().is_empty()); + + let context = context(); + let token = context.cancel.clone(); + let canceller = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(20)); + token.cancel(); + }); + let error = block_on(adapter.read(tracked("value", "watermark"), context)) + .expect_err("hung command is abandoned on cancel"); + canceller.join().unwrap(); + assert!(error.is::(), "{error}"); + assert_eq!( + connection.calls().len(), + 1, + "the command was dispatched once" + ); + } +} diff --git a/rust/src/remote.rs b/rust/src/remote.rs new file mode 100644 index 00000000..e99027f0 --- /dev/null +++ b/rust/src/remote.rs @@ -0,0 +1,160 @@ +//! The remote (Redis) adapter boundary. + +use futures::future::BoxFuture; + +use crate::cancel::CancelToken; +use crate::codec::Payload; +use crate::error::BoxError; + +/// One stored value: the writer's wall-clock stamp and the serialized payload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + /// Epoch milliseconds from the writer's clock; a nonnegative safe integer. + pub created_at_ms: u64, + /// The stored payload, envelope included; + /// [`decompress_payload`](crate::protocol::decompress_payload) unwraps it. + pub payload: Payload, +} + +/// Bounded cause of a semantic miss. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MissReason { + /// The layer had no retrievable value. + ValueAbsent, + /// A valid frame's logical age reached its ceiling. + Expired, + /// A tracked frame was rejected by an invalidation watermark. + WatermarkFenced, + /// A real miss with no decisive cause: malformed frames, unknown replies, + /// invalid or future timestamps, decode failures. + Unclassified, +} + +impl MissReason { + /// The `snake_case` label value shared with the TypeScript and Go ports. + pub fn as_str(self) -> &'static str { + match self { + MissReason::ValueAbsent => "value_absent", + MissReason::Expired => "expired", + MissReason::WatermarkFenced => "watermark_fenced", + MissReason::Unclassified => "unclassified", + } + } + + /// The inverse of [`as_str`](Self::as_str). `None` for any other text, + /// which callers treat as [`MissReason::Unclassified`]. + pub fn parse(text: &str) -> Option { + Some(match text { + "value_absent" => MissReason::ValueAbsent, + "expired" => MissReason::Expired, + "watermark_fenced" => MissReason::WatermarkFenced, + "unclassified" => MissReason::Unclassified, + _ => return None, + }) + } +} + +/// Semantic result of one remote read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReadResult { + /// A complete frame that passed decoding and, for tracked keys, the fence. + Hit(Frame), + /// A classified miss. `observed_watermark_ms` is the valid invalidation + /// watermark read atomically with a tracked value, when one existed; a + /// refill stamped at or before it is known to remain unreadable. + Miss { + /// Why no servable frame was produced. + reason: MissReason, + /// The fence described on the variant; always `None` for untracked + /// keys once normalized. + observed_watermark_ms: Option, + }, +} + +impl ReadResult { + /// A miss that observed no watermark. + pub fn miss(reason: MissReason) -> Self { + ReadResult::Miss { + reason, + observed_watermark_ms: None, + } + } + + /// `true` for [`ReadResult::Miss`] of any reason. + pub fn is_miss(&self) -> bool { + matches!(self, ReadResult::Miss { .. }) + } +} + +/// Keys of one read: the value key and, for tracked identities, the watermark key. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReadRequest { + /// The stored value key: the logical key plus the frame suffix. + pub value_key: String, + /// The entity watermark key, present only for tracked identities, whose + /// value and watermark must be read atomically. + pub watermark_key: Option, +} + +/// The effective read budget and a cooperative cancellation request. +/// +/// The cache's own deadline remains authoritative; adapters may use the token +/// to stop waiting, but a dispatched command may still execute. +#[derive(Debug, Clone)] +pub struct ReadContext { + /// The resolved read budget in milliseconds; the cache stops waiting and + /// cancels the token once it elapses. + pub timeout_ms: u64, + /// Cancelled by the cache when the budget elapses; adapters may stop + /// waiting on it. + pub cancel: CancelToken, +} + +/// One complete-frame write: a single native `SET value_key frame PX ttl_ms`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WriteRequest { + /// The stored value key to `SET`. + pub value_key: String, + /// The complete frame, stamped by the cache; store it exactly. + pub frame: Frame, + /// Positive milliseconds no greater than 365 days. + pub ttl_ms: u64, +} + +/// One invalidation: advance the entity watermark monotonically. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InvalidateRequest { + /// The entity watermark key, `{namespace:keyType:id}#watermark`. + pub watermark_key: String, + /// The invalidating process's wall clock sample; reused through retries. + pub invalidated_at_ms: u64, + /// Nonnegative milliseconds no greater than 365 days. + pub future_buffer_ms: u64, +} + +/// Caller-owned semantic Redis boundary. +/// +/// The cache borrows the adapter and never connects, drains or closes it. +/// Tracked reads must observe the value and watermark atomically from an +/// authoritative primary. Writes are one native `SET` of a complete frame +/// stamped by the cache; they never create or extend watermarks. +/// Invalidation runs the shared Lua transition. Use +/// [`crate::protocol::decode_frame`] and [`crate::protocol::encode_frame`] +/// for the wire format. +pub trait Remote: Send + Sync + 'static { + /// Read one value, and for tracked requests its watermark atomically from + /// a primary, then classify the result. An `Err` counts as a `cache_read` + /// error and the call falls through to the source. + fn read( + &self, + request: ReadRequest, + context: ReadContext, + ) -> BoxFuture<'_, Result>; + /// Store the complete frame under `value_key` with `ttl_ms` retention as + /// one native `SET ... PX`, without touching watermarks. + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>>; + /// Advance the entity watermark to at least + /// `invalidated_at_ms + future_buffer_ms`, only ever widening its retention. + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>>; +} diff --git a/rust/src/runtime.rs b/rust/src/runtime.rs new file mode 100644 index 00000000..a3ad71b0 --- /dev/null +++ b/rust/src/runtime.rs @@ -0,0 +1,87 @@ +//! Detached task admission and timers. + +use std::time::Duration; + +use futures::future::BoxFuture; + +/// Runs detached work the cache does not await and supplies its timers. +/// +/// Detached work is raw source or adapter work that outlives a caller's +/// deadline, and shadow validation jobs. Timers deliver deadlines; a +/// controlled runtime may deliver them on its own schedule. +pub trait Runtime: Send + Sync + 'static { + /// Start `task` concurrently with the caller. + fn spawn(&self, task: BoxFuture<'static, ()>); + /// Start `task` after work that is already runnable has progressed. + /// + /// Production runtimes treat this like [`Runtime::spawn`]. A controlled + /// test runtime runs deferred work only once everything else is blocked. + fn defer(&self, task: BoxFuture<'static, ()>) { + self.spawn(task) + } + /// Admit synchronous CPU work without blocking the async executor. + /// + /// The default uses a process-wide pool of two threads and two queued jobs. + /// A full queue or thread-creation failure returns an error; cache plumbing + /// fails open. Once admitted, a job owns its resources until execution or + /// destruction, independently of the future waiting for its result. + /// Custom runtimes may override this with their own bounded CPU executor. + /// A controlled test runtime may queue the job for deterministic execution. + fn spawn_blocking( + &self, + task: Box, + ) -> Result<(), crate::BoxError> { + crate::blocking::submit(task) + } + /// A timer that completes once `duration` has passed. + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()>; +} + +/// Spawns onto one tokio runtime, captured as a handle at construction, and +/// uses that runtime's timers. +/// +/// The default instance is [`TokioRuntime::current`], taken while the cache is +/// built; [`DialCacheBuilder::build`](crate::DialCacheBuilder::build) fails +/// with a configuration error outside a tokio context instead of panicking on +/// first use. The runtime must have its time driver enabled (tokio's +/// `enable_time` or `enable_all`), or the first deadline panics inside the +/// detached work and surfaces as [`Error::Panic`](crate::Error::Panic). +#[cfg(feature = "tokio")] +#[derive(Debug, Clone)] +pub struct TokioRuntime { + handle: tokio::runtime::Handle, +} + +#[cfg(feature = "tokio")] +impl TokioRuntime { + /// The runtime of the current tokio context. + pub fn current() -> Result { + tokio::runtime::Handle::try_current() + .map(Self::from_handle) + .map_err(|_| { + crate::error::ConfigError::invalid( + "DialCache requires a tokio runtime: build the cache inside one, or \ + configure DialCacheBuilder::runtime with TokioRuntime::from_handle", + ) + }) + } + + /// Use the runtime behind `handle`, whichever context later calls the cache. + pub fn from_handle(handle: tokio::runtime::Handle) -> Self { + TokioRuntime { handle } + } +} + +#[cfg(feature = "tokio")] +impl Runtime for TokioRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.handle.spawn(task); + } + + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + // Enter the captured runtime so the timer binds to its time driver + // rather than to whichever runtime happens to call the cache. + let _enter = self.handle.enter(); + Box::pin(tokio::time::sleep(duration)) + } +} diff --git a/rust/src/scope.rs b/rust/src/scope.rs new file mode 100644 index 00000000..923cc0e0 --- /dev/null +++ b/rust/src/scope.rs @@ -0,0 +1,119 @@ +//! Enabled scopes and request memoization. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use parking_lot::Mutex; + +use crate::flight::Flight; +use crate::local::StoredValue; + +pub(crate) struct OwnerState { + pub(crate) live: bool, + pub(crate) memo: HashMap, + pub(crate) flights: HashMap>, +} + +/// The outermost enabled scope's request state: memo and registered request flights. +pub(crate) struct Owner { + pub(crate) state: Mutex, +} + +impl fmt::Debug for Owner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Owner") + .field("live", &self.is_live()) + .finish() + } +} + +impl Owner { + pub(crate) fn new() -> Arc { + Arc::new(Owner { + state: Mutex::new(OwnerState { + live: true, + memo: HashMap::new(), + flights: HashMap::new(), + }), + }) + } + + pub(crate) fn is_live(&self) -> bool { + self.state.lock().live + } + + pub(crate) fn close(&self) { + let retired = { + let mut state = self.state.lock(); + state.live = false; + ( + std::mem::take(&mut state.memo), + std::mem::take(&mut state.flights), + ) + }; + // Memoized values drop here, outside the lock: a value's destructor + // may call back into the cache. + drop(retired); + } +} + +/// A caching scope handle. +/// +/// Caching is disabled by default. [`DialCache::enable`](crate::DialCache::enable) +/// opens the outermost enabled scope and hands its `Scope` to the callback; +/// pass it to every cached call made on behalf of that request. Nested +/// [`DialCache::enable_in`](crate::DialCache::enable_in) and +/// [`DialCache::disable_in`](crate::DialCache::disable_in) derive child +/// scopes that share the outer request memo. When the outermost callback +/// completes its scope closes: retained clones no longer enable caching and +/// late work cannot publish into the request memo. +/// +/// [`Scope::outside`] is the pass-through scope of code that runs on behalf of +/// no request: calls made with it invoke their source directly. +#[derive(Clone)] +pub struct Scope { + pub(crate) cache_id: u64, + pub(crate) owner: Option>, + pub(crate) enabled: bool, +} + +impl fmt::Debug for Scope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Scope") + .field("cache_id", &self.cache_id) + .field("enabled", &self.is_enabled()) + .field("has_owner", &self.owner.is_some()) + .finish() + } +} + +impl Scope { + /// The scope of work that runs on behalf of no request. Calls pass + /// straight through to their source. + pub fn outside() -> Scope { + Scope { + cache_id: 0, + owner: None, + enabled: false, + } + } + + /// Whether caching is enabled for calls made with this scope: it was + /// derived from an enabled scope and its outermost scope is still open. + pub fn is_enabled(&self) -> bool { + self.enabled && self.owner.as_ref().is_some_and(|owner| owner.is_live()) + } + + pub(crate) fn live_owner(&self) -> Option> { + self.owner.as_ref().filter(|owner| owner.is_live()).cloned() + } + + pub(crate) fn disabled_view(&self) -> Scope { + Scope { + cache_id: self.cache_id, + owner: self.owner.clone(), + enabled: false, + } + } +} diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs new file mode 100644 index 00000000..6c945742 --- /dev/null +++ b/rust/src/shadow.rs @@ -0,0 +1,478 @@ +//! Detached shadow validation: dark reads and fills on ramped-down remote +//! serving, and served-hit re-validation against the source of truth. + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::{select, Either}; +use futures::FutureExt; + +use crate::deadline::{await_deadline, since}; +use crate::engine::Core; +use crate::error::Error; +use crate::execution::{call_load, Execution, RawRead, ReadSnapshot}; +use crate::flight::{start_pending, yield_deferred, Settled, ValueResult}; +use crate::limits::DEFAULT_FALLBACK_TIMEOUT_MS; +use crate::local::StoredValue; +use crate::observe::{ + ErrorKind, Event, Layer, LogEvent, MissReason, ShadowMismatchDetails, ShadowOutcome, +}; +use crate::preview::{preview_key, preview_value}; +use crate::remote::Frame; + +/// One admitted shadow job holding an instance slot. +/// One held entry of the instance-wide shadow table. Dropping it frees the +/// slot, so a job dropped unpolled at runtime shutdown cannot pin capacity. +pub(crate) struct ShadowSlot { + core: Arc, + key: String, + flight: Arc, +} + +impl Drop for ShadowSlot { + fn drop(&mut self) { + let mut state = self.core.state.lock(); + if state + .shadows + .get(&self.key) + .is_some_and(|f| Arc::ptr_eq(f, &self.flight)) + { + state.shadows.remove(&self.key); + } + } +} + +pub(crate) struct ShadowFlight { + abandoned: AtomicBool, + stop: Settled<()>, +} + +impl ShadowFlight { + fn new() -> Arc { + Arc::new(ShadowFlight { + abandoned: AtomicBool::new(false), + stop: Settled::new(), + }) + } + + fn abandon(&self) { + self.abandoned.store(true, Ordering::SeqCst); + self.stop.settle(()); + } + + fn is_abandoned(&self) -> bool { + self.abandoned.load(Ordering::SeqCst) + } +} + +/// The deadline and owned capacity of one shadow job. CPU closures retain a +/// clone so runtime teardown cannot release capacity while raw work continues. +#[derive(Clone)] +pub(crate) struct ShadowWork { + flight: Arc, + slot: Arc, + started: Duration, + budget: Duration, +} + +impl ShadowWork { + pub(crate) fn expired(&self) -> bool { + if since(self.slot.core.clock.as_ref(), self.started) >= self.budget { + self.flight.abandon(); + } + self.flight.is_abandoned() + } +} + +#[derive(Clone)] +struct Verdict { + outcome: ShadowOutcome, + age_ms: Option, + compared: Option<(StoredValue, StoredValue)>, +} + +impl Verdict { + fn of(outcome: ShadowOutcome) -> Verdict { + Verdict { + outcome, + age_ms: None, + compared: None, + } + } +} + +impl Execution { + /// Run the caller's source while a dark shadow job reads and may fill the + /// ramped-down remote layer without delaying the caller. + pub(crate) async fn dark_source( + self: &Arc, + layer: Layer, + local_miss: bool, + ) -> ValueResult { + let start = self.elapsed(); + let source: Settled = start_pending( + self.core.runtime.as_ref(), + { + let x = self.clone(); + async move { x.source(layer).await } + }, + |message| Err(Error::Panic(message)), + ); + self.schedule_shadow(None, Some(source.clone()), start); + let result = source.wait().await; + if let Ok(value) = &result { + if local_miss { + self.put_local(value.clone()); + } + } + result + } + + /// Admit one shadow job for this key if policy, the outcome hook and + /// instance capacity allow it. + pub(crate) fn schedule_shadow( + self: &Arc, + frame: Option>, + source: Option>, + started: Duration, + ) { + let p = self.policy.shadow; + if p.config_error { + self.error_event(Layer::Remote, ErrorKind::ConfigResolution, false); + return; + } + if !p.enabled || !self.core.shadow_hook_enabled() { + return; + } + let flight = { + let mut state = self.core.state.lock(); + if state.shadows.contains_key(&self.keys.logical) + || state.shadows.len() >= self.core.shadow_max_in_flight + { + drop(state); + self.shadow_event(&Verdict::of(ShadowOutcome::Dropped)); + return; + } + let flight = ShadowFlight::new(); + state + .shadows + .insert(self.keys.logical.clone(), flight.clone()); + flight + }; + let slot = ShadowSlot { + core: self.core.clone(), + key: self.keys.logical.clone(), + flight: flight.clone(), + }; + if p.logging_config_error { + self.error_event(Layer::Remote, ErrorKind::ConfigResolution, false); + } + let x = self.clone(); + self.core.runtime.defer(Box::pin(async move { + x.run_shadow(flight, frame, source, started, slot).await + })); + } + + fn shadow_event(&self, verdict: &Verdict) { + self.emit(Event::ShadowValidation { + labels: self.labels.clone(), + outcome: verdict.outcome, + }); + if let Some(age) = verdict.age_ms { + self.emit(Event::ShadowValueAge { + labels: self.labels.clone(), + outcome: verdict.outcome, + seconds: age.max(0) as f64 / 1000.0, + }); + } + } + + async fn log_shadow_mismatch(&self, verdict: Verdict, slot: Arc) { + if verdict.outcome != ShadowOutcome::Mismatch || !self.policy.shadow.log_mismatches { + return; + } + let (cached, source) = match (self.op.metadata.preview.clone(), verdict.compared) { + (Some(preview), Some((cached, source))) => { + let cpu_slot = slot.clone(); + crate::blocking::run(self.core.runtime.as_ref(), move || { + // Raw diagnostic work keeps capacity even if the awaiting + // async task is dropped during runtime shutdown. + let _slot = cpu_slot; + let render = |value: &StoredValue| { + catch_unwind(AssertUnwindSafe(|| preview(value))) + .ok() + .flatten() + .map(|json| preview_value(&json)) + }; + Ok((render(&cached), render(&source))) + }) + .await + .unwrap_or_default() + } + _ => (None, None), + }; + self.core + .log(LogEvent::ShadowMismatch(ShadowMismatchDetails { + namespace: self.labels.namespace.clone(), + use_case: self.labels.use_case.clone(), + key_type: self.labels.key_type.clone(), + cache_key: preview_key(&self.keys.logical), + cached_value_json: cached, + source_value_json: source, + })); + drop(slot); + } + + async fn run_shadow( + self: Arc, + flight: Arc, + frame: Option>, + source: Option>, + started: Duration, + slot: ShadowSlot, + ) { + let clock = self.core.clock.clone(); + let started = if source.is_none() { + clock.elapsed() + } else { + started + }; + let budget_ms = self + .op + .metadata + .budget + .millis() + .unwrap_or(DEFAULT_FALLBACK_TIMEOUT_MS); + let slot = Arc::new(slot); + let work = ShadowWork { + flight: flight.clone(), + slot: slot.clone(), + started, + budget: Duration::from_millis(budget_ms), + }; + let log_slot = slot.clone(); + let validation: Settled = start_pending( + self.core.runtime.as_ref(), + { + let x = self.clone(); + async move { + let mut reads = Vec::new(); + let verdict = + match AssertUnwindSafe(x.clone().validate(work, frame, source, &mut reads)) + .catch_unwind() + .await + { + Ok(verdict) => verdict, + Err(_) => Verdict::of(ShadowOutcome::Timeout), + }; + // Reads keep the slot after a read timeout; owned source, codec and + // write work already keeps this operation running until raw completion. + // Dropping the slot frees it, whether the reads settled or the + // runtime dropped this task first. + let runtime = x.core.runtime.clone(); + runtime.spawn(Box::pin(async move { + for read in reads { + let _ = read.wait().await; + } + drop(slot); + })); + verdict + } + }, + |_| Verdict::of(ShadowOutcome::Timeout), + ); + let verdict = await_deadline( + clock.as_ref(), + self.core.runtime.as_ref(), + &validation, + started, + Some(budget_ms), + || { + flight.abandon(); + Verdict::of(ShadowOutcome::Timeout) + }, + ) + .await; + self.shadow_event(&verdict); + self.log_shadow_mismatch(verdict, log_slot).await; + } + + async fn shadow_read( + &self, + reads: &mut Vec>, + max_age: bool, + retain_future: bool, + ) -> RawRead { + let begin = self.elapsed(); + self.emit(Event::Request { + labels: self.labels(Layer::RemoteShadow), + }); + let (bounded, raw) = self.raw_read(); + reads.push(raw); + let result = bounded.await; + let result = match result { + Err(error) => { + let kind = if error + .downcast_ref::() + .is_some() + { + ErrorKind::CacheReadTimeout + } else { + ErrorKind::CacheRead + }; + self.error_event(Layer::RemoteShadow, kind, false); + Err(error) + } + Ok(mut read) => { + if let ReadSnapshot::Hit(frame) = &read { + let (age, valid) = self.frame_age(frame, Layer::RemoteShadow); + if !valid && !(retain_future && age < 0) { + read = ReadSnapshot::miss(MissReason::Unclassified); + } else if max_age && valid && age as u64 >= self.policy.remote.ttl_ms { + read = ReadSnapshot::miss(MissReason::Expired); + } + } + if let ReadSnapshot::Miss { reason, .. } = &read { + self.emit(Event::Miss { + labels: self.labels(Layer::RemoteShadow), + reason: *reason, + }); + } + Ok(read) + } + }; + self.emit(Event::Get { + labels: self.labels(Layer::RemoteShadow), + seconds: self.seconds_since(begin), + }); + result + } + + async fn validate( + self: Arc, + work: ShadowWork, + mut frame: Option>, + source: Option>, + reads: &mut Vec>, + ) -> Verdict { + let expired = || work.expired(); + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + let mut fill = false; + let mut fence: Option = None; + if source.is_some() { + let read = match self.shadow_read(reads, true, false).await { + Ok(read) => read, + Err(_) => return Verdict::of(ShadowOutcome::RedisError), + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + match read { + ReadSnapshot::Miss { + observed_watermark_ms, + .. + } => { + fill = true; + fence = observed_watermark_ms; + } + ReadSnapshot::Hit(read_frame) => frame = Some(read_frame), + } + } + let value: ValueResult = match &source { + Some(source) => { + let result = match select(source.wait(), work.flight.stop.wait()).await { + Either::Left((result, _)) => result, + Either::Right(((), _)) => return Verdict::of(ShadowOutcome::Timeout), + }; + if result.is_ok() { + // Let the caller finish its own continuation before any shadow + // decode, comparison or dump work. + yield_deferred(self.core.runtime.as_ref()).await; + } + result + } + None => call_load(&self.op, self.scope.disabled_view()).await, + }; + let value = match value { + Ok(value) => value, + Err(Error::FallbackTimeout(_)) if source.is_some() => { + return Verdict::of(ShadowOutcome::Timeout); + } + Err(_) => return Verdict::of(ShadowOutcome::SourceError), + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + if fill { + let filled = self + .put_remote(&value, fence, Layer::RemoteShadow, Some(&work)) + .await; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + return match filled { + Ok(true) => Verdict::of(ShadowOutcome::Filled), + Ok(false) => Verdict::of(ShadowOutcome::FillFenced), + Err(error) => { + self.core.log(LogEvent::ShadowFillFailed(error)); + Verdict::of(ShadowOutcome::FillError) + } + }; + } + let Some(frame) = frame else { + return Verdict::of(ShadowOutcome::Timeout); + }; + let decoded = self.decode(&frame, Layer::RemoteShadow, Some(&work)).await; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + let cached = match decoded { + Ok(cached) => cached, + Err(_) => return Verdict::of(ShadowOutcome::DeserializationError), + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + let compare = self.op.metadata.compare.clone(); + let matches = match catch_unwind(AssertUnwindSafe(|| compare(&cached, &value))) { + Ok(Ok(matches)) => matches, + _ => return Verdict::of(ShadowOutcome::ComparisonError), + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + let age = self + .wall_ms() + .saturating_sub(frame.created_at_ms.min(i64::MAX as u64) as i64); + if matches { + return Verdict { + outcome: ShadowOutcome::Match, + age_ms: Some(age), + compared: None, + }; + } + let confirmation = match self.shadow_read(reads, false, true).await { + Ok(confirmation) => confirmation, + Err(_) => return Verdict::of(ShadowOutcome::ConfirmationError), + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + match confirmation { + ReadSnapshot::Hit(confirmed) if confirmed.payload.bytes == frame.payload.bytes => {} + _ => return Verdict::of(ShadowOutcome::Superseded), + } + let age = self + .wall_ms() + .saturating_sub(frame.created_at_ms.min(i64::MAX as u64) as i64); + Verdict { + outcome: ShadowOutcome::Mismatch, + age_ms: Some(age), + compared: Some((cached, value)), + } + } +} diff --git a/rust/src/testing.rs b/rust/src/testing.rs new file mode 100644 index 00000000..2f18cc35 --- /dev/null +++ b/rust/src/testing.rs @@ -0,0 +1,423 @@ +//! Deterministic test doubles: a virtual clock and a single-threaded runtime +//! that runs the cache's detached work to quiescence on demand (feature `test-util`). +//! +//! The cache's detached work and timers go through [`Runtime`], so a +//! single-threaded pool can run every task to quiescence after each command +//! (the `causally-ready-v1` settlement of the formal replay) and deliver timers +//! only when a test advances the clock. + +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll, Waker}; +use std::time::Duration; + +use crate::clock::{Clock, SystemClock}; +use crate::runtime::Runtime; +use futures::executor::{LocalPool, LocalSpawner}; +use futures::future::BoxFuture; +use futures::task::LocalSpawnExt; +use parking_lot::Mutex; + +/// Epoch of every controlled history: 2026-09-08T12:00:00.000Z. +pub const WALL_EPOCH_MS: i64 = 1_788_868_800_000; + +impl std::fmt::Debug for VirtualClock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VirtualClock") + .field("wall_ms", &self.wall_ms()) + .field("elapsed", &self.elapsed()) + .finish() + } +} + +struct TimerState { + at_ns: u128, + id: u64, + fired: AtomicBool, + cancelled: AtomicBool, + waker: Mutex>, +} + +struct ClockState { + wall_ns: i128, + elapsed_ns: u128, + scheduler_ns: u128, + next_timer: u64, + timers: Vec>, +} + +/// Virtual wall, elapsed and scheduler clocks. Scheduler time is what timers +/// are registered against; it moves only when a history delivers timers, so a +/// silent shift never fires a deliberately held timer. +pub struct VirtualClock { + state: Mutex, +} + +impl VirtualClock { + /// A clock at `wall_ms` epoch milliseconds with zero elapsed and + /// scheduler time. + pub fn new(wall_ms: i64) -> Arc { + Arc::new(VirtualClock { + state: Mutex::new(ClockState { + wall_ns: wall_ms as i128 * 1_000_000, + elapsed_ns: 0, + scheduler_ns: 0, + next_timer: 0, + timers: Vec::new(), + }), + }) + } + + /// Move wall (and unless `wall_only`, elapsed) time without delivering timers. + pub fn shift(&self, delta_ms: i64, wall_only: bool) { + self.shift_ns(delta_ms as i128 * 1_000_000, wall_only); + } + + /// [`shift`](Self::shift) in nanoseconds; elapsed time saturates at zero. + pub fn shift_ns(&self, delta_ns: i128, wall_only: bool) { + let mut state = self.state.lock(); + state.wall_ns += delta_ns; + if !wall_only { + state.elapsed_ns = (state.elapsed_ns as i128 + delta_ns).max(0) as u128; + } + } + + /// The earliest live timer due at or before `target_ns` scheduler time. + fn pop_due(&self, target_ns: u128) -> Option> { + let mut state = self.state.lock(); + state + .timers + .retain(|t| !t.cancelled.load(Ordering::SeqCst) && !t.fired.load(Ordering::SeqCst)); + let mut best: Option = None; + for (index, timer) in state.timers.iter().enumerate() { + if timer.at_ns <= target_ns { + let better = match best { + None => true, + Some(current) => { + let c = &state.timers[current]; + timer.at_ns < c.at_ns || (timer.at_ns == c.at_ns && timer.id < c.id) + } + }; + if better { + best = Some(index); + } + } + } + best.map(|index| state.timers.remove(index)) + } + + fn set_scheduler(&self, at_ns: u128) { + let mut state = self.state.lock(); + let delta = at_ns.saturating_sub(state.scheduler_ns); + state.wall_ns += delta as i128; + state.elapsed_ns += delta; + state.scheduler_ns = at_ns; + } + + fn scheduler_ns(&self) -> u128 { + self.state.lock().scheduler_ns + } + + /// Timers registered and neither fired nor dropped. + pub fn pending_timers(&self) -> usize { + let state = self.state.lock(); + state + .timers + .iter() + .filter(|t| !t.cancelled.load(Ordering::SeqCst) && !t.fired.load(Ordering::SeqCst)) + .count() + } +} + +impl Clock for VirtualClock { + fn wall_ms(&self) -> i64 { + (self.state.lock().wall_ns.div_euclid(1_000_000)) as i64 + } + + fn elapsed(&self) -> Duration { + let ns = self.state.lock().elapsed_ns; + Duration::from_nanos(ns.min(u64::MAX as u128) as u64) + } +} + +impl VirtualClock { + /// Register a timer against scheduler time. + pub fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + // A millisecond timer must never shorten a fractional remaining budget: + // round the delay up to whole milliseconds, as the reference ports do. + let delay_ns = duration.as_nanos(); + let delay_ms = delay_ns.div_ceil(1_000_000); + let mut state = self.state.lock(); + state.next_timer += 1; + let timer = Arc::new(TimerState { + at_ns: state.scheduler_ns + delay_ms * 1_000_000, + id: state.next_timer, + fired: AtomicBool::new(false), + cancelled: AtomicBool::new(false), + waker: Mutex::new(None), + }); + state.timers.push(timer.clone()); + Box::pin(Sleep { timer }) + } +} + +struct Sleep { + timer: Arc, +} + +impl Future for Sleep { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + if self.timer.fired.load(Ordering::SeqCst) { + return Poll::Ready(()); + } + *self.timer.waker.lock() = Some(cx.waker().clone()); + Poll::Pending + } +} + +impl Drop for Sleep { + fn drop(&mut self) { + self.timer.cancelled.store(true, Ordering::SeqCst); + } +} + +/// Tasks handed to the spawner. Immediate tasks run on the next drain; +/// deferred tasks run only once nothing else is runnable. +#[derive(Default)] +struct Queues { + immediate: Vec>, + deferred: Vec>, +} + +/// The test runtime handle: collects tasks for the pool and registers timers +/// on the virtual clock. +pub struct StepRuntime { + queues: Mutex, + spawned: AtomicU64, + clock: Arc, +} + +impl StepRuntime { + /// A runtime whose timers register on `clock` and whose tasks wait for + /// a [`TestExecutor`] to drain them. + pub fn new(clock: Arc) -> Arc { + Arc::new(StepRuntime { + queues: Mutex::new(Queues::default()), + spawned: AtomicU64::new(0), + clock, + }) + } + + /// Tasks handed over so far, immediate and deferred. + pub fn spawned(&self) -> u64 { + self.spawned.load(Ordering::Relaxed) + } +} + +impl std::fmt::Debug for StepRuntime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StepRuntime") + .field("spawned", &self.spawned()) + .finish() + } +} + +impl Runtime for StepRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.spawned.fetch_add(1, Ordering::Relaxed); + self.queues.lock().immediate.push(task); + } + + fn defer(&self, task: BoxFuture<'static, ()>) { + self.spawned.fetch_add(1, Ordering::Relaxed); + self.queues.lock().deferred.push(task); + } + + fn spawn_blocking( + &self, + task: Box, + ) -> Result<(), crate::BoxError> { + self.spawn(Box::pin(async move { task() })); + Ok(()) + } + + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + self.clock.sleep(duration) + } +} + +/// The controlled executor of one test: a single-threaded pool plus the +/// clock and runtime handle every cache instance under test shares. +pub struct TestExecutor { + pool: LocalPool, + local: LocalSpawner, + polls: Arc, + /// The virtual clock to share with every instance under test. + pub clock: Arc, + /// The runtime handle to pass to every instance under test. + pub runtime: Arc, +} + +impl std::fmt::Debug for TestExecutor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TestExecutor") + .field("wall_ms", &self.clock.wall_ms()) + .finish() + } +} + +impl TestExecutor { + /// A fresh pool with its clock at `wall_ms`; replayed histories use + /// [`WALL_EPOCH_MS`]. + pub fn new(wall_ms: i64) -> Self { + let pool = LocalPool::new(); + let local = pool.spawner(); + let clock = VirtualClock::new(wall_ms); + let runtime = StepRuntime::new(clock.clone()); + TestExecutor { + pool, + local, + polls: Arc::new(AtomicU64::new(0)), + clock, + runtime, + } + } + + /// Actual task polls performed by this executor. A zero-time verification + /// drain must leave this count unchanged when every task is already blocked. + pub fn poll_count(&self) -> u64 { + self.polls.load(Ordering::Relaxed) + } + + fn tracked(&self, future: F) -> impl Future + use { + let polls = self.polls.clone(); + let mut future = Box::pin(future); + futures::future::poll_fn(move |cx| { + polls.fetch_add(1, Ordering::Relaxed); + future.as_mut().poll(cx) + }) + } + + /// Run a future to completion on the pool, draining detached work as it + /// is spawned. Panics if the future stays blocked once nothing is runnable: + /// it would be waiting on a gate only the test can release. + pub fn block_on(&mut self, future: impl Future + 'static) -> R { + let slot: std::rc::Rc>> = + std::rc::Rc::new(std::cell::RefCell::new(None)); + let sink = slot.clone(); + self.local + .spawn_local(self.tracked(async move { + let result = future.await; + *sink.borrow_mut() = Some(result); + })) + .expect("pool accepts tasks"); + self.drain(); + let result = slot.borrow_mut().take(); + result.expect("block_on future stayed blocked on a test-owned gate") + } + + /// Spawn driver-owned work onto the pool. + pub fn spawn(&self, task: impl Future + 'static) { + self.local + .spawn_local(self.tracked(task)) + .expect("pool accepts tasks"); + } + + fn move_immediate(&mut self) -> bool { + let tasks = std::mem::take(&mut self.runtime.queues.lock().immediate); + let moved = !tasks.is_empty(); + for task in tasks { + self.local + .spawn_local(self.tracked(task)) + .expect("pool accepts tasks"); + } + moved + } + + fn move_deferred(&mut self) -> bool { + let tasks = std::mem::take(&mut self.runtime.queues.lock().deferred); + let moved = !tasks.is_empty(); + for task in tasks { + self.local + .spawn_local(self.tracked(task)) + .expect("pool accepts tasks"); + } + moved + } + + /// Run until every task is blocked on a gate, an undelivered timer or a + /// scope gate and nothing is runnable, including deferred work that + /// becomes runnable once everything else has stalled. + pub fn drain(&mut self) { + loop { + loop { + self.pool.run_until_stalled(); + if !self.move_immediate() { + break; + } + } + if !self.move_deferred() { + return; + } + } + } + + /// Advance elapsed and wall time by `ms`. With `deliver`, scheduler time + /// moves too and every timer due on the way fires in order, draining after + /// each; without it the clocks jump silently. + pub fn advance(&mut self, ms: i64, deliver: bool) { + if !deliver { + self.clock.shift(ms, false); + self.drain(); + return; + } + let target = self.clock.scheduler_ns() + (ms.max(0) as u128) * 1_000_000; + loop { + self.drain(); + match self.clock.pop_due(target) { + Some(timer) => { + self.clock.set_scheduler(timer.at_ns); + timer.fired.store(true, Ordering::SeqCst); + if let Some(waker) = timer.waker.lock().take() { + waker.wake(); + } + } + None => { + self.clock.set_scheduler(target); + break; + } + } + } + self.drain(); + } + + /// Advance by fractional microseconds without delivering timers (the + /// local-clock profile's environment ticks). + pub fn advance_micros(&mut self, micros: i64) { + self.clock.shift_ns(micros as i128 * 1_000, false); + self.drain(); + } +} + +/// The production [`SystemClock`] built over a [`VirtualClock`]. +/// +/// Its origin aligns to the shared millisecond grid of the virtual elapsed +/// reading exactly as default instances align to the process grid, so +/// instances constructed at different fractional virtual times share one +/// expiry grid and the default alignment itself is what a controlled history +/// exercises. +pub fn grid_clock(base: &Arc) -> SystemClock { + let monotonic = base.clone(); + let wall = base.clone(); + SystemClock::with_sources( + move || { + let ns = monotonic.state.lock().elapsed_ns; + Duration::from_nanos(ns.min(u64::MAX as u128) as u64) + }, + move || wall.wall_ms(), + ) +} diff --git a/rust/src/use_case.rs b/rust/src/use_case.rs new file mode 100644 index 00000000..a814e0fa --- /dev/null +++ b/rust/src/use_case.rs @@ -0,0 +1,439 @@ +//! Registered, typed use cases. + +use std::future::Future; +use std::marker::PhantomData; +use std::sync::Arc; + +use futures::future::BoxFuture; +use serde::de::DeserializeOwned; +use serde::Serialize; + +use crate::codec::{Codec, JsonCodec}; +use crate::engine::DialCache; +use crate::error::{BoxError, ConfigError, Error}; +use crate::identity::{normalize_args, ArgValue, Identity, IntoKeyId}; +use crate::operation::{ + downcast_value, erase_load, Comparator, ErasedOperation, Operation, OperationMetadata, Preview, + RecoveryPredicate, SourceBudget, +}; +use crate::policy::Policy; +use crate::scope::Scope; + +/// The key of one call: the entity id and any secondary dimensions. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct KeySpec { + /// The entity identifier, spelled as text. + pub id: String, + /// Secondary dimensions; absent values are dropped and names sorted when + /// the key is built. + pub args: Vec<(String, ArgValue)>, +} + +impl KeySpec { + /// A key with `id` and no secondary dimensions. See [`IntoKeyId`] for + /// the shared numeric spelling and supported input types. + pub fn new(id: impl IntoKeyId) -> Self { + KeySpec { + id: id.into_key_id(), + args: Vec::new(), + } + } + + /// Add a secondary key dimension. Absent values are omitted; names are + /// sorted by UTF-16 code units when the key is built. Primitive integers + /// preserve their exact decimal value; floats use JavaScript number + /// spelling, with `f32` promoted to `f64`. Shared references to supported + /// inputs are also accepted. + pub fn arg(mut self, name: impl Into, value: impl Into) -> Self { + self.args.push((name.into(), value.into())); + self + } +} + +impl From for KeySpec { + fn from(id: String) -> Self { + KeySpec::new(id) + } +} + +impl From<&str> for KeySpec { + fn from(id: &str) -> Self { + KeySpec::new(id) + } +} + +macro_rules! key_from_number { + ($($t:ty),*) => { $(impl From<$t> for KeySpec { fn from(id: $t) -> Self { KeySpec::new(id) } })* }; +} +key_from_number!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize, f64, f32); + +impl From<&str> for ArgValue { + fn from(value: &str) -> Self { + ArgValue::Str(value.to_string()) + } +} + +impl From for ArgValue { + fn from(value: String) -> Self { + ArgValue::Str(value) + } +} + +impl> From<&T> for ArgValue { + fn from(value: &T) -> Self { + value.clone().into() + } +} + +impl From for ArgValue { + fn from(value: bool) -> Self { + ArgValue::Bool(value) + } +} + +macro_rules! integer_arg_value { + ($($t:ty),*) => { $(impl From<$t> for ArgValue { + fn from(value: $t) -> Self { ArgValue::Int(i64::from(value)) } + })* }; +} +integer_arg_value!(i8, i16, i32, i64, u8, u16, u32); + +macro_rules! wide_integer_arg_value { + ($($t:ty),*) => { $(impl From<$t> for ArgValue { + fn from(value: $t) -> Self { + match i64::try_from(value) { + Ok(value) => ArgValue::Int(value), + Err(_) => ArgValue::BigInt(value.to_string()), + } + } + })* }; +} +wide_integer_arg_value!(i128, isize, u64, u128, usize); + +impl From for ArgValue { + fn from(value: f64) -> Self { + ArgValue::Number(value) + } +} + +impl From for ArgValue { + fn from(value: f32) -> Self { + ArgValue::Number(f64::from(value)) + } +} + +impl> From> for ArgValue { + fn from(value: Option) -> Self { + match value { + Some(value) => value.into(), + None => ArgValue::Absent, + } + } +} + +type KeyFn = Arc KeySpec + Send + Sync>; +type SourceFn = + Arc BoxFuture<'static, Result> + Send + Sync>; + +/// Builds a registered use case. Finish with [`register`](Self::register) +/// for `serde` JSON values or [`register_custom`](Self::register_custom) +/// with an explicit codec and comparator. +pub struct UseCaseBuilder { + cache: DialCache, + key_type: String, + use_case: String, + tracked: bool, + policy: Policy, + budget: SourceBudget, + codec: Option>>, + comparator: Option>, + should_recover: Option, + preview: Option>, + key: Option>, + source: Option>, +} + +impl std::fmt::Debug for UseCaseBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UseCaseBuilder") + .field("key_type", &self.key_type) + .field("use_case", &self.use_case) + .finish_non_exhaustive() + } +} + +impl UseCaseBuilder +where + Args: Clone + Send + Sync + 'static, + T: Send + Sync + 'static, +{ + pub(crate) fn new(cache: DialCache, key_type: String, use_case: String) -> Self { + UseCaseBuilder { + cache, + key_type, + use_case, + tracked: false, + policy: Policy::default(), + budget: SourceBudget::Default, + codec: None, + comparator: None, + should_recover: None, + preview: None, + key: None, + source: None, + } + } + + /// Static policy captured for every call of this use case. + pub fn policy(mut self, policy: Policy) -> Self { + self.policy = policy; + self + } + + /// Share the entity's invalidation watermark with every tracked variant. + pub fn tracked(mut self, tracked: bool) -> Self { + self.tracked = tracked; + self + } + + /// The source deadline. Defaults to 60 s. + pub fn budget(mut self, budget: SourceBudget) -> Self { + self.budget = budget; + self + } + + /// Replace the JSON codec; required by [`register_custom`](Self::register_custom). + pub fn codec(mut self, codec: Arc>) -> Self { + self.codec = Some(codec); + self + } + + /// Application equality for shadow validation. Defaults to `PartialEq`. + pub fn comparator( + mut self, + comparator: impl Fn(&T, &T) -> bool + Send + Sync + 'static, + ) -> Self { + self.comparator = Some(Arc::new(comparator)); + self + } + + /// Override the instance classifier for stale recovery. + pub fn should_recover( + mut self, + predicate: impl Fn(&Error) -> bool + Send + Sync + 'static, + ) -> Self { + self.should_recover = Some(Arc::new(predicate)); + self + } + + /// Bounded textual preview of a value for mismatch warnings. + /// Runs through [`Runtime::spawn_blocking`](crate::Runtime::spawn_blocking) + /// after confirmation; callbacks may run on a CPU worker thread. + pub fn preview( + mut self, + preview: impl Fn(&T) -> Option + Send + Sync + 'static, + ) -> Self { + self.preview = Some(Arc::new(preview)); + self + } + + /// Select every input dimension that affects the value. Runs only for + /// enabled calls. + pub fn key(mut self, key: impl Fn(&Args) -> KeySpec + Send + Sync + 'static) -> Self { + self.key = Some(Arc::new(key)); + self + } + + /// The source of truth. It receives the call's scope so nested cached + /// calls can participate; served-hit shadow validation invokes it again + /// under disabled caching. + pub fn source(mut self, source: F) -> Self + where + F: Fn(Scope, Args) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + self.source = Some(Arc::new(move |scope, args| Box::pin(source(scope, args)))); + self + } + + fn finish( + self, + codec: Arc>, + comparator: Comparator, + preview: Option>, + ) -> Result, ConfigError> { + let key = self + .key + .ok_or_else(|| ConfigError::invalid("DialCache use case needs a key selector"))?; + let source = self + .source + .ok_or_else(|| ConfigError::invalid("DialCache use case needs a source"))?; + self.policy + .validate() + .map_err(|e| ConfigError::invalid(e.to_string()))?; + if let SourceBudget::Millis(ms) = self.budget { + if !(1..=crate::limits::MAX_DEADLINE_MS).contains(&ms) { + return Err(ConfigError::invalid( + "DialCache source budget is outside its domain", + )); + } + } + self.cache.register_use_case(&self.use_case)?; + let (identity, metadata) = Operation { + identity: Identity::new(self.key_type, "", self.use_case).tracked(self.tracked), + policy: self.policy, + budget: self.budget, + codec, + comparator, + should_recover: self.should_recover, + preview, + } + .erase(); + Ok(UseCase { + inner: Arc::new(UseCaseInner { + cache: self.cache, + identity, + metadata, + key, + source, + }), + _args: PhantomData, + }) + } + + /// Register with an explicit codec and comparator (set with + /// [`codec`](Self::codec) and [`comparator`](Self::comparator)). + pub fn register_custom(self) -> Result, ConfigError> { + let codec = self + .codec + .clone() + .ok_or_else(|| ConfigError::invalid("DialCache use case needs a codec"))?; + let comparator = self + .comparator + .clone() + .ok_or_else(|| ConfigError::invalid("DialCache use case needs a comparator"))?; + let preview = self.preview.clone(); + self.finish(codec, comparator, preview) + } +} + +impl UseCaseBuilder +where + Args: Clone + Send + Sync + 'static, + T: Serialize + DeserializeOwned + PartialEq + Send + Sync + 'static, +{ + /// Register with the JSON codec, `PartialEq` comparison and JSON previews + /// unless overridden. + pub fn register(self) -> Result, ConfigError> { + let codec = self.codec.clone().unwrap_or_else(|| Arc::new(JsonCodec)); + let comparator = self + .comparator + .clone() + .unwrap_or_else(|| Arc::new(|a: &T, b: &T| a == b)); + let preview = self + .preview + .clone() + .or_else(|| Some(Arc::new(crate::preview::json_preview::))); + self.finish(codec, comparator, preview) + } +} + +struct UseCaseInner { + cache: DialCache, + identity: Identity, + metadata: Arc, + key: KeyFn, + source: SourceFn, +} + +/// A registered use case: a typed cached function. +/// +/// Values are shared by reference and must be treated as immutable. +pub struct UseCase { + inner: Arc>, + _args: PhantomData, +} + +impl Clone for UseCase { + fn clone(&self) -> Self { + UseCase { + inner: self.inner.clone(), + _args: PhantomData, + } + } +} + +impl std::fmt::Debug for UseCase { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UseCase") + .field("key_type", &self.inner.identity.key_type) + .field("use_case", &self.inner.identity.use_case) + .finish() + } +} + +impl UseCase +where + Args: Clone + Send + Sync + 'static, + T: Send + Sync + 'static, +{ + /// The operation name given at registration. + pub fn use_case(&self) -> &str { + &self.inner.identity.use_case + } + + /// The entity type given at registration. + pub fn key_type(&self) -> &str { + &self.inner.identity.key_type + } + + /// Get the value for `args` through the configured layers. + pub async fn get(&self, scope: &Scope, args: Args) -> Result, Error> { + let inner = self.inner.clone(); + let identity = inner.identity.clone(); + let provider_identity = identity.clone(); + let provider_args = args.clone(); + let provider_inner = inner.clone(); + let identity_provider = Arc::new(move || -> Result { + let spec = (provider_inner.key)(&provider_args); + let args = normalize_args(spec.args.clone())?; + Ok(Identity { + id: spec.id, + args, + ..provider_identity.clone() + }) + }); + let source_inner = inner.clone(); + let load = erase_load(move |scope: Scope| (source_inner.source)(scope, args.clone())); + let erased = ErasedOperation { + identity, + identity_provider: Some(identity_provider), + metadata: inner.metadata.clone(), + load, + }; + let value = inner.cache.execute(scope, erased).await?; + downcast_value::(value) + } + + /// Invoke the source directly, outside any scope. Equivalent to + /// [`get`](Self::get) with [`Scope::outside`]. + pub async fn get_uncached(&self, args: Args) -> Result, Error> { + self.get(&Scope::outside(), args).await + } +} + +impl DialCache { + /// Start registering a use case. `key_type` names the entity type and + /// `use_case` the operation; the pair identifies cached values. + pub fn use_case( + &self, + key_type: impl Into, + use_case: impl Into, + ) -> UseCaseBuilder + where + Args: Clone + Send + Sync + 'static, + T: Send + Sync + 'static, + { + UseCaseBuilder::new(self.clone(), key_type.into(), use_case.into()) + } +} diff --git a/rust/tests/callback_boundaries.rs b/rust/tests/callback_boundaries.rs new file mode 100644 index 00000000..698250e3 --- /dev/null +++ b/rust/tests/callback_boundaries.rs @@ -0,0 +1,473 @@ +//! Native callback boundaries: a callback may panic while constructing its +//! future, before the asynchronous effects exercised by the portable replay. + +use std::future::{ready, Ready}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; + +use dialcache::observe::{ErrorKind, Layer, RecoveryOutcome, ShadowOutcome}; +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, DialCache, Event, Frame, FromSync, Identity, InvalidateRequest, JsonCodec, + MissReason, Observer, Operation, Payload, Policy, ReadContext, ReadRequest, ReadResult, Remote, + ShadowPolicy, SyncCodec, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; + +struct SnapshotRemote { + result: ReadResult, + writes: AtomicUsize, +} + +impl Remote for SnapshotRemote { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(ready(Ok(self.result.clone()))) + } + + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.writes.fetch_add(1, Ordering::SeqCst); + Box::pin(ready(Ok(()))) + } + + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(ready(Ok(()))) + } +} + +#[derive(Default)] +struct Events(Mutex>); + +impl Observer for Events { + fn observe(&self, event: &Event) { + self.0.lock().push(event.clone()); + } + + fn observes_shadow_outcomes(&self) -> bool { + true + } +} + +fn setup(result: ReadResult) -> (TestExecutor, DialCache, Arc, Arc) { + let executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(SnapshotRemote { + result, + writes: AtomicUsize::new(0), + }); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .observer_arc(events.clone()) + .build() + .expect("valid configuration"); + (executor, cache, remote, events) +} + +fn hit(age_ms: u64) -> ReadResult { + ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64 - age_ms, + payload: Payload::text("1"), + }) +} + +enum PanickingCodec { + Encode, + Decode, +} + +impl SyncCodec for PanickingCodec { + fn encode(&self, value: &u64) -> Result { + if matches!(self, Self::Encode) { + panic!("synchronous encode failure"); + } + JsonCodec::encode_value(value) + } + + fn decode(&self, payload: Payload) -> Result { + if matches!(self, Self::Decode) { + panic!("synchronous decode failure"); + } + JsonCodec::decode_value(&payload) + } +} + +fn operation(codec: PanickingCodec) -> Operation { + Operation::with_codec( + Identity::new("thing", "one", "CallbackBoundary"), + Arc::new(FromSync(codec)), + |a, b| a == b, + ) + .policy(Policy::default().remote_ttl_sec(60)) +} + +#[test] +fn synchronous_encode_panic_preserves_source_success_and_local_publication() { + let (mut executor, cache, remote, events) = setup(ReadResult::miss(MissReason::ValueAbsent)); + let result = executor.block_on(async move { + let request = cache.enable_guard(); + let operation = operation(PanickingCodec::Encode).policy(Policy::enabled(60)); + let first = cache + .get_or_load(request.scope(), operation.clone(), |_| ready(Ok(7))) + .await?; + let second = cache + .get_or_load( + request.scope(), + operation, + |_| -> Ready> { + panic!("the source must not run after local publication") + }, + ) + .await?; + Ok::<_, dialcache::Error>((first, second)) + }); + let (first, second) = result.expect("a codec panic must fail open"); + assert_eq!((*first, *second), (7, 7)); + assert_eq!(remote.writes.load(Ordering::SeqCst), 0); + assert!(events.0.lock().iter().any(|event| matches!( + event, + Event::Error { + error: ErrorKind::SerializationDump, + .. + } + ))); +} + +#[test] +fn synchronous_decode_panic_falls_through_to_source_and_refills() { + let (mut executor, cache, remote, events) = setup(hit(0)); + let result = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load(request.scope(), operation(PanickingCodec::Decode), |_| { + ready(Ok(7)) + }) + .await + }); + assert_eq!(*result.expect("decode panic must reach the source"), 7); + assert_eq!(remote.writes.load(Ordering::SeqCst), 1); + assert!(events.0.lock().iter().any(|event| matches!( + event, + Event::Error { + error: ErrorKind::SerializationLoad, + .. + } + ))); +} + +#[test] +fn synchronous_recovery_decode_panic_preserves_the_source_error() { + let (mut executor, cache, remote, events) = setup(hit(2_000)); + let result = executor.block_on(async move { + let request = cache.enable_guard(); + let operation = operation(PanickingCodec::Decode) + .policy( + Policy::default() + .remote_ttl_sec(1) + .stale_on_error_max_age_sec(3), + ) + .should_recover(|_| true); + cache + .get_or_load(request.scope(), operation, |_| { + ready(Err("source failed".into())) + }) + .await + }); + let error = result.expect_err("recovery cannot decode the retained frame"); + assert_eq!( + error + .source_error() + .expect("original source error") + .to_string(), + "source failed" + ); + assert_eq!(remote.writes.load(Ordering::SeqCst), 0); + assert!(events.0.lock().iter().any(|event| matches!( + event, + Event::StaleRecovery { + outcome: RecoveryOutcome::DeserializationError, + .. + } + ))); +} + +#[test] +fn synchronous_shadow_source_panic_is_a_source_error_and_releases_capacity() { + let (mut executor, cache, _, events) = setup(hit(0)); + let request = cache.enable_guard(); + let source_enabled = Arc::new(AtomicBool::new(true)); + for _ in 0..2 { + let cache = cache.clone(); + let scope = request.scope().clone(); + let source_enabled = source_enabled.clone(); + let result = executor.block_on(async move { + let operation = Operation::::new(Identity::new("thing", "one", "ShadowBoundary")) + .policy(Policy::default().remote_ttl_sec(60).shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: None, + })); + cache + .get_or_load( + &scope, + operation, + move |scope| -> Ready> { + source_enabled.store(scope.is_enabled(), Ordering::SeqCst); + panic!("synchronous source failure"); + }, + ) + .await + }); + assert_eq!(*result.expect("served hit survives shadow failure"), 1); + } + assert!( + !source_enabled.load(Ordering::SeqCst), + "shadow source must run disabled" + ); + let outcomes: Vec<_> = events + .0 + .lock() + .iter() + .filter_map(|event| match event { + Event::ShadowValidation { outcome, .. } => Some(*outcome), + _ => None, + }) + .collect(); + assert_eq!( + outcomes, + vec![ShadowOutcome::SourceError, ShadowOutcome::SourceError] + ); +} + +#[test] +fn dark_shadow_distinguishes_its_deadline_from_application_timeout_errors() { + for error_kind in ["own deadline", "nested deadline", "source error"] { + let (mut executor, cache, _, events) = setup(hit(0)); + let result = Arc::new(Mutex::new(None)); + let sink = result.clone(); + executor.spawn(async move { + let request = cache.enable_guard(); + let operation = Operation::::new(Identity::new("thing", "one", "DarkDeadline")) + .policy( + Policy::default() + .remote_ttl_sec(60) + .remote_ramp(0.0) + .shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: None, + }), + ) + .budget(dialcache::SourceBudget::Millis(5)); + *sink.lock() = Some( + cache + .get_or_load(request.scope(), operation, move |_| async move { + match error_kind { + "own deadline" => std::future::pending::>().await, + "nested deadline" => Err(Box::new(dialcache::Error::FallbackTimeout( + Arc::new(dialcache::FallbackTimeout { + use_case: "nested".to_owned(), + timeout_ms: 1, + }), + )) as BoxError), + _ => Err("source failed".into()), + } + }) + .await, + ); + }); + executor.drain(); + if error_kind == "own deadline" { + assert!(result.lock().is_none()); + executor.advance(5, true); + assert!(matches!( + result.lock().as_ref().unwrap(), + Err(dialcache::Error::FallbackTimeout(_)) + )); + } else { + assert!(matches!( + result.lock().as_ref().unwrap(), + Err(dialcache::Error::Source(_)) + )); + } + let outcomes: Vec<_> = events + .0 + .lock() + .iter() + .filter_map(|event| { + if let Event::ShadowValidation { outcome, .. } = event { + Some(*outcome) + } else { + None + } + }) + .collect(); + assert_eq!( + outcomes, + vec![if error_kind == "own deadline" { + ShadowOutcome::Timeout + } else { + ShadowOutcome::SourceError + }], + "{error_kind}" + ); + } +} + +#[test] +fn provider_construction_poll_and_returned_failures_preserve_source_results() { + for failure in ["construct", "poll", "error"] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .observer_arc(events.clone()) + .policy_provider( + move |_| -> BoxFuture<'static, Result, BoxError>> { + if failure == "construct" { + panic!("provider future construction"); + } + Box::pin(async move { + if failure == "poll" { + panic!("provider future poll"); + } + Err("provider failed".into()) + }) + }, + ) + .build() + .unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let request = cache.enable_guard(); + for expected in 1..=2 { + let (cache, scope, calls) = (cache.clone(), request.scope().clone(), calls.clone()); + let value = executor.block_on(async move { + cache + .get_or_load( + &scope, + Operation::::new(Identity::new("thing", "one", "ProviderFailure")) + .policy(Policy::default().local_ttl_sec(60)), + move |_| { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + async move { Ok(value) } + }, + ) + .await + .unwrap() + }); + assert_eq!( + *value, expected, + "provider failure must bypass cache publication" + ); + } + let policy_errors = events.0.lock().iter().filter(|event| matches!(event, + Event::Error { labels, error: ErrorKind::ConfigResolution, .. } if labels.layer == Layer::Noop + )).count(); + assert_eq!(policy_errors, 2, "{failure}"); + } +} + +#[test] +fn held_policy_survives_caller_cancellation_and_rechecks_scope_closure() { + for close_scope in [false, true] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let (release, held) = futures::channel::oneshot::channel::<()>(); + let held = Arc::new(Mutex::new(Some(held))); + let provider_calls = Arc::new(AtomicUsize::new(0)); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .observer_arc(events.clone()) + .policy_provider({ + let provider_calls = provider_calls.clone(); + move |_| { + provider_calls.fetch_add(1, Ordering::SeqCst); + let held = held.lock().take(); + async move { + if let Some(held) = held { + held.await.unwrap(); + } + Ok(None) + } + } + }) + .build() + .unwrap(); + let mut request = Some(cache.enable_guard()); + let scope = request.as_ref().unwrap().scope().clone(); + let operation = Operation::::new(Identity::new("thing", "one", "HeldProvider")) + .policy(Policy::default().local_ttl_sec(60)); + let calls = Arc::new(AtomicUsize::new(0)); + let source_enabled = Arc::new(Mutex::new(Vec::new())); + let caller_finished = Arc::new(AtomicBool::new(false)); + let (cancel, registration) = futures::future::AbortHandle::new_pair(); + executor.spawn({ + let (cache, operation, calls) = (cache.clone(), operation.clone(), calls.clone()); + let (source_enabled, caller_finished) = + (source_enabled.clone(), caller_finished.clone()); + async move { + let pending = cache.get_or_load(&scope, operation, move |scope| { + calls.fetch_add(1, Ordering::SeqCst); + source_enabled.lock().push(scope.is_enabled()); + async { Ok(7) } + }); + let result = futures::future::Abortable::new(pending, registration).await; + assert_eq!(result.is_err(), !close_scope); + if let Ok(value) = result { + assert_eq!(*value.unwrap(), 7); + } + caller_finished.store(true, Ordering::SeqCst); + } + }); + executor.drain(); + assert_eq!(provider_calls.load(Ordering::SeqCst), 1); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert!(!caller_finished.load(Ordering::SeqCst)); + if close_scope { + drop(request.take()); + } else { + cancel.abort(); + } + executor.drain(); + assert_eq!(caller_finished.load(Ordering::SeqCst), !close_scope); + release.send(()).unwrap(); + executor.drain(); + assert!(caller_finished.load(Ordering::SeqCst)); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(*source_enabled.lock(), vec![!close_scope]); + let source_layers: Vec<_> = events + .0 + .lock() + .iter() + .filter_map(|event| match event { + Event::Fallback { labels, .. } => Some(labels.layer), + _ => None, + }) + .collect(); + assert_eq!( + source_layers, + vec![if close_scope { + Layer::Noop + } else { + Layer::Local + }] + ); + let calls_for_next = calls.clone(); + let value = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load(request.scope(), operation, move |_| { + calls_for_next.fetch_add(1, Ordering::SeqCst); + async { Ok(8) } + }) + .await + .unwrap() + }); + assert_eq!(*value, if close_scope { 8 } else { 7 }); + assert_eq!( + calls.load(Ordering::SeqCst), + if close_scope { 2 } else { 1 } + ); + } +} diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs new file mode 100644 index 00000000..39845539 --- /dev/null +++ b/rust/tests/conformance.rs @@ -0,0 +1,409 @@ +//! The Rust conformance harness: replays the shared corpus through the real +//! cache API against the Node replay coordinator, runs the fixed behavioral +//! scenarios and protocol vectors, and checks the shared witness evidence. +//! +//! Without selectors it replays the committed smoke histories. With the +//! `DIALCACHE_*_TRACE_DIR` selectors it replays the complete generated corpus +//! and writes the JSONL report named by `DIALCACHE_RUST_REPORT`. +//! `DIALCACHE_RUST_SUITE=generated|fixed` runs only the Quint-generated +//! evidence or only the fixed scenarios (the mutation measurement's cohorts). +//! Node 24 must be on `PATH` for the coordinator. + +#![allow(dead_code)] + +#[path = "formal/digest.rs"] +mod digest; + +#[path = "formal/fixtures.rs"] +mod fixtures; +mod formal; +#[path = "formal/frame_vectors.rs"] +mod frame_vectors; +#[path = "formal/key_vectors.rs"] +mod key_vectors; + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use formal::core_driver::CoreDriver; +use formal::driver::{install_panic_hook, Driver}; +use formal::inventory::{self, repo_root, Selection, TraceSource}; +use formal::local_clock::LocalClockDriver; +use formal::report::Report; +use formal::scenarios; +use formal::transport::{Coordinator, Prepared}; +use serde_json::{Map, Value}; + +struct Run { + selection: Selection, + report: Report, + coordinator: Coordinator, + profile_actions: BTreeMap>, + seen_actions: BTreeMap>, + failures: usize, + coverage_failures: Vec, +} + +fn main() -> ExitCode { + install_panic_hook(); + match run() { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::FAILURE, + Err(error) => { + eprintln!("conformance harness failed: {error}"); + ExitCode::FAILURE + } + } +} + +fn run() -> Result { + let selection = Selection::from_env()?; + let mut report = Report::from_env()?; + report.start()?; + inventory::registry_check()?; + let mut coordinator = Coordinator::spawn()?; + let mut request = Map::new(); + request.insert("op".to_string(), Value::from("profiles")); + let info = coordinator.call(request)?; + let mut profile_actions = BTreeMap::new(); + if let Some(profiles) = info.get("profiles").and_then(Value::as_object) { + for (name, actions) in profiles { + let actions: BTreeSet = actions + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_string) + .collect(); + profile_actions.insert(name.clone(), actions); + } + } + if let Some(selected) = &selection.feature_profile { + if !profile_actions.contains_key(selected) + || matches!(selected.as_str(), "core" | "effects") + { + return Err(format!("unknown selected feature profile: {selected}")); + } + } + let mut run = Run { + selection, + report, + coordinator, + profile_actions, + seen_actions: BTreeMap::new(), + failures: 0, + coverage_failures: Vec::new(), + }; + let suite = run.selection.suite; + if suite.runs_generated() { + run.core()?; + run.effects()?; + run.features()?; + run.local_clock()?; + } + if suite.runs_fixed() { + run.scenarios()?; + } + run.protocol_vectors()?; + if suite.runs_generated() { + run.witnesses()?; + } + for (profile, expected) in &run.profile_actions { + let selected_full = match profile.as_str() { + "core" => run.selection.core.is_directory(), + "effects" => run.selection.effects.is_directory(), + _ => { + run.selection.features.is_directory() + && run + .selection + .feature_profile + .as_deref() + .is_none_or(|p| p == profile) + } + }; + if !selected_full { + continue; + } + let seen = run.seen_actions.get(profile).cloned().unwrap_or_default(); + for action in expected { + if action != "init" && !seen.contains(action) { + run.coverage_failures + .push(format!("{profile} corpus omits action {action}")); + } + } + } + let summary = run.report.finish()?; + let Run { + coordinator, + coverage_failures, + .. + } = run; + coordinator.finish()?; + for failure in &coverage_failures { + eprintln!("coverage: {failure}"); + } + Ok(summary.failed == 0 && coverage_failures.is_empty()) +} + +fn note_actions(seen: &mut BTreeMap>, profile: &str, prepared: &Prepared) { + let entry = seen.entry(profile.to_string()).or_default(); + for action in &prepared.actions { + entry.insert(action.clone()); + } +} + +impl Run { + fn replay_core(&mut self, path: &Path) -> Result<(), String> { + let raw = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + let prepared = self.coordinator.prepare("core", path, Some(&raw))?; + note_actions(&mut self.seen_actions, "core", &prepared); + let mut driver = CoreDriver::new(); + let cell = std::cell::RefCell::new(&mut driver); + let mut apply = |input: &Value| cell.borrow_mut().apply(input); + let mut observation = || cell.borrow().observation(); + let mut wall = || cell.borrow().wall_ms(); + self.coordinator.execute( + &prepared, + &mut apply, + &mut observation, + &mut wall, + None, + &mut [], + ) + } + + fn core(&mut self) -> Result<(), String> { + let full = self.selection.core.is_directory(); + for path in self.selection.core_paths()? { + let id = inventory::trace_case_id("core", &path, full); + let started = formal::report::now_ms(); + let outcome = self.replay_core(&path); + self.record(&id, started, outcome)?; + } + Ok(()) + } + + fn replay_behavior( + &mut self, + profile: &str, + path: &Path, + effects_monitor: bool, + ) -> Result<(), String> { + let prepared = self.coordinator.prepare(profile, path, None)?; + note_actions(&mut self.seen_actions, profile, &prepared); + let mut driver = Driver::new(prepared.fixture.clone()); + let result = { + let driver_ref = &mut driver; + let cell = std::cell::RefCell::new(driver_ref); + let mut apply = |input: &Value| cell.borrow_mut().apply(input); + let mut observation = || cell.borrow().observation(); + let mut wall = || cell.borrow().observation_wall_ms(); + let mut receipt = || cell.borrow().receipt(); + let mut monitor = || cell.borrow().assert_effects_history(); + let mut monitors: Vec<&mut dyn FnMut() -> Result<(), String>> = Vec::new(); + if effects_monitor { + monitors.push(&mut monitor); + } + self.coordinator.execute( + &prepared, + &mut apply, + &mut observation, + &mut wall, + Some(&mut receipt), + &mut monitors, + ) + }; + driver.close(); + result + } + + fn effects(&mut self) -> Result<(), String> { + inventory::require_behavior_profile("effects")?; + let full = self.selection.effects.is_directory(); + for path in self.selection.effects_paths()? { + let id = inventory::trace_case_id("effects", &path, full); + let started = formal::report::now_ms(); + let outcome = self.replay_behavior("effects", &path, true); + self.record(&id, started, outcome)?; + } + Ok(()) + } + + fn features(&mut self) -> Result<(), String> { + let mut names: Vec = self + .profile_actions + .keys() + .filter(|n| !matches!(n.as_str(), "core" | "effects" | "local-clock")) + .cloned() + .collect(); + names.sort(); + let full = self.selection.features.is_directory(); + for name in names { + if let Some(selected) = &self.selection.feature_profile { + if selected != &name { + continue; + } + } + let paths = self.selection.feature_paths(&name)?; + if !paths.is_empty() { + inventory::require_behavior_profile(&name)?; + } + for path in paths { + let id = inventory::trace_case_id(&name, &path, full); + let started = formal::report::now_ms(); + let outcome = self.replay_behavior(&name, &path, false); + self.record(&id, started, outcome)?; + } + } + Ok(()) + } + + fn local_clock(&mut self) -> Result<(), String> { + let paths = self.selection.feature_paths("local-clock")?; + if paths.is_empty() { + return Ok(()); + } + if let Some(selected) = &self.selection.feature_profile { + if selected != "local-clock" { + return Ok(()); + } + } + inventory::require_behavior_profile("local-clock")?; + let full = self.selection.features.is_directory(); + for path in paths { + let id = inventory::trace_case_id("local-clock", &path, full); + let started = formal::report::now_ms(); + let outcome = (|| { + let prepared = self.coordinator.prepare("local-clock", &path, None)?; + note_actions(&mut self.seen_actions, "local-clock", &prepared); + let mut driver = LocalClockDriver::new(); + let cell = std::cell::RefCell::new(&mut driver); + let mut apply = |input: &Value| cell.borrow_mut().apply(input); + let mut observation = || cell.borrow().observation(); + let mut wall = || formal::report::now_ms(); + self.coordinator.execute( + &prepared, + &mut apply, + &mut observation, + &mut wall, + None, + &mut [], + ) + })(); + self.record(&id, started, outcome)?; + } + Ok(()) + } + + fn scenarios(&mut self) -> Result<(), String> { + let scenarios = scenarios::load_scenarios()?; + let filter = self.selection.behavior_scenario.clone(); + let mut matched = 0; + for scenario in &scenarios { + let name = scenario.get("name").and_then(Value::as_str).unwrap_or(""); + let feature = scenario + .get("feature") + .and_then(Value::as_str) + .unwrap_or(""); + if let Some(filter) = &filter { + if !name.contains(filter.as_str()) { + continue; + } + } + matched += 1; + let id = inventory::scenario_case_id(feature, name); + let started = formal::report::now_ms(); + let outcome = scenarios::replay_scenario(scenario); + self.record(&id, started, outcome)?; + } + if matched == 0 { + return Err("no behavioral scenario matched".to_string()); + } + Ok(()) + } + + fn protocol_vectors(&mut self) -> Result<(), String> { + let selection = match self.selection.protocol_corpus { + inventory::ProtocolCorpus::All => "all", + inventory::ProtocolCorpus::Generated => "generated", + inventory::ProtocolCorpus::Fixed => "fixed", + }; + let groups = fixtures::protocol_groups(selection); + for (group, vectors) in &groups { + let check: fn(&Value) -> Result<(), String> = match group.as_str() { + "keyVectors" => key_vectors::check_key_vector, + "invalidKeyVectors" => key_vectors::check_invalid_key_vector, + "normalizeArgsVectors" => key_vectors::check_normalize_args_vector, + "rampVectors" => key_vectors::check_ramp_vector, + "frameVectors" => frame_vectors::check_frame_vector, + "trackedDecodeVectors" => frame_vectors::check_tracked_decode_vector, + "untrackedDecodeVectors" => frame_vectors::check_untracked_decode_vector, + "invalidTimestampVectors" => frame_vectors::check_invalid_timestamp_vector, + "durationVectors" => frame_vectors::check_duration_vector, + "envelopeVectors" => frame_vectors::check_envelope_vector, + "compressedDecodeVectors" => frame_vectors::check_compressed_decode_vector, + "compressionWriteVectors" => frame_vectors::check_compression_write_vector, + other => return Err(format!("unknown protocol vector group {other}")), + }; + for vector in vectors { + let name = vector.get("name").and_then(Value::as_str).unwrap_or(""); + let id = inventory::protocol_case_id(group, name); + let started = formal::report::now_ms(); + let outcome = check(vector); + self.record(&id, started, outcome)?; + } + } + Ok(()) + } + + fn witnesses(&mut self) -> Result<(), String> { + let mut profiles: BTreeMap> = BTreeMap::new(); + if self.selection.effects.is_directory() { + profiles.insert("effects".to_string(), self.selection.effects_paths()?); + } + if self.selection.features.is_directory() && self.selection.feature_profile.is_none() { + for name in self.profile_actions.keys() { + if matches!(name.as_str(), "core" | "effects") { + continue; + } + profiles.insert(name.clone(), self.selection.feature_paths(name)?); + } + } + if profiles.is_empty() { + return Ok(()); + } + let Some(directory) = self.selection.witness_evidence_dir.clone() else { + return Err("full generated replay requires DIALCACHE_WITNESS_EVIDENCE_DIR with matching evaluated witness evidence".to_string()); + }; + let root = repo_root(); + for (profile, paths) in profiles { + let id = inventory::witness_case_id(&profile); + let started = formal::report::now_ms(); + let outcome = + formal::witness::check_witness_evidence(&root, &profile, &directory, &paths); + self.record(&id, started, outcome)?; + } + Ok(()) + } + + fn record( + &mut self, + id: &str, + started: i64, + outcome: Result<(), String>, + ) -> Result<(), String> { + if let Err(message) = &outcome { + self.failures += 1; + eprintln!("FAIL {id}\n{message}"); + } + self.report + .case(id, &outcome, started, formal::report::now_ms())?; + Ok(()) + } +} + +#[allow(dead_code)] +fn _trace(source: &TraceSource) -> bool { + source.is_directory() +} diff --git a/rust/tests/default_logger.rs b/rust/tests/default_logger.rs new file mode 100644 index 00000000..6c7e8a22 --- /dev/null +++ b/rust/tests/default_logger.rs @@ -0,0 +1,42 @@ +//! Exercise the actual default log facade, including optional value previews. +use dialcache::observe::LogFacadeLogger; +use dialcache::{LogEvent, Logger, ShadowMismatchDetails}; +use std::sync::Mutex; + +struct Capture(Mutex>); +impl log::Log for Capture { + fn enabled(&self, _: &log::Metadata<'_>) -> bool { + true + } + fn log(&self, record: &log::Record<'_>) { + self.0.lock().unwrap().push(record.args().to_string()); + } + fn flush(&self) {} +} +static CAPTURE: Capture = Capture(Mutex::new(Vec::new())); + +#[test] +fn default_warning_includes_only_available_bounded_previews() { + log::set_logger(&CAPTURE).unwrap(); + log::set_max_level(log::LevelFilter::Warn); + for (cached, source) in [ + (Some("\"old\""), Some("\"new\"")), + (None, Some("null")), + (None, None), + ] { + LogFacadeLogger.log(&LogEvent::ShadowMismatch(ShadowMismatchDetails { + namespace: "app".into(), + use_case: "lookup".into(), + key_type: "thing".into(), + cache_key: "one".into(), + cached_value_json: cached.map(str::to_owned), + source_value_json: source.map(str::to_owned), + })); + } + let messages = CAPTURE.0.lock().unwrap(); + assert!(messages[0].contains("cachedValue=\"old\" sourceValue=\"new\"")); + assert!(!messages[1].contains("cachedValue=")); + assert!(messages[1].contains("sourceValue=null")); + assert!(!messages[2].contains("cachedValue=")); + assert!(!messages[2].contains("sourceValue=")); +} diff --git a/rust/tests/formal/causal.rs b/rust/tests/formal/causal.rs new file mode 100644 index 00000000..472c6031 --- /dev/null +++ b/rust/tests/formal/causal.rs @@ -0,0 +1,222 @@ +//! Independent publication checks over driver-owned invocation contexts and +//! actual source/write callbacks. No model predictions or cache internals enter +//! this journal. The runtime decorator preserves context across detached tasks. + +use std::cell::Cell; +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use dialcache::Runtime; +use futures::future::BoxFuture; +use serde_json::{json, Value}; + +thread_local! { + static INVOCATION: Cell> = const { Cell::new(None) }; +} + +pub fn current_invocation() -> Option { + INVOCATION.get() +} + +struct RestoreInvocation(Option); + +impl Drop for RestoreInvocation { + fn drop(&mut self) { + INVOCATION.set(self.0); + } +} + +/// Installs the owner only while polling, so unrelated interleaved tasks never +/// inherit it. Restoration also happens if the wrapped future panics. +pub struct InvocationFuture { + owner: Option, + future: Pin>, +} + +impl InvocationFuture { + pub fn new(owner: Option, future: F) -> Self { + Self { + owner, + future: Box::pin(future), + } + } +} + +impl Future for InvocationFuture { + type Output = F::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + let _restore = RestoreInvocation(INVOCATION.replace(this.owner)); + this.future.as_mut().poll(cx) + } +} + +pub struct InvocationRuntime(pub Arc); + +impl Runtime for InvocationRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.0 + .spawn(Box::pin(InvocationFuture::new(current_invocation(), task))); + } + + fn defer(&self, task: BoxFuture<'static, ()>) { + self.0 + .defer(Box::pin(InvocationFuture::new(current_invocation(), task))); + } + + fn spawn_blocking( + &self, + task: Box, + ) -> Result<(), dialcache::BoxError> { + let owner = current_invocation(); + self.0.spawn_blocking(Box::new(move || { + let _restore = RestoreInvocation(INVOCATION.replace(owner)); + task(); + })) + } + + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + self.0.sleep(duration) + } +} + +#[derive(Clone, Debug)] +pub enum CausalEvent { + SourceStart { + id: usize, + owner: usize, + at_ms: i64, + budget_ms: Option, + }, + SourceSettlement { + id: usize, + at_ms: i64, + outcome: &'static str, + }, + WriteDispatch { + source: Option, + owner: Option, + at_ms: i64, + }, +} + +/// Only semantically valid journals can reach this discriminator. The mutation +/// measurement independently validates its rule and evidence fields. +pub fn property_failure(rule: &str, condition: &str, mut event: Value) -> String { + event["condition"] = Value::from(condition); + format!("CAUSAL_PROPERTY_FAILURE rule={rule} event={event}") +} + +/// Necessary C25/C26 conditions; fence and payload provenance still have their +/// separate replay assertions. Pending prefixes and late write completion are +/// allowed when the exact source settled successfully within its own budget. +pub fn assert_publication_causality(history: &[CausalEvent]) -> Result<(), String> { + struct Source { + owner: usize, + started: i64, + budget: Option, + settled: Option, + outcome: &'static str, + } + let mut sources: HashMap = HashMap::new(); + let mut owners = HashSet::new(); + let mut previous = -1; + for (index, event) in history.iter().enumerate() { + let fail = |reason| format!("C25/C26 causal event {index}: {reason}"); + let at_ms = match *event { + CausalEvent::SourceStart { at_ms, .. } + | CausalEvent::SourceSettlement { at_ms, .. } + | CausalEvent::WriteDispatch { at_ms, .. } => at_ms, + }; + if at_ms < 0 || at_ms < previous { + return Err(fail( + "elapsed observations must be nonnegative and monotonic", + )); + } + previous = at_ms; + match *event { + CausalEvent::SourceStart { + id, + owner, + at_ms, + budget_ms, + } => { + if sources.contains_key(&id) || !owners.insert(owner) { + return Err(fail("source and invocation ownership must be unique")); + } + if budget_ms == Some(0) { + return Err(fail( + "source budget must be positive or explicitly unbounded", + )); + } + sources.insert( + id, + Source { + owner, + started: at_ms, + budget: budget_ms, + settled: None, + outcome: "", + }, + ); + } + CausalEvent::SourceSettlement { id, at_ms, outcome } => { + let source = sources + .get_mut(&id) + .ok_or_else(|| fail("settlement must identify one actual pending source"))?; + if source.settled.is_some() || !["resolve", "reject"].contains(&outcome) { + return Err(fail("settlement must identify one actual pending source")); + } + source.settled = Some(at_ms); + source.outcome = outcome; + } + CausalEvent::WriteDispatch { source, owner, .. } => { + let (Some(id), Some(owner)) = (source, owner) else { + return Err(fail("write has no observed source ownership")); + }; + let source = sources + .get(&id) + .ok_or_else(|| fail("write has no observed source ownership"))?; + let evidence = json!({"event":"writeDispatch", "index":index, "atMs":at_ms, "source":id, "owner":owner}); + if source.owner != owner { + let mut evidence = evidence; + evidence["sourceOwner"] = json!(source.owner); + return Err(property_failure( + "C26", + "write belongs to a different invocation's source", + evidence, + )); + } + if source.outcome != "resolve" { + let mut evidence = evidence; + evidence["outcome"] = json!(source.outcome); + return Err(property_failure( + "C26", + "write requires that exact source's successful settlement", + evidence, + )); + } + if let Some(budget) = source.budget { + let settled = source.settled.expect("resolved source has a settlement"); + if (settled - source.started) as u64 >= budget { + let mut evidence = evidence; + evidence["startedAtMs"] = json!(source.started); + evidence["settledAtMs"] = json!(settled); + evidence["budgetMs"] = json!(budget); + return Err(property_failure( + "C25", + "late raw settlement cannot authorize publication", + evidence, + )); + } + } + } + } + } + Ok(()) +} diff --git a/rust/tests/formal/core_driver.rs b/rust/tests/formal/core_driver.rs new file mode 100644 index 00000000..8c9fb0b7 --- /dev/null +++ b/rust/tests/formal/core_driver.rs @@ -0,0 +1,376 @@ +//! The core profile driver: integer values, an in-memory remote and a wall +//! clock that moves only through `advanceWall`. + +use std::collections::HashMap; +use std::sync::Arc; + +use dialcache::protocol::{decode_frame, encode_frame}; +use dialcache::testing::{TestExecutor, VirtualClock}; +use dialcache::{ + BoxError, Clock, DialCache, Error, Identity, InvalidateRequest, Operation, Policy, ReadContext, + ReadRequest, ReadResult, Remote, Scope, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; +use serde_json::{json, Map, Value}; + +use super::driver::WALL_EPOCH_MS; +use super::gate::Gate; + +struct Entry { + raw: Vec, + expires: i64, +} + +#[derive(Default)] +struct RemoteState { + values: HashMap, + watermarks: HashMap, + reads: u64, + writes: u64, + read_failure: bool, + pub discard_writes: bool, +} + +struct MemoryRemote { + state: Arc>, + clock: Arc, +} + +impl Remote for MemoryRemote { + fn read( + &self, + request: ReadRequest, + _context: ReadContext, + ) -> BoxFuture<'_, Result> { + let state = self.state.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + let mut s = state.lock(); + s.reads += 1; + if s.read_failure { + return Err("controlled read failure".into()); + } + let elapsed = clock.elapsed_ms(); + let raw = s + .values + .get(&request.value_key) + .filter(|entry| elapsed < entry.expires) + .map(|entry| entry.raw.clone()); + let watermark = request + .watermark_key + .as_ref() + .and_then(|key| s.watermarks.get(key).cloned()); + let tracked = request.watermark_key.is_some(); + Ok(decode_frame( + raw.as_deref(), + tracked, + watermark.as_deref().map(str::as_bytes), + )?) + }) + } + + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + let state = self.state.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + let mut s = state.lock(); + s.writes += 1; + if s.discard_writes { + return Ok(()); + } + let raw = encode_frame(&request.frame)?; + let expires = clock.elapsed_ms() + request.ttl_ms as i64; + s.values.insert(request.value_key, Entry { raw, expires }); + Ok(()) + }) + } + + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + let state = self.state.clone(); + Box::pin(async move { + let mut s = state.lock(); + s.writes += 1; + let mut cutoff = request.invalidated_at_ms + request.future_buffer_ms; + let old: u64 = s + .watermarks + .get(&request.watermark_key) + .and_then(|w| w.parse().ok()) + .unwrap_or(0); + if cutoff < old { + cutoff = old; + } + s.watermarks + .insert(request.watermark_key, cutoff.to_string()); + Ok(()) + }) + } +} + +/// The core driver over one cache instance. +pub struct CoreDriver { + pub exec: TestExecutor, + cache: DialCache, + remote: Arc>, + clock: Arc, + source: Arc>, + last: i64, + loaders: Arc>>, +} + +impl std::fmt::Debug for CoreDriver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CoreDriver").finish() + } +} + +fn text(value: Option<&Value>) -> String { + value.and_then(Value::as_str).unwrap_or("").to_string() +} + +impl CoreDriver { + pub fn new() -> CoreDriver { + Self::with_discarded_writes(false) + } + + /// A broken driver whose remote acknowledges writes it never stores. + pub fn with_discarded_writes(discard: bool) -> CoreDriver { + let exec = TestExecutor::new(WALL_EPOCH_MS); + let clock = exec.clock.clone(); + let remote = Arc::new(Mutex::new(RemoteState { + discard_writes: discard, + ..RemoteState::default() + })); + let cache = DialCache::builder() + .clock_arc(clock.clone()) + .runtime_arc(exec.runtime.clone()) + .remote_arc(Arc::new(MemoryRemote { + state: remote.clone(), + clock: clock.clone(), + })) + .local_capacity(10_000) + .build() + .expect("core driver configuration"); + CoreDriver { + exec, + cache, + remote, + clock, + source: Arc::new(Mutex::new(1)), + last: 0, + loaders: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub fn wall_ms(&self) -> i64 { + self.clock.wall_ms() + } + + fn operation(identity: Identity, policy: &Value) -> Result, String> { + let policy = Policy::from_json(policy).map_err(|e| e.to_string())?; + Ok(Operation::::new(identity).policy(policy)) + } + + fn loader( + &self, + counter: &str, + gate: Option>, + started: Option>>, + ) -> impl Fn(Scope) -> BoxFuture<'static, Result> + Send + Sync + Clone + 'static + { + let loaders = self.loaders.clone(); + let source = self.source.clone(); + let counter = counter.to_string(); + move |_scope: Scope| { + let loaders = loaders.clone(); + let source = source.clone(); + let counter = counter.clone(); + let gate = gate.clone(); + let started = started.clone(); + Box::pin(async move { + let value = { + *loaders.lock().entry(counter).or_insert(0) += 1; + *source.lock() + }; + if let Some(started) = started { + *started.lock() += 1; + } + if let Some(gate) = gate { + gate.wait().await; + } + Ok(value) + }) + } + } + + /// Apply one core command. Only the selected public action reaches the + /// driver; publication is observed through later actual calls. + pub fn apply(&mut self, input: &Value) -> Result<(), String> { + match text(input.get("op")).as_str() { + "advanceWall" => { + let ms = input.get("ms").and_then(Value::as_i64).unwrap_or(0); + self.clock.shift(ms, true); + Ok(()) + } + "bumpSource" => { + *self.source.lock() += 1; + Ok(()) + } + "invalidate" => { + let identity = input.get("identity").cloned().unwrap_or(Value::Null); + let cache = self.cache.clone(); + let key_type = text(identity.get("keyType")); + let id = text(identity.get("id")); + self.exec + .block_on(async move { cache.invalidate(&key_type, &id, 0).await }) + .map_err(|e| e.to_string()) + } + "call" => self.call(input), + other => Err(format!("unknown core driver command {other}")), + } + } + + fn call(&mut self, input: &Value) -> Result<(), String> { + let identity_json = input.get("identity").cloned().unwrap_or(Value::Null); + let identity = Identity::new( + text(identity_json.get("keyType")), + text(identity_json.get("id")), + text(identity_json.get("useCase")), + ) + .namespace("urn") + .tracked(identity_json.get("tracked") == Some(&Value::Bool(true))); + let operation = Self::operation(identity, input.get("policy").unwrap_or(&Value::Null))?; + let counter = text(input.get("counter")); + let read_failure = input.get("readFailure") == Some(&Value::Bool(true)); + self.remote.lock().read_failure = read_failure; + let outcome: Result = match text(input.get("mode")).as_str() { + "outside" => { + let cache = self.cache.clone(); + let load = self.loader(&counter, None, None); + self.exec.block_on(async move { + cache + .get_or_load(&Scope::outside(), operation, load) + .await + .map(|v| *v) + }) + } + "single" => { + let cache = self.cache.clone(); + let load = self.loader(&counter, None, None); + self.exec.block_on(async move { + let inner = cache.clone(); + cache + .enable(|scope| async move { + inner.get_or_load(&scope, operation, load).await.map(|v| *v) + }) + .await + }) + } + "request-pair" => { + let cache = self.cache.clone(); + let load = self.loader(&counter, None, None); + self.exec.block_on(async move { + let inner = cache.clone(); + cache + .enable(|scope| async move { + let first = inner + .get_or_load(&scope, operation.clone(), load.clone()) + .await?; + let second = inner.get_or_load(&scope, operation, load).await?; + if *first != *second { + return Err(Error::Config(dialcache::ConfigError::Invalid( + "request pair differs".to_string(), + ))); + } + Ok(*first) + }) + .await + }) + } + "coalesced-pair" => self.pair(operation, &counter), + other => Err(Error::Config(dialcache::ConfigError::Invalid(format!( + "unknown core call mode {other}" + )))), + }; + self.remote.lock().read_failure = false; + match outcome { + Ok(value) => { + self.last = value; + Ok(()) + } + Err(error) => Err(error.to_string()), + } + } + + /// Two enabled callers whose leader source stays open until both have + /// started, so overlap is real. A warm cache completes the first call + /// before the second starts; lost sharing stays observable in loader counts. + fn pair(&mut self, operation: Operation, counter: &str) -> Result { + let gate: Gate<()> = Gate::new(); + let started = Arc::new(Mutex::new(0u64)); + let results: Arc>>> = Arc::new(Mutex::new(Vec::new())); + let results_handle = results.clone(); + let load = self.loader(counter, Some(gate.clone()), Some(started.clone())); + let cache = self.cache.clone(); + let make_call = move || { + let cache = cache.clone(); + let operation = operation.clone(); + let load = load.clone(); + let results = results.clone(); + async move { + let inner = cache.clone(); + let result = cache + .enable(|scope| async move { + inner.get_or_load(&scope, operation, load).await.map(|v| *v) + }) + .await; + results.lock().push(result); + } + }; + self.exec.spawn(make_call()); + self.exec.drain(); + self.exec.spawn(make_call()); + self.exec.drain(); + gate.settle(()); + self.exec.drain(); + let results = std::mem::take(&mut *results_handle.lock()); + if results.len() != 2 { + return Err(Error::Config(dialcache::ConfigError::Invalid( + "coalesced pair did not complete".to_string(), + ))); + } + let mut values = Vec::new(); + for result in results { + values.push(result?); + } + if values[0] != values[1] { + return Err(Error::Config(dialcache::ConfigError::Invalid( + "pair returned different values".to_string(), + ))); + } + Ok(values[0]) + } + + pub fn observation(&self) -> Value { + let remote = self.remote.lock(); + let loaders = self.loaders.lock(); + let mut out = Map::new(); + out.insert("sourceVersion".to_string(), json!(*self.source.lock())); + out.insert("lastResult".to_string(), json!(self.last)); + out.insert("redisReads".to_string(), json!(remote.reads)); + out.insert("redisWrites".to_string(), json!(remote.writes)); + for field in [ + "outsideLoaderCalls", + "requestLoaderCalls", + "localLoaderCalls", + "coalescedLoaderCalls", + "remoteLoaderCalls", + ] { + out.insert( + field.to_string(), + json!(loaders.get(field).copied().unwrap_or(0)), + ); + } + Value::Object(out) + } +} diff --git a/rust/tests/formal/digest.rs b/rust/tests/formal/digest.rs new file mode 100644 index 00000000..83e3466d --- /dev/null +++ b/rust/tests/formal/digest.rs @@ -0,0 +1,6 @@ +//! Shared test-only source and artifact fingerprints. +use sha2::{Digest, Sha256}; + +pub fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} diff --git a/rust/tests/formal/driver.rs b/rust/tests/formal/driver.rs new file mode 100644 index 00000000..9c3a59e7 --- /dev/null +++ b/rust/tests/formal/driver.rs @@ -0,0 +1,1661 @@ +//! The behavior driver: executes the shared replay commands against the real +//! Rust cache under the deterministic test executor. It controls external +//! effects only (sources, codecs, the fake remote, policy replies, clocks, +//! faults) and reports actual observations; no expected state enters it. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use dialcache::observe::{Event, LogEvent, Logger, Observer}; +use dialcache::protocol::{decode_frame, encode_frame, read_result_from_untrusted_json}; +use dialcache::testing::{TestExecutor, VirtualClock}; +use dialcache::{ + BoxError, Codec, DialCache, Error, FallbackTimeout, Identity, InvalidateRequest, LocalEntry, + LocalRead, LocalStore, LruLocalStore, Operation, Payload, Policy, ReadContext, ReadRequest, + ReadResult, Remote, RuntimePolicy, Scope, SourceBudget, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; +use serde_json::{json, Map, Value}; + +use super::causal::{ + assert_publication_causality, current_invocation, property_failure, CausalEvent, + InvocationFuture, InvocationRuntime, +}; +use super::gate::Gate; +use super::json::json_equal; + +/// Epoch of every controlled history: 2026-09-08T12:00:00.000Z. +pub const WALL_EPOCH_MS: i64 = 1_788_868_800_000; + +/// Panic payload of every controlled callback failure; the harness panic hook +/// keeps it out of the test output. +pub struct ControlledFailure; + +pub fn controlled_panic() -> ! { + std::panic::panic_any(ControlledFailure) +} + +/// Install a panic hook that stays silent for controlled failures. +pub fn install_panic_hook() { + let default = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + if info.payload().downcast_ref::().is_none() { + default(info); + } + })); +} + +type ComparatorFn = Box bool + Send + Sync>; + +/// The N-th controlled source failure. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("source failure {0}")] +pub struct SourceFailure(pub usize); + +/// The controlled mutation failure returned by the fake remote. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("controlled mutation failure")] +pub struct MaintenanceFailure; + +/// The cached value domain of the histories: JSON primitives plus absence. +#[derive(Debug, Clone)] +pub enum Val { + Absent, + Json(Value), +} + +impl Val { + fn from_json(value: Value) -> Val { + Val::Json(value) + } + + /// The observation encoding: `{absent:true}` for absence, else the primitive. + pub fn observed(&self) -> Value { + match self { + Val::Absent => json!({ "absent": true }), + Val::Json(value) => value.clone(), + } + } + + pub fn json_preview(&self) -> Option { + match self { + Val::Absent => None, + Val::Json(value) => serde_json::to_string(value).ok(), + } + } +} + +impl PartialEq for Val { + fn eq(&self, other: &Val) -> bool { + match (self, other) { + (Val::Absent, Val::Absent) => true, + (Val::Json(a), Val::Json(b)) => json_equal(a, b), + _ => false, + } + } +} + +#[derive(Clone)] +enum LoaderOutcome { + Value(Val), + Fail(usize), + Timeout(usize), +} + +struct Stored { + raw: Vec, + expires: i64, +} + +#[derive(Clone, Debug)] +pub struct HistoryEvent { + pub event: &'static str, + pub id: usize, + pub at: i64, + pub outcome: &'static str, + pub duration_ms: f64, + pub failed: bool, +} + +pub struct Shared { + fixture: Value, + observed: Map, + loaders: Vec>, + timeout_errors: Vec, + faults: HashMap, + runtime_policy: Value, + values: HashMap, + reply: Option, + effects: HashMap>>>, + history: Vec, + causal_history: Vec, + source_by_invocation: HashMap, + fallback_failed: bool, + pub discard_writes: bool, + pub discard_invalidations: bool, +} + +impl Shared { + fn increment(&mut self, field: &str) -> usize { + let current = self + .observed + .get(field) + .and_then(Value::as_u64) + .unwrap_or(0); + self.observed + .insert(field.to_string(), Value::from(current + 1)); + current as usize + } + + fn push(&mut self, field: &str, value: Value) { + if let Some(Value::Array(items)) = self.observed.get_mut(field) { + items.push(value); + } + } + + fn fault(&self, name: &str) -> bool { + self.faults.get(name).copied().unwrap_or(false) + } + + fn observes(&self, kind: &str) -> bool { + self.fixture + .get("observe") + .and_then(Value::as_array) + .is_some_and(|kinds| kinds.iter().any(|k| k.as_str() == Some(kind))) + } + + fn record(&mut self, kind: &str, fields: Map) { + if !self.observes(kind) { + return; + } + let mut event = Map::new(); + event.insert("event".to_string(), Value::from(kind)); + for (k, v) in fields { + event.insert(k, v); + } + self.push("events", Value::Object(event)); + } + + fn observer_fails(&self) -> bool { + self.fixture.get("observerFailure") == Some(&Value::Bool(true)) || self.fault("observer") + } + + fn raw(&mut self, key: &str, elapsed_ms: i64) -> Option> { + let expired = self.values.get(key)?.expires <= elapsed_ms; + if expired { + self.values.remove(key); + return None; + } + self.values.get(key).map(|item| item.raw.clone()) + } + + fn timeout_index(&mut self, error: &Error) -> usize { + let ptr = |error: &Error| -> usize { + match error { + Error::FallbackTimeout(arc) => Arc::as_ptr(arc) as *const () as usize, + Error::Source(arc) => Arc::as_ptr(arc) as *const () as usize, + _ => 0, + } + }; + let wanted = ptr(error); + if let Some(index) = self.timeout_errors.iter().position(|e| ptr(e) == wanted) { + return index; + } + self.timeout_errors.push(error.clone()); + self.timeout_errors.len() - 1 + } + + fn classify_error(&mut self, error: &Error) -> String { + match error { + Error::Source(source) => { + if let Some(SourceFailure(id)) = source.downcast_ref::() { + format!("source:{id}") + } else if let Some(timeout) = source.downcast_ref::() { + // The source returned a deadline error of its own (a nested call's); + // it is that loader's failure, not this call's deadline. + match timeout.use_case.strip_prefix("NestedSource:") { + Some(id) => format!("source:{id}"), + None => format!("timeout:{}", self.timeout_index(error)), + } + } else { + format!("unexpected:{error}") + } + } + Error::FallbackTimeout(_) => format!("timeout:{}", self.timeout_index(error)), + other => format!("unexpected:{other}"), + } + } +} + +/// The empty observation record for a fixture: `events` only when it observes. +pub fn empty_observation(fixture: &Value) -> Map { + let mut o = Map::new(); + if fixture.get("observe").is_some() { + o.insert("events".to_string(), json!([])); + } + o.insert("calls".to_string(), json!([])); + for key in [ + "loaders", + "reads", + "writes", + "invalidations", + "loads", + "dumps", + "policyCalls", + "classifications", + "comparisons", + ] { + o.insert(key.to_string(), json!(0)); + } + for key in [ + "maintenance", + "sourceScopes", + "writeTtls", + "shadow", + "recovery", + ] { + o.insert(key.to_string(), json!([])); + } + o +} + +struct ScopeHandle { + instance: String, + scope: Arc>>, + gate: Gate<()>, + done: Gate<()>, +} + +/// One history's driver. +pub struct Driver { + pub exec: TestExecutor, + shared: Arc>, + clock: Arc, + fixture: Value, + instances: HashMap, + scopes: HashMap, + reported: Value, + settlement_receipt: Value, + reported_wall_ms: i64, + /// Harness control only: skip the first settlement drain and let the + /// verification drain attest to the work it finds still runnable. + pub skip_settle: bool, +} + +impl fmt::Debug for Driver { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Driver") + .field("instances", &self.instances.len()) + .finish() + } +} + +fn number(value: Option<&Value>) -> i64 { + match value { + Some(Value::Number(n)) => n + .as_i64() + .or_else(|| n.as_f64().map(|f| f as i64)) + .unwrap_or(0), + _ => 0, + } +} + +fn text(value: Option<&Value>) -> String { + value.and_then(Value::as_str).unwrap_or("").to_string() +} + +fn shared_labels(shared: &Shared) -> Map { + let _ = shared; + Map::new() +} + +impl Driver { + pub fn new(fixture: Value) -> Driver { + let exec = TestExecutor::new(WALL_EPOCH_MS); + let clock = exec.clock.clone(); + let shared = Shared { + observed: empty_observation(&fixture), + fixture: fixture.clone(), + loaders: Vec::new(), + timeout_errors: Vec::new(), + faults: HashMap::new(), + runtime_policy: json!({}), + values: HashMap::new(), + reply: None, + effects: HashMap::new(), + history: Vec::new(), + causal_history: Vec::new(), + source_by_invocation: HashMap::new(), + fallback_failed: false, + discard_writes: false, + discard_invalidations: false, + }; + let mut driver = Driver { + exec, + shared: Arc::new(Mutex::new(shared)), + clock, + fixture, + instances: HashMap::new(), + scopes: HashMap::new(), + reported: Value::Null, + settlement_receipt: Value::Null, + reported_wall_ms: WALL_EPOCH_MS, + skip_settle: false, + }; + driver.instance("default"); + driver.settle(); + driver + } + + pub fn shared(&self) -> &Arc> { + &self.shared + } + + pub fn wall_ms(&self) -> i64 { + use dialcache::Clock; + self.clock.wall_ms() + } + + fn elapsed_ms(&self) -> i64 { + use dialcache::Clock; + self.clock.elapsed_ms() + } + + fn identity(&self, key: &str, use_case: &str) -> Identity { + let key = if key.is_empty() { "1" } else { key }; + let use_case = if use_case.is_empty() { + "Behavior" + } else { + use_case + }; + Identity::new("id", key, use_case) + .namespace("urn") + .tracked(self.fixture.get("tracked") == Some(&Value::Bool(true))) + } + + fn classifier(&self, mode: &str) -> impl Fn(&Error) -> bool + Send + Sync + 'static { + let shared = self.shared.clone(); + let mode = mode.to_string(); + move |_error: &Error| { + shared.lock().increment("classifications"); + if mode == "error" { + controlled_panic(); + } + mode == "allow" + } + } + + fn instance(&mut self, id: &str) -> DialCache { + if let Some(cache) = self.instances.get(id) { + return cache.clone(); + } + let fixture = &self.fixture; + let shared = self.shared.clone(); + let clock = self.clock.clone(); + let capacity = fixture + .get("localMaxSize") + .map(|v| number(Some(v)) as usize) + .unwrap_or(10_000); + let mut builder = DialCache::builder() + .clock_arc(self.clock.clone()) + .runtime(InvocationRuntime(self.exec.runtime.clone())) + .disable_compression() + .local_capacity(capacity) + .shadow_max_in_flight( + fixture + .get("shadowMaxInFlight") + .map(|v| number(Some(v)) as usize) + .unwrap_or(1), + ) + .observer_arc(Arc::new(DriverObserver { + shared: shared.clone(), + clock: clock.clone(), + shadow_hook: fixture.get("shadowHook") != Some(&Value::Bool(false)), + })) + .logger_arc(Arc::new(DriverLogger { + shared: shared.clone(), + })) + .policy_provider({ + let shared = shared.clone(); + move |_identity: Identity| { + let shared = shared.clone(); + async move { + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("policyCalls"); + (index, s.fault("holdPolicies")) + }; + if hold { + hold_effect(&shared, "policy", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + let s = shared.lock(); + if s.fault("policy") { + return Err("controlled policy failure".into()); + } + Ok(Some(RuntimePolicy::from_json(s.runtime_policy.clone()))) + } + } + }); + if fixture.get("remote") != Some(&Value::Bool(false)) { + builder = builder.remote_arc(Arc::new(DriverRemote { + shared: shared.clone(), + clock: clock.clone(), + })); + } + if fixture.get("localFaultInjection") == Some(&Value::Bool(true)) { + let inner = std::num::NonZeroUsize::new(capacity).map(LruLocalStore::new); + builder = builder.local_store(Box::new(FaultStore { + inner, + shared: shared.clone(), + })); + } + if fixture.get("readTimeoutMs") != Some(&Value::from("default")) { + builder = builder.remote_read_timeout_ms( + fixture + .get("readTimeoutMs") + .map(|v| number(Some(v)) as u64) + .unwrap_or(50), + ); + } + if let Some(mode) = fixture.get("recovery").and_then(Value::as_str) { + if mode != "default" { + builder = builder.should_recover(self.classifier(mode)); + } + } + let cache = builder.build().expect("valid fixture configuration"); + self.instances.insert(id.to_string(), cache.clone()); + cache + } + + fn budget(&self) -> SourceBudget { + match self.fixture.get("fallbackTimeoutMs") { + None => SourceBudget::Millis(10), + Some(Value::Null) => SourceBudget::Unbounded, + Some(Value::String(s)) if s == "default" => SourceBudget::Default, + Some(other) => SourceBudget::Millis(number(Some(other)) as u64), + } + } + + fn scope_of(&self, id: &str) -> Result<(Scope, String), String> { + let handle = self + .scopes + .get(id) + .ok_or_else(|| format!("unknown scope {id}"))?; + let scope = handle + .scope + .lock() + .clone() + .ok_or_else(|| format!("scope {id} not ready"))?; + Ok((scope, handle.instance.clone())) + } + + /// Apply one external command, then settle authorized work. + pub fn apply(&mut self, input: &Value) -> Result<(), String> { + let op = text(input.get("op")); + match op.as_str() { + "begin" => self.begin(input)?, + "resolve" | "reject" => { + let index = number(input.get("loader")); + let mut s = self.shared.lock(); + if index < 0 + || index as usize >= s.loaders.len() + || s.loaders[index as usize].is_settled() + { + return Err(format!("no unsettled source {index}")); + } + let index = index as usize; + let at = self.elapsed_ms(); + let outcome = if op == "reject" { + let gate = s.loaders[index].clone(); + if input.get("error") == Some(&Value::from("timeout")) { + gate.settle(LoaderOutcome::Timeout(index)); + } else { + gate.settle(LoaderOutcome::Fail(index)); + } + "reject" + } else { + let value = match input.get("value") { + None => Val::Absent, + Some(value) => Val::from_json(value.clone()), + }; + let gate = s.loaders[index].clone(); + gate.settle(LoaderOutcome::Value(value)); + "resolve" + }; + s.causal_history.push(CausalEvent::SourceSettlement { + id: index, + at_ms: at, + outcome, + }); + s.history.push(HistoryEvent { + event: "sourceSettlement", + id: index, + at, + outcome, + duration_ms: 0.0, + failed: false, + }); + } + "advance" => { + let ms = input.get("ms").and_then(Value::as_f64).unwrap_or(0.0); + if ms < 0.0 { + return Err("negative elapsed advance".to_string()); + } + let deliver = input.get("deliverTimers") != Some(&Value::Bool(false)); + self.exec.advance(ms as i64, deliver); + } + "shiftWall" => self.clock.shift(number(input.get("ms")), true), + "seed" => { + let identity = self.identity(&text(input.get("key")), &text(input.get("useCase"))); + let keys = identity.keys().map_err(|e| e.to_string())?; + let raw = if let Some(frame) = input.get("frameHex").and_then(Value::as_str) { + hex::decode(frame).map_err(|e| e.to_string())? + } else { + let mut encoding = 0u8; + let payload: Vec = + if let Some(h) = input.get("payloadHex").and_then(Value::as_str) { + encoding = 1; + hex::decode(h).map_err(|e| e.to_string())? + } else if let Some(t) = input.get("payloadText").and_then(Value::as_str) { + t.as_bytes().to_vec() + } else if let Some(value) = input.get("value") { + serde_json::to_vec(value).map_err(|e| e.to_string())? + } else { + b"undefined".to_vec() + }; + let stamp = (self.wall_ms() - number(input.get("ageMs"))) as u64; + let mut raw = Vec::with_capacity(10 + payload.len()); + raw.push(1); + raw.extend_from_slice(&stamp.to_be_bytes()); + raw.push(encoding); + raw.extend_from_slice(&payload); + raw + }; + let ttl = input + .get("ttlMs") + .map(|v| number(Some(v))) + .unwrap_or(60_000); + let expires = self.elapsed_ms() + ttl; + self.shared + .lock() + .values + .insert(keys.value, Stored { raw, expires }); + } + "invalidate" => { + let cache = self.instance("default"); + let key = text(input.get("key")); + let key = if key.is_empty() { "1".to_string() } else { key }; + let buffer = number(input.get("futureBufferMs")).max(0) as u64; + let result = self + .exec + .block_on(async move { cache.invalidate("id", &key, buffer).await }); + let status = match result { + Ok(()) => "ok", + Err(Error::Remote(error)) + if error.downcast_ref::().is_some() => + { + "mutation_error" + } + Err(Error::MissingRemote) => "missing_remote", + Err(other) => return Err(format!("unexpected invalidation error: {other}")), + }; + self.shared.lock().push("maintenance", Value::from(status)); + } + "observeMarker" => { + let identity = self.identity(&text(input.get("key")), "").tracked(true); + let keys = identity.keys().map_err(|e| e.to_string())?; + let watermark = keys.watermark.ok_or("tracked identity without watermark")?; + let elapsed = self.elapsed_ms(); + let (cutoff, ttl) = { + let mut s = self.shared.lock(); + match s.raw(&watermark, elapsed) { + Some(raw) => { + let stamp: i64 = String::from_utf8_lossy(&raw) + .parse() + .map_err(|e| format!("{e}"))?; + let ttl = s + .values + .get(&watermark) + .map(|item| item.expires - elapsed) + .unwrap_or(-2); + (stamp - WALL_EPOCH_MS, ttl) + } + None => (-1, -2), + } + }; + let mut fields = Map::new(); + fields.insert("cutoffMs".to_string(), Value::from(cutoff)); + fields.insert("ttlMs".to_string(), Value::from(ttl)); + self.shared.lock().record("marker", fields); + } + "inspectCoalescing" => { + let instance = input + .get("instance") + .and_then(Value::as_str) + .unwrap_or("default"); + let state = self.instance(instance).coalescing_state().process; + let fields = json!({ + "instance": instance, + "activeLeaders": state.active_leaders, + "activeFollowers": state.active_followers, + "oldestLeaderAgeMs": state.oldest_leader_age_ms, + }); + self.shared.lock().record( + "coalescingState", + fields.as_object().expect("fields").clone(), + ); + } + "adapterReply" => { + let mut s = self.shared.lock(); + if s.reply.is_some() { + return Err("unconsumed adapter reply".to_string()); + } + s.reply = Some(input.get("value").cloned().unwrap_or(Value::Null)); + } + "policy" => { + self.shared.lock().runtime_policy = + input.get("value").cloned().unwrap_or(Value::Null) + } + "faults" => { + let mut s = self.shared.lock(); + if let Some(Value::Object(faults)) = input.get("value") { + for (k, v) in faults { + s.faults.insert(k.clone(), v.as_bool().unwrap_or(false)); + } + } + } + "release" => { + let effect = text(input.get("effect")); + let index = number(input.get("index")).max(0) as usize; + let gate = { + let mut s = self.shared.lock(); + s.effects + .get_mut(&effect) + .and_then(|gates| gates.remove(&index)) + }; + let Some(gate) = gate.filter(|gate| !gate.is_settled()) else { + return Err(format!("no pending {effect} {index}")); + }; + if input.get("fail") == Some(&Value::Bool(true)) { + gate.settle(Err(format!("controlled {effect} failure"))); + } else { + gate.settle(Ok(())); + } + } + "openScope" => self.open_scope(input)?, + "closeScope" => { + let id = text(input.get("id")); + let handle = self.scopes.get(&id).ok_or("unknown/already closed scope")?; + if handle.gate.is_settled() { + return Err("unknown/already closed scope".to_string()); + } + handle.gate.settle(()); + let done = handle.done.clone(); + self.exec.drain(); + if !done.is_settled() { + return Err(format!("scope {id} did not close")); + } + } + other => return Err(format!("unknown behavior input {other}: {input}")), + } + self.settle(); + self.assert_publication_causality() + } + + /// Attest to a single instant, then verify it with a second zero-time drain. + /// Counts come from actual executor polls and external gates, never predictions. + fn settle(&mut self) { + if !self.skip_settle { + self.exec.drain(); + } + let shared = self.shared.lock(); + self.reported = Value::Object(shared.observed.clone()); + let mut held = Map::new(); + held.insert( + "loaders".to_string(), + json!(shared + .loaders + .iter() + .filter(|gate| !gate.is_settled()) + .count()), + ); + for (kind, field) in [ + ("read", "reads"), + ("write", "writes"), + ("dump", "dumps"), + ("load", "loads"), + ("policy", "policies"), + ] { + held.insert( + field.to_string(), + json!(shared.effects.get(kind).map_or(0, HashMap::len)), + ); + } + held.insert( + "scopes".to_string(), + json!(self + .scopes + .values() + .filter(|scope| !scope.gate.is_settled()) + .count()), + ); + drop(shared); + let elapsed = self.elapsed_ms(); + self.reported_wall_ms = self.wall_ms(); + let polls = self.exec.poll_count(); + let timers = self.clock.pending_timers(); + self.exec.drain(); + let changed = self.reported != Value::Object(self.shared.lock().observed.clone()); + let runnable = self.exec.poll_count() - polls + + u64::from(changed) + + timers.abs_diff(self.clock.pending_timers()) as u64; + self.settlement_receipt = json!({"elapsedMs": elapsed, "runnable": runnable, "held": held}); + } + + fn begin(&mut self, input: &Value) -> Result<(), String> { + let index = { + let mut s = self.shared.lock(); + let index = s + .observed + .get("calls") + .and_then(Value::as_array) + .map(|c| c.len()) + .unwrap_or(0); + s.push("calls", json!({ "status": "pending" })); + index + }; + let (ctx, instance) = match input.get("scope").and_then(Value::as_str) { + Some(scope_id) => { + let (scope, scope_instance) = self.scope_of(scope_id)?; + let instance = input + .get("instance") + .and_then(Value::as_str) + .map(str::to_string) + .unwrap_or(scope_instance); + (Some(scope), instance) + } + None => ( + None, + input + .get("instance") + .and_then(Value::as_str) + .unwrap_or("default") + .to_string(), + ), + }; + let cache = self.instance(&instance); + let policy = Policy::from_json(self.fixture.get("policy").unwrap_or(&Value::Null)) + .map_err(|e| format!("invalid fixture policy: {e}"))?; + let identity = self.identity(&text(input.get("key")), &text(input.get("useCase"))); + let shared = self.shared.clone(); + let clock = self.clock.clone(); + let comparator_mode = self + .fixture + .get("comparator") + .and_then(Value::as_str) + .map(str::to_string); + let comparison_ms = number(self.fixture.get("comparisonMs")); + let comparator: ComparatorFn = match comparator_mode { + None => Box::new(|a: &Val, b: &Val| a == b), + Some(mode) => { + let shared = shared.clone(); + let clock = clock.clone(); + Box::new(move |_a: &Val, _b: &Val| { + shared.lock().increment("comparisons"); + clock.shift(comparison_ms, false); + if mode == "error" { + controlled_panic(); + } + mode == "equal" + }) + } + }; + let codec: Arc> = Arc::new(DriverCodec { + shared: shared.clone(), + }); + let mut operation = Operation::with_codec(identity, codec, comparator) + .policy(policy) + .budget(self.budget()) + .preview(|value: &Val| value.json_preview()); + if let Some(mode) = input.get("recovery").and_then(Value::as_str) { + operation = operation.should_recover(self.classifier(mode)); + } + let probe_scope = self.fixture.get("probeSourceScope") == Some(&Value::Bool(true)); + let source_work_ms = number(self.fixture.get("sourceWorkMs")); + let source_cache = cache.clone(); + let source_shared = shared.clone(); + let source_clock = clock.clone(); + // Filled from the actual scope when this driver-owned call starts. + let source_budget = Arc::new(Mutex::new(None)); + let loader_budget = source_budget.clone(); + let load = move |scope: Scope| { + let shared = source_shared.clone(); + let clock = source_clock.clone(); + let cache = source_cache.clone(); + let source_budget = loader_budget.clone(); + async move { + let gate = { + let mut s = shared.lock(); + let id = s.loaders.len(); + let gate: Gate = Gate::new(); + s.loaders.push(gate.clone()); + s.increment("loaders"); + use dialcache::Clock; + let at = clock.elapsed_ms(); + s.source_by_invocation.insert(index, id); + s.causal_history.push(CausalEvent::SourceStart { + id, + owner: index, + at_ms: at, + budget_ms: *source_budget.lock(), + }); + s.history.push(HistoryEvent { + event: "sourceStart", + id, + at, + outcome: "", + duration_ms: 0.0, + failed: false, + }); + if probe_scope { + let enabled = cache.is_enabled(&scope); + s.push("sourceScopes", Value::Bool(enabled)); + } + gate + }; + clock.shift(source_work_ms, false); + match gate.wait().await { + LoaderOutcome::Value(value) => Ok(value), + LoaderOutcome::Fail(id) => Err(Box::new(SourceFailure(id)) as BoxError), + LoaderOutcome::Timeout(id) => { + // A timeout propagated by the source keeps the loader's identity. + Err(Box::new(FallbackTimeout { + use_case: format!("NestedSource:{id}"), + timeout_ms: 10, + }) as BoxError) + } + } + } + }; + let call_shared = shared.clone(); + let call_cache = cache.clone(); + let call = move |scope: Scope| { + let shared = call_shared.clone(); + let cache = call_cache.clone(); + let operation = operation.clone(); + let load = load.clone(); + let source_budget = source_budget.clone(); + InvocationFuture::new(Some(index), async move { + *source_budget.lock() = if !cache.is_enabled(&scope) { + None + } else { + match operation.budget { + SourceBudget::Default => Some(60_000), + SourceBudget::Unbounded => None, + SourceBudget::Millis(ms) => Some(ms), + } + }; + let result = cache.get_or_load(&scope, operation, load).await; + let mut s = shared.lock(); + let record = match result { + Ok(value) => json!({ "status": "value", "value": value.observed() }), + Err(error) => json!({ "status": "error", "error": s.classify_error(&error) }), + }; + if let Some(Value::Array(calls)) = s.observed.get_mut("calls") { + calls[index] = record; + } + }) + }; + let disabled = input.get("disabled") == Some(&Value::Bool(true)); + let outside = input.get("outside") == Some(&Value::Bool(true)); + match ctx { + Some(ctx) => { + if disabled { + self.exec + .spawn(async move { cache.disable_in(&ctx, call).await }); + } else { + self.exec.spawn(call(ctx)); + } + } + None if outside => { + let ctx = Scope::outside(); + if disabled { + self.exec + .spawn(async move { cache.disable_in(&ctx, call).await }); + } else { + self.exec.spawn(call(ctx)); + } + } + None => { + if disabled { + let inner = cache.clone(); + self.exec.spawn(async move { + cache + .enable(|scope| async move { inner.disable_in(&scope, call).await }) + .await + }); + } else { + self.exec.spawn(async move { cache.enable(call).await }); + } + } + } + Ok(()) + } + + fn open_scope(&mut self, input: &Value) -> Result<(), String> { + let id = text(input.get("id")); + if self.scopes.contains_key(&id) { + return Err(format!("duplicate scope {id}")); + } + let parent = match input.get("parent").and_then(Value::as_str) { + Some(parent_id) => Some(self.scope_of(parent_id)?), + None => None, + }; + let instance = input + .get("instance") + .and_then(Value::as_str) + .map(str::to_string) + .or_else(|| parent.as_ref().map(|(_, instance)| instance.clone())) + .unwrap_or_else(|| "default".to_string()); + let cache = self.instance(&instance); + let handle = ScopeHandle { + instance, + scope: Arc::new(Mutex::new(None)), + gate: Gate::new(), + done: Gate::new(), + }; + let slot = handle.scope.clone(); + let gate = handle.gate.clone(); + let done = handle.done.clone(); + let disabled = input.get("disabled") == Some(&Value::Bool(true)); + let body = move |scope: Scope| { + let slot = slot.clone(); + let gate = gate.clone(); + async move { + *slot.lock() = Some(scope); + gate.wait().await; + } + }; + let parent_scope = parent.map(|(scope, _)| scope); + self.exec.spawn(async move { + match (parent_scope, disabled) { + (Some(parent), true) => cache.disable_in(&parent, body).await, + (Some(parent), false) => cache.enable_in(&parent, body).await, + (None, true) => cache.disable_in(&Scope::outside(), body).await, + (None, false) => cache.enable(body).await, + } + done.settle(()); + }); + self.exec.drain(); + if handle.scope.lock().is_none() { + return Err(format!("scope {id} did not open")); + } + self.scopes.insert(id, handle); + Ok(()) + } + + /// The reported observation at the instant its receipt describes. + pub fn observation(&self) -> Value { + self.reported.clone() + } + + pub fn receipt(&self) -> Value { + self.settlement_receipt.clone() + } + + pub fn observation_wall_ms(&self) -> i64 { + self.reported_wall_ms + } + + /// Release every driver-owned gate so no work leaks into the next history. + pub fn close(&mut self) { + { + let mut s = self.shared.lock(); + for name in [ + "holdReads", + "holdWrites", + "holdDumps", + "holdLoads", + "holdPolicies", + ] { + s.faults.insert(name.to_string(), false); + } + } + for handle in self.scopes.values() { + handle.gate.settle(()); + } + loop { + let mut pending = false; + { + let mut s = self.shared.lock(); + for gates in s.effects.values_mut() { + for gate in gates.values() { + if gate.settle(Ok(())) { + pending = true; + } + } + } + for gate in &s.loaders { + if gate.settle(LoaderOutcome::Value(Val::Json(json!(0)))) { + pending = true; + } + } + } + self.exec.drain(); + if !pending { + break; + } + } + } + + pub fn causal_history(&self) -> Vec { + self.shared.lock().causal_history.clone() + } + + pub fn assert_publication_causality(&self) -> Result<(), String> { + assert_publication_causality(&self.causal_history()) + } + + pub fn history(&self) -> Vec { + self.shared.lock().history.clone() + } + + /// C23/C25/C26 monitor over the actual callback history (effects profile). + pub fn assert_effects_history(&self) -> Result<(), String> { + Self::assert_effects_events(&self.history()) + } + + pub fn assert_effects_events(history: &[HistoryEvent]) -> Result<(), String> { + struct Source { + at: i64, + settled: &'static str, + } + let mut sources: HashMap = HashMap::new(); + let mut active: Option = None; + let mut authorized = false; + let mut previous = -1i64; + for (index, e) in history.iter().enumerate() { + let fail = |reason: &str| { + Err(format!( + "effects contract event {index} {}: {reason}", + e.event + )) + }; + if e.at < previous { + return fail("elapsed time moved backward"); + } + previous = e.at; + match e.event { + "sourceStart" => { + if sources.contains_key(&e.id) || active.is_some() { + return fail("source started before prior fallback completed"); + } + sources.insert( + e.id, + Source { + at: e.at, + settled: "", + }, + ); + active = Some(e.id); + authorized = false; + } + "sourceSettlement" => { + let Some(source) = sources.get_mut(&e.id) else { + return fail("invalid source settlement identity"); + }; + if !source.settled.is_empty() + || (e.outcome != "resolve" && e.outcome != "reject") + { + return fail("invalid source settlement identity"); + } + source.settled = e.outcome; + } + "fallbackCompletion" => { + let Some(id) = active else { + return fail("fallback completion has no source"); + }; + let source = &sources[&id]; + if !e.duration_ms.is_finite() || e.duration_ms < 0.0 { + return fail("invalid observed fallback duration"); + } + let elapsed = (e.at - source.at) as f64; + if (e.duration_ms - elapsed).abs() > 1e-7 { + return Err(property_failure( + "C23", + "duration includes lookup or omits source time", + json!({"event":e.event, "index":index, "atMs":e.at, "elapsedMs":elapsed, "durationMs":e.duration_ms}), + )); + } + if !e.failed && (elapsed >= 10.0 || source.settled != "resolve") { + return Err(property_failure( + "C25", + "success must be accepted before its source deadline", + json!({"event":e.event, "index":index, "atMs":e.at, "elapsedMs":elapsed, "budgetMs":10, "settlement":source.settled, "failed":e.failed}), + )); + } + if e.failed && elapsed < 10.0 && source.settled != "reject" { + return Err(property_failure( + "C23", + "source lost its full source-relative budget", + json!({"event":e.event, "index":index, "atMs":e.at, "elapsedMs":elapsed, "budgetMs":10, "settlement":source.settled, "failed":e.failed}), + )); + } + active = None; + authorized = !e.failed; + } + "writeDispatch" => { + if !authorized { + return Err(property_failure( + "C26", + "publication without accepted source success", + json!({"event":e.event, "index":index, "atMs":e.at, "authorized":false}), + )); + } + } + _ => return fail("unknown monitor event"), + } + } + Ok(()) + } +} + +/// Block an external effect until the history releases it. +fn hold_effect( + shared: &Arc>, + effect: &str, + index: usize, +) -> impl std::future::Future> + Send { + let gate: Gate> = Gate::new(); + shared + .lock() + .effects + .entry(effect.to_string()) + .or_default() + .insert(index, gate.clone()); + async move { gate.wait().await } +} + +struct DriverCodec { + shared: Arc>, +} + +impl Codec for DriverCodec { + fn encode<'a>(&'a self, value: &'a Val) -> BoxFuture<'a, Result> { + let shared = self.shared.clone(); + let value = value.clone(); + Box::pin(async move { + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("dumps"); + (index, s.fault("holdDumps")) + }; + if hold { + hold_effect(&shared, "dump", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + if shared.lock().fault("dump") { + return Err("controlled serialization failure".into()); + } + Ok(match value { + Val::Absent => Payload::text("undefined"), + Val::Json(json) => Payload::text(serde_json::to_string(&json)?), + }) + }) + } + + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result> { + let shared = self.shared.clone(); + Box::pin(async move { + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("loads"); + (index, s.fault("holdLoads")) + }; + if hold { + hold_effect(&shared, "load", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + if shared.lock().fault("load") { + return Err("controlled deserialization failure".into()); + } + if payload.bytes == b"undefined" { + return Ok(Val::Absent); + } + let value: Value = serde_json::from_slice(&payload.bytes)?; + Ok(Val::Json(value)) + }) + } +} + +struct DriverRemote { + shared: Arc>, + clock: Arc, +} + +impl Remote for DriverRemote { + fn read( + &self, + request: ReadRequest, + context: ReadContext, + ) -> BoxFuture<'_, Result> { + let shared = self.shared.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("reads"); + let mut fields = Map::new(); + fields.insert("index".to_string(), Value::from(index)); + fields.insert("timeoutMs".to_string(), Value::from(context.timeout_ms)); + fields.insert( + "aborted".to_string(), + Value::Bool(context.cancel.is_cancelled()), + ); + s.record("readContext", fields); + (index, s.fault("holdReads")) + }; + { + let shared = shared.clone(); + context.cancel.on_cancel(move || { + let mut fields = Map::new(); + fields.insert("index".to_string(), Value::from(index)); + shared.lock().record("readAbort", fields); + }); + } + if hold { + hold_effect(&shared, "read", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + let mut s = shared.lock(); + if s.fault("read") { + return Err("controlled read failure".into()); + } + if let Some(reply) = s.reply.take() { + return Ok(read_result_from_untrusted_json(&reply)); + } + use dialcache::Clock; + let elapsed = clock.elapsed_ms(); + let raw = s.raw(&request.value_key, elapsed); + let marker = match &request.watermark_key { + Some(key) => s.raw(key, elapsed), + None => None, + }; + let tracked = request.watermark_key.is_some(); + Ok(decode_frame(raw.as_deref(), tracked, marker.as_deref())?) + }) + } + + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + let shared = self.shared.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + use dialcache::Clock; + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("writes"); + let at = clock.elapsed_ms(); + let owner = current_invocation(); + let source = owner.and_then(|owner| s.source_by_invocation.get(&owner).copied()); + s.causal_history.push(CausalEvent::WriteDispatch { + source, + owner, + at_ms: at, + }); + s.history.push(HistoryEvent { + event: "writeDispatch", + id: 0, + at, + outcome: "", + duration_ms: 0.0, + failed: false, + }); + let mut fields = Map::new(); + fields.insert("index".to_string(), Value::from(index)); + s.record("writeDispatch", fields); + s.push("writeTtls", Value::from(request.ttl_ms)); + (index, s.fault("holdWrites")) + }; + if hold { + hold_effect(&shared, "write", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + let mut s = shared.lock(); + if s.fault("write") { + return Err(Box::new(MaintenanceFailure) as BoxError); + } + let raw = encode_frame(&request.frame)?; + if !s.discard_writes { + let expires = clock.elapsed_ms() + request.ttl_ms as i64; + s.values.insert(request.value_key, Stored { raw, expires }); + } + Ok(()) + }) + } + + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + let shared = self.shared.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + use dialcache::Clock; + let mut s = shared.lock(); + s.increment("invalidations"); + if s.fault("write") { + return Err(Box::new(MaintenanceFailure) as BoxError); + } + if s.discard_invalidations { + return Ok(()); + } + let now = request.invalidated_at_ms as i64; + let mut cutoff = now + request.future_buffer_ms as i64; + let elapsed = clock.elapsed_ms(); + let prior = s.raw(&request.watermark_key, elapsed); + let old: i64 = prior + .and_then(|raw| String::from_utf8_lossy(&raw).parse().ok()) + .unwrap_or(0); + if old > cutoff { + cutoff = old; + } + let mut ttl: i64 = 7_200_000; + if cutoff - now + 3_660_000 > ttl { + ttl = cutoff - now + 3_660_000; + } + if let Some(item) = s.values.get(&request.watermark_key) { + if item.expires - elapsed > ttl { + ttl = item.expires - elapsed; + } + } + s.values.insert( + request.watermark_key, + Stored { + raw: cutoff.to_string().into_bytes(), + expires: elapsed + ttl, + }, + ); + Ok(()) + }) + } +} + +struct DriverObserver { + shared: Arc>, + clock: Arc, + shadow_hook: bool, +} + +fn labels_map(labels: &dialcache::Labels) -> Map { + let mut m = Map::new(); + m.insert( + "cacheNamespace".to_string(), + Value::from(labels.namespace.as_ref()), + ); + m.insert("useCase".to_string(), Value::from(labels.use_case.as_ref())); + m.insert("keyType".to_string(), Value::from(labels.key_type.as_ref())); + m.insert("layer".to_string(), Value::from(labels.layer.as_str())); + m +} + +fn outcome_map(labels: &dialcache::observe::OutcomeLabels) -> Map { + let mut m = Map::new(); + m.insert( + "cacheNamespace".to_string(), + Value::from(labels.namespace.as_ref()), + ); + m.insert("useCase".to_string(), Value::from(labels.use_case.as_ref())); + m.insert("keyType".to_string(), Value::from(labels.key_type.as_ref())); + m +} + +impl Observer for DriverObserver { + fn observe(&self, event: &Event) { + use dialcache::Clock; + let mut s = self.shared.lock(); + let (kind, mut fields): (&str, Map) = match event { + Event::Request { labels } => ("request", labels_map(labels)), + Event::Miss { labels, reason } => { + let mut m = labels_map(labels); + m.insert("reason".to_string(), Value::from(reason.as_str())); + ("miss", m) + } + Event::Disabled { labels, reason } => { + let mut m = labels_map(labels); + m.insert("reason".to_string(), Value::from(reason.as_str())); + ("disabled", m) + } + Event::Error { + labels, + error, + in_fallback, + } => { + let mut m = labels_map(labels); + m.insert("error".to_string(), Value::from(error.as_str())); + m.insert("inFallback".to_string(), Value::Bool(*in_fallback)); + if *in_fallback && *error == dialcache::observe::ErrorKind::Fallback { + s.fallback_failed = true; + } + ("error", m) + } + Event::Invalidation { + namespace, + key_type, + layer, + } => { + let mut m = Map::new(); + m.insert( + "cacheNamespace".to_string(), + Value::from(namespace.as_ref()), + ); + m.insert("keyType".to_string(), Value::from(key_type.as_ref())); + m.insert("layer".to_string(), Value::from(layer.as_str())); + ("invalidation", m) + } + Event::Coalesced { labels, scope } => { + let mut m = outcome_map(labels); + m.insert("scope".to_string(), Value::from(scope.as_str())); + ("coalesced", m) + } + Event::ShadowValidation { labels, outcome } => { + s.push("shadow", Value::from(outcome.as_str())); + let mut m = outcome_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + ("shadowValidation", m) + } + Event::ShadowValueAge { + labels, + outcome, + seconds, + } => { + let mut m = outcome_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("shadowAge", m) + } + Event::FutureTimestampOffset { labels, seconds } => { + let mut m = labels_map(labels); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("futureOffset", m) + } + Event::StaleRecovery { labels, outcome } => { + s.push("recovery", Value::from(outcome.as_str())); + let mut m = outcome_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + ("staleRecovery", m) + } + Event::StaleRecoveryValueAge { + labels, + outcome, + seconds, + } => { + let mut m = outcome_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("recoveryAge", m) + } + Event::Compression { labels, outcome } => { + let mut m = labels_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + ("compression", m) + } + Event::Get { labels, seconds } => { + let mut m = labels_map(labels); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("get", m) + } + Event::Fallback { labels, seconds } => { + let mut m = labels_map(labels); + m.insert("seconds".to_string(), Value::from(*seconds)); + let at = self.clock.elapsed_ms(); + let failed = s.fallback_failed; + s.history.push(HistoryEvent { + event: "fallbackCompletion", + id: 0, + at, + outcome: "", + duration_ms: seconds * 1000.0, + failed, + }); + s.fallback_failed = false; + ("fallback", m) + } + Event::Serialization { + labels, + operation, + seconds, + } => { + let mut m = labels_map(labels); + m.insert("operation".to_string(), Value::from(operation.as_str())); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("serialization", m) + } + Event::Size { labels, bytes } => { + let mut m = labels_map(labels); + m.insert("bytes".to_string(), Value::from(*bytes)); + ("size", m) + } + Event::StoredSize { labels, bytes } => { + let mut m = labels_map(labels); + m.insert("bytes".to_string(), Value::from(*bytes)); + ("storedSize", m) + } + Event::CompressionRatio { labels, ratio } => { + let mut m = labels_map(labels); + m.insert("value".to_string(), Value::from(*ratio)); + ("compressionRatio", m) + } + Event::CompressionDuration { + labels, + operation, + seconds, + } => { + let mut m = labels_map(labels); + m.insert("operation".to_string(), Value::from(operation.as_str())); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("compressionDuration", m) + } + }; + fields.retain(|k, _| k != "value" || kind == "compressionRatio"); + s.record(kind, fields); + let fail = s.observer_fails(); + drop(s); + if fail { + controlled_panic(); + } + } + + fn observes_shadow_outcomes(&self) -> bool { + self.shadow_hook + } +} + +struct DriverLogger { + shared: Arc>, +} + +impl Logger for DriverLogger { + fn log(&self, event: &LogEvent) { + let mut s = self.shared.lock(); + if let LogEvent::ShadowMismatch(details) = event { + let mut fields = Map::new(); + fields.insert( + "cacheNamespace".to_string(), + Value::from(details.namespace.as_ref()), + ); + fields.insert( + "useCase".to_string(), + Value::from(details.use_case.as_ref()), + ); + fields.insert( + "keyType".to_string(), + Value::from(details.key_type.as_ref()), + ); + fields.insert("outcome".to_string(), Value::from("mismatch")); + fields.insert( + "cacheKey".to_string(), + Value::from(details.cache_key.as_str()), + ); + fields.insert( + "cachedValueJson".to_string(), + details + .cached_value_json + .clone() + .map(Value::from) + .unwrap_or(Value::Null), + ); + fields.insert( + "sourceValueJson".to_string(), + details + .source_value_json + .clone() + .map(Value::from) + .unwrap_or(Value::Null), + ); + s.record("mismatchWarning", fields); + } + let fail = s.observer_fails(); + drop(s); + if fail { + controlled_panic(); + } + } +} + +/// Local storage that fails on command, at the native storage boundary. +struct FaultStore { + inner: Option, + shared: Arc>, +} + +impl LocalStore for FaultStore { + fn get(&mut self, key: &str, now_ms: i64) -> Result { + if self.shared.lock().fault("localStorage") { + return Err("controlled local storage failure".into()); + } + match self.inner.as_mut() { + Some(inner) => inner.get(key, now_ms), + None => Ok(LocalRead::Absent), + } + } + + fn put(&mut self, key: String, entry: LocalEntry) -> Result, BoxError> { + if self.shared.lock().fault("localStorage") { + return Err("controlled local storage failure".into()); + } + match self.inner.as_mut() { + Some(inner) => inner.put(key, entry), + None => Ok(None), + } + } +} + +#[allow(dead_code)] +fn _unused(shared: &Shared) -> Map { + shared_labels(shared) +} diff --git a/rust/tests/formal/fixtures.rs b/rust/tests/formal/fixtures.rs new file mode 100644 index 00000000..f969a8e4 --- /dev/null +++ b/rust/tests/formal/fixtures.rs @@ -0,0 +1,327 @@ +//! Shared loading of the portable protocol vector corpus. +//! +//! Mirrors `go/protocol_test.go` `vectors()` and +//! `go/generated_protocol_test.go`: the fixed vectors in +//! `formal/protocol-vectors.json` are merged with every generated protocol +//! artifact registered in `formal/execution.json`, after the artifact's +//! provenance fingerprints are verified against the model sources. +//! +//! Include from a test binary with `#[path = "formal/fixtures.rs"] mod fixtures;`. + +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde_json::Value; + +/// Replaces every lone UTF-16 surrogate escape in fixture JSON so the text +/// can be parsed at all. Components carrying this marker are unrepresentable +/// as Rust strings and are rejected at the fixture boundary. +pub const LONE_SURROGATE_MARKER: char = '\u{E000}'; + +/// Environment variable selecting the corpus: `all` (default), `generated` or `fixed`. +pub const CORPUS_ENV: &str = "DIALCACHE_PROTOCOL_CORPUS"; + +/// Resolve a repository-relative path from the crate manifest directory's parent. +pub fn repo_path(relative: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(relative) +} + +fn read_repo_text(relative: &str) -> String { + let path = repo_path(relative); + fs::read_to_string(&path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())) +} + +/// Parse fixture JSON that may contain lone surrogate escapes (`\ud800`, +/// `\udc00`), which `serde_json` rejects. Each lone escape is replaced with +/// [`LONE_SURROGATE_MARKER`] before parsing; valid surrogate pairs and every +/// other escape are left untouched. +pub fn load_json_marking_lone_surrogates(path: impl AsRef) -> Value { + let path = path.as_ref(); + let text = + fs::read_to_string(path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + let marked = mark_lone_surrogate_escapes(&text); + serde_json::from_str(&marked) + .unwrap_or_else(|error| panic!("parse {}: {error}", path.display())) +} + +/// Replace every lone surrogate escape in JSON source text with `\ue000`. +pub fn mark_lone_surrogate_escapes(text: &str) -> String { + const MARKER_ESCAPE: &str = "\\ue000"; + let bytes = text.as_bytes(); + let mut out = String::with_capacity(text.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] != b'\\' { + let rest = &text[index..]; + let ch = rest.chars().next().expect("index is on a char boundary"); + out.push(ch); + index += ch.len_utf8(); + continue; + } + match unicode_escape_at(bytes, index) { + Some(unit) if (0xD800..=0xDBFF).contains(&unit) => { + let pair = unicode_escape_at(bytes, index + 6) + .filter(|low| (0xDC00..=0xDFFF).contains(low)); + if pair.is_some() { + out.push_str(&text[index..index + 12]); + index += 12; + } else { + out.push_str(MARKER_ESCAPE); + index += 6; + } + } + Some(unit) if (0xDC00..=0xDFFF).contains(&unit) => { + // A low surrogate preceded by a high one was consumed with its pair above. + out.push_str(MARKER_ESCAPE); + index += 6; + } + Some(_) => { + out.push_str(&text[index..index + 6]); + index += 6; + } + None => { + // Any other escape: copy the backslash and the escaped byte so a + // `\\` cannot be mistaken for the start of a `\u` escape. + let end = (index + 2).min(bytes.len()); + out.push_str(&text[index..end]); + index = end; + } + } + } + out +} + +/// The code unit of a `\uXXXX` escape starting at `index`, when one is there. +fn unicode_escape_at(bytes: &[u8], index: usize) -> Option { + if index + 6 > bytes.len() || bytes[index] != b'\\' || bytes[index + 1] != b'u' { + return None; + } + let hex = std::str::from_utf8(&bytes[index + 2..index + 6]).ok()?; + if !hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + u16::from_str_radix(hex, 16).ok() +} + +pub use crate::digest::sha256_hex; + +/// Every generated protocol vector group registered in `formal/execution.json`, +/// keyed by field name, after validating each artifact's schema, provenance +/// and case inventory the way `go/generated_protocol_test.go` does. +pub fn quint_protocol_groups() -> BTreeMap> { + let manifest: Value = serde_json::from_str(&read_repo_text("formal/execution.json")) + .expect("parse formal/execution.json"); + let models = manifest["models"] + .as_array() + .expect("execution.json models array"); + let mut groups: BTreeMap> = BTreeMap::new(); + for model in models { + let Some(export) = model + .get("vectorExport") + .filter(|export| export["kind"] == "protocol") + else { + continue; + }; + let model_path = model["path"].as_str().expect("model path"); + let artifact = export["artifact"].as_str().expect("vectorExport.artifact"); + let sources: Vec<&str> = export["sources"] + .as_array() + .expect("vectorExport.sources") + .iter() + .map(|s| s.as_str().expect("source path")) + .collect(); + let cases = export["cases"].as_u64().expect("vectorExport.cases") as usize; + + let envelope = load_json_marking_lone_surrogates(repo_path(artifact)); + let object = envelope + .as_object() + .unwrap_or_else(|| panic!("{artifact}: artifact is not an object")); + assert_eq!( + object.get("schemaVersion"), + Some(&Value::from(3)), + "{artifact}: unsupported schemaVersion" + ); + let provenance = &object["provenance"]; + assert_eq!( + provenance["model"].as_str(), + Some(model_path), + "{artifact}: provenance.model mismatch" + ); + let digests = provenance["sourceSha256"] + .as_object() + .unwrap_or_else(|| panic!("{artifact}: provenance.sourceSha256 missing")); + assert_eq!( + digests.len(), + sources.len(), + "{artifact}: provenance source count differs from execution.json" + ); + for source in &sources { + let bytes = fs::read(repo_path(source)) + .unwrap_or_else(|error| panic!("read {source}: {error}")); + let expected = digests + .get(*source) + .and_then(Value::as_str) + .unwrap_or_else(|| panic!("{artifact}: no provenance digest for {source}")); + assert_eq!( + sha256_hex(&bytes), + expected, + "stale generated protocol source: {source}" + ); + } + + let mut seen = std::collections::BTreeSet::new(); + let mut count = 0; + for (field, rows) in object { + if field == "schemaVersion" || field == "provenance" { + continue; + } + let rows = rows + .as_array() + .unwrap_or_else(|| panic!("{artifact}: field {field} is not an array")); + for row in rows { + let name = row["name"].as_str().unwrap_or(""); + assert!( + !name.is_empty() && seen.insert(name.to_owned()), + "{artifact}: duplicate or missing generated protocol case {name:?}" + ); + } + count += rows.len(); + groups + .entry(field.clone()) + .or_default() + .extend(rows.iter().cloned()); + } + assert_eq!( + count, cases, + "{artifact}: incomplete generated protocol vector inventory" + ); + } + groups +} + +/// The effective corpus selection: the `DIALCACHE_PROTOCOL_CORPUS` variable +/// when set and non-empty, otherwise `default`. +pub fn corpus_selection(default: &str) -> String { + match std::env::var(CORPUS_ENV) { + Ok(value) if !value.is_empty() => value, + _ => default.to_owned(), + } +} + +/// The fixed protocol vectors merged with the generated groups, keyed by +/// group name. `selection` is the default corpus (`all`, `generated` or +/// `fixed`); `DIALCACHE_PROTOCOL_CORPUS` overrides it like `go/protocol_test.go`. +pub fn protocol_groups(selection: &str) -> BTreeMap> { + let selection = corpus_selection(selection); + assert!( + matches!(selection.as_str(), "all" | "generated" | "fixed"), + "unknown protocol corpus selection {selection:?}" + ); + + let fixed = load_json_marking_lone_surrogates(repo_path("formal/protocol-vectors.json")); + let fixed = fixed + .as_object() + .expect("protocol-vectors.json is an object"); + assert_eq!( + fixed.get("schemaVersion"), + Some(&Value::from(3)), + "unsupported protocol vector schema" + ); + let mut groups: BTreeMap> = fixed + .iter() + .filter(|(name, _)| name.as_str() != "schemaVersion") + .map(|(name, rows)| { + let rows = rows + .as_array() + .unwrap_or_else(|| panic!("protocol-vectors.json: {name} is not an array")); + ( + name.clone(), + if selection == "generated" { + Vec::new() + } else { + rows.clone() + }, + ) + }) + .collect(); + if selection != "fixed" { + for (name, rows) in quint_protocol_groups() { + groups + .get_mut(&name) + .unwrap_or_else(|| panic!("unknown generated protocol group {name}")) + .extend(rows); + } + } + groups +} + +/// An executed protocol assertion, distinguished from malformed fixture input +/// and environmental codec failures by the mutation measurement. +pub fn assertion_mismatch(expected: impl std::fmt::Debug, actual: impl std::fmt::Debug) -> String { + format!("PROTOCOL_ASSERTION_FAILURE expected: {expected:?}\nactual: {actual:?}") +} + +#[cfg(test)] +mod tests { + #[allow(unused_imports)] + use super::*; + + #[test] + fn sha256_matches_known_digests() { + assert_eq!( + sha256_hex(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + sha256_hex(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let long = b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + assert_eq!( + sha256_hex(long), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ); + assert_eq!( + sha256_hex(&[b'a'; 1_000_000]), + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0" + ); + } + + #[test] + fn lone_surrogates_are_marked_and_pairs_kept() { + assert_eq!(mark_lone_surrogate_escapes(r#""\ud800""#), r#""\ue000""#); + assert_eq!(mark_lone_surrogate_escapes(r#""\udc00""#), r#""\ue000""#); + assert_eq!( + mark_lone_surrogate_escapes(r#""\ud83d\ude00""#), + r#""\ud83d\ude00""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""\udc00\ud800""#), + r#""\ue000\ue000""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""\ud800\ud800\udc00""#), + r#""\ue000\ud800\udc00""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""a\\ud800""#), + r#""a\\ud800""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""\u00e9\ue000""#), + r#""\u00e9\ue000""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""\uD800x\uDC00""#), + r#""\ue000x\ue000""# + ); + let parsed: Value = + serde_json::from_str(&mark_lone_surrogate_escapes(r#"{"id":"x\ud800y"}"#)).unwrap(); + assert_eq!(parsed["id"].as_str(), Some("x\u{e000}y")); + } +} diff --git a/rust/tests/formal/frame_vectors.rs b/rust/tests/formal/frame_vectors.rs new file mode 100644 index 00000000..405499ca --- /dev/null +++ b/rust/tests/formal/frame_vectors.rs @@ -0,0 +1,390 @@ +//! Runners for the frame, decode, timestamp, duration and envelope vector +//! groups of the portable protocol corpus. Each mirrors the corresponding +//! part of `go/protocol_test.go` field for field. + +use dialcache::limits::MAX_DECOMPRESSED_BYTES; +use dialcache::observe::CompressionOutcome; +use dialcache::protocol::{ + ceil_supported_cache_ttl_ms, compress_payload, decode_frame, decompress_payload, encode_frame, + escape_raw_payload, validate_timestamp_ms, CompressionConfig, ProtocolError, + MARKER_ZSTD_BINARY, MARKER_ZSTD_UTF8, +}; +use dialcache::{Frame, Payload, ReadResult}; +use serde_json::{json, Value}; + +use super::fixtures::assertion_mismatch; + +use super::fixtures::LONE_SURROGATE_MARKER; + +/// The fixture level every `codecBytes` entry was measured at. +const FIXTURE_ZSTD_LEVEL: i32 = 3; + +fn field<'a>(vector: &'a Value, name: &str) -> Result<&'a Value, String> { + vector + .get(name) + .ok_or_else(|| format!("missing field {name}")) +} + +fn text_field<'a>(vector: &'a Value, name: &str) -> Result<&'a str, String> { + field(vector, name)? + .as_str() + .ok_or_else(|| format!("field {name} is not a string")) +} + +fn unhex(text: &str) -> Result, String> { + hex::decode(text).map_err(|error| format!("invalid hex {text:?}: {error}")) +} + +fn usize_field(vector: &Value, name: &str) -> Result { + let number = field(vector, name)? + .as_u64() + .ok_or_else(|| format!("field {name} is not an unsigned integer"))?; + usize::try_from(number).map_err(|_| format!("field {name} exceeds usize")) +} + +/// The `payloadType` / `payloadUtf8` / `payloadHex` triple as a payload. +/// `replace_lone_surrogates` applies the binding's input conversion: an +/// unpaired UTF-16 surrogate in caller text becomes U+FFFD. +fn vector_payload(vector: &Value, replace_lone_surrogates: bool) -> Result { + match text_field(vector, "payloadType")? { + "binary" => Ok(Payload::binary(unhex(text_field(vector, "payloadHex")?)?)), + "string" => { + let text = text_field(vector, "payloadUtf8")?; + if replace_lone_surrogates { + Ok(Payload::text( + text.replace(LONE_SURROGATE_MARKER, "\u{FFFD}"), + )) + } else { + Ok(Payload::text(text)) + } + } + other => Err(format!("unknown payloadType {other:?}")), + } +} + +/// A numeric fixture input; JSON carries nonfinite values as `specialInput`. +fn number_input(vector: &Value) -> Result { + match vector.get("specialInput").and_then(Value::as_str) { + Some("NaN") => Ok(f64::NAN), + Some("Infinity") => Ok(f64::INFINITY), + Some("-Infinity") => Ok(f64::NEG_INFINITY), + Some(other) => Err(format!("unknown specialInput {other:?}")), + None => field(vector, "input")? + .as_f64() + .ok_or_else(|| "input is not a number".to_string()), + } +} + +fn outcome_name(outcome: Option) -> &'static str { + outcome.map_or("passthrough", CompressionOutcome::as_str) +} + +fn max_decompressed_bytes(vector: &Value) -> Result { + match vector.get("maxDecompressedBytes") { + None | Some(Value::Null) => Ok(MAX_DECOMPRESSED_BYTES), + Some(_) => usize_field(vector, "maxDecompressedBytes"), + } +} + +/// `frameVectors`: encoding a payload with a stamp yields the exact frame bytes. +pub fn check_frame_vector(vector: &Value) -> Result<(), String> { + let created_at_ms = field(vector, "createdAtMs")? + .as_u64() + .ok_or("createdAtMs is not an unsigned integer")?; + let payload = vector_payload(vector, true)?; + let expected = text_field(vector, "frameHex")?; + let frame = encode_frame(&Frame { + created_at_ms, + payload, + }) + .map_err(|error| assertion_mismatch(expected, error))?; + let actual = hex::encode(&frame); + if actual != expected { + return Err(assertion_mismatch(expected, actual)); + } + Ok(()) +} + +pub fn check_tracked_decode_vector(vector: &Value) -> Result<(), String> { + check_decode_vector(vector, true) +} + +pub fn check_untracked_decode_vector(vector: &Value) -> Result<(), String> { + check_decode_vector(vector, false) +} + +/// `trackedDecodeVectors` / `untrackedDecodeVectors`: `frameHex` null is an +/// absent value and `""` a present empty one; `watermarkUtf8` null is absent. +/// The classified result is compared with `expected` as JSON, retaining +/// integer precision. +fn check_decode_vector(vector: &Value, tracked: bool) -> Result<(), String> { + let raw = match field(vector, "frameHex")? { + Value::Null => None, + Value::String(text) => Some(unhex(text)?), + _ => return Err("frameHex is neither null nor a string".to_string()), + }; + let watermark = match vector.get("watermarkUtf8").unwrap_or(&Value::Null) { + Value::Null => None, + Value::String(text) => Some(text.as_bytes().to_vec()), + _ => return Err("watermarkUtf8 is neither null nor a string".to_string()), + }; + let expected = field(vector, "expected")?; + let actual = match decode_frame(raw.as_deref(), tracked, watermark.as_deref()) { + Err(ProtocolError::PayloadEncoding) => json!({ "kind": "payload_encoding_error" }), + Err(other) => return Err(assertion_mismatch(expected, other)), + Ok(ReadResult::Miss { + reason, + observed_watermark_ms, + }) => { + let mut object = json!({ "kind": "miss", "reason": reason.as_str() }); + if let Some(fence) = observed_watermark_ms { + object["observedWatermarkMs"] = json!(fence); + } + object + } + Ok(ReadResult::Hit(frame)) => { + let mut object = json!({ "kind": "hit", "createdAtMs": frame.created_at_ms }); + if frame.payload.binary { + object["payloadType"] = json!("binary"); + object["payloadHex"] = json!(hex::encode(&frame.payload.bytes)); + } else { + object["payloadType"] = json!("string"); + let text = String::from_utf8(frame.payload.bytes) + .map_err(|error| assertion_mismatch(expected, error))?; + object["payloadUtf8"] = json!(text); + } + object + } + }; + if &actual != expected { + return Err(assertion_mismatch(expected, actual)); + } + Ok(()) +} + +/// `invalidTimestampVectors`: every input is outside the timestamp domain. +pub fn check_invalid_timestamp_vector(vector: &Value) -> Result<(), String> { + match validate_timestamp_ms(number_input(vector)?) { + Ok(accepted) => Err(assertion_mismatch("invalid timestamp rejected", accepted)), + Err(_) => Ok(()), + } +} + +/// `durationVectors`: `expected` null rejects; otherwise the ceiled TTL. +pub fn check_duration_vector(vector: &Value) -> Result<(), String> { + let expected = match field(vector, "expected")? { + Value::Null => None, + value => Some( + value + .as_u64() + .ok_or("expected is neither null nor an unsigned integer")?, + ), + }; + let result = ceil_supported_cache_ttl_ms(number_input(vector)?); + match (expected, result) { + (None, Err(_)) => Ok(()), + (None, Ok(accepted)) => Err(assertion_mismatch("invalid duration rejected", accepted)), + (Some(expected), Ok(actual)) if actual == expected => Ok(()), + (Some(expected), other) => Err(assertion_mismatch(expected, other)), + } +} + +/// `envelopeVectors`: escaping, unconditional read interpretation and the +/// escape round trip over raw binary input. +pub fn check_envelope_vector(vector: &Value) -> Result<(), String> { + let raw = Payload::binary(unhex(text_field(vector, "inputHex")?)?); + let escaped = escape_raw_payload(raw.clone()); + if escaped.bytes != unhex(text_field(vector, "escapedHex")?)? { + return Err(assertion_mismatch( + text_field(vector, "escapedHex")?, + hex::encode(&escaped.bytes), + )); + } + let decoded = decompress_payload(raw.clone(), MAX_DECOMPRESSED_BYTES); + let expected_outcome = text_field(vector, "outcome")?; + if outcome_name(decoded.outcome) != expected_outcome + || !decoded.payload.binary + || decoded.payload.bytes != unhex(text_field(vector, "decodedHex")?)? + { + return Err(assertion_mismatch( + (expected_outcome, text_field(vector, "decodedHex")?), + &decoded, + )); + } + if decompress_payload(escaped, MAX_DECOMPRESSED_BYTES) + .payload + .bytes + != raw.bytes + { + return Err(assertion_mismatch(&raw.bytes, "escape roundtrip differs")); + } + Ok(()) +} + +/// `compressedDecodeVectors`: the native decoder confirms the environmental +/// `codecFixture`, then the wrapper's outcome, type and bytes are checked +/// under the optional per-call cap. +pub fn check_compressed_decode_vector(vector: &Value) -> Result<(), String> { + let input = unhex(text_field(vector, "inputHex")?)?; + if let Some(fixture) = vector + .get("codecFixture") + .filter(|fixture| !fixture.is_null()) + { + let succeeds = field(fixture, "succeeds")? + .as_bool() + .ok_or("codecFixture.succeeds is not a boolean")?; + let native = zstd::decode_all(input.get(1..).unwrap_or_default()); + match (succeeds, native) { + (true, Ok(decoded)) if decoded == unhex(text_field(fixture, "decodedHex")?)? => {} + (true, other) => return Err(format!("native codec fixture differs: {other:?}")), + (false, Ok(decoded)) => { + return Err(format!( + "native decoder accepted rejected fixture: {}", + hex::encode(decoded) + )) + } + (false, Err(_)) => {} + } + } + let want = vector_payload(vector, false)?; + let max = max_decompressed_bytes(vector)?; + let outcome = vector + .get("outcome") + .and_then(Value::as_str) + .unwrap_or("decompressed"); + let got = decompress_payload(Payload::binary(input), max); + if outcome_name(got.outcome) != outcome || got.payload != want { + return Err(assertion_mismatch((outcome, want), got)); + } + Ok(()) +} + +/// `compressionWriteVectors`: when this binding's native level-3 encoder +/// reproduces the TypeScript fixture length, the TypeScript per-binding +/// expectations apply; otherwise only the generic outcome and the selection +/// rule are checked. Every result must round-trip and respect the escape rule. +pub fn check_compression_write_vector(vector: &Value) -> Result<(), String> { + let name = vector + .get("name") + .and_then(Value::as_str) + .unwrap_or(""); + let raw = vector_payload(vector, false)?; + let escaped = escape_raw_payload(raw.clone()); + let threshold_bytes = usize_field(vector, "thresholdBytes")?; + let max = max_decompressed_bytes(vector)?; + + let mut modeled = None; + if let Some(codec_bytes) = vector.get("codecBytes").filter(|codec| !codec.is_null()) { + let fixture = usize_field(codec_bytes, "typescript")?; + let native = native_zstd_len(&raw.bytes)?; + if native == fixture { + modeled = Some(field(field(vector, "expectedByBinding")?, "typescript")?); + println!("compressionWriteVectors {name}: native level-3 length {native} matches the TypeScript fixture; using its expectations"); + } else { + println!("compressionWriteVectors {name}: native level-3 length {native} differs from the TypeScript fixture {fixture}; checking the generic outcome and selection rule only"); + } + } + + let expected_outcome = match modeled { + Some(expected) => text_field(expected, "outcome")?, + None => text_field(vector, "outcome")?, + }; + let got = compress_payload( + raw.clone(), + &CompressionConfig { + threshold_bytes, + level: FIXTURE_ZSTD_LEVEL, + }, + max, + ) + .map_err(|error| assertion_mismatch(expected_outcome, error))?; + if got.outcome.as_str() != expected_outcome { + return Err(assertion_mismatch(expected_outcome, &got)); + } + if let Some(expected) = modeled { + let stored = usize_field(expected, "storedBytes")?; + if got.stored_bytes != stored { + return Err(assertion_mismatch(stored, got.stored_bytes)); + } + if vector.get("originalBytes").is_none() + || vector.get("rawStoredBytes").is_none() + || vector.get("escapedHex").is_none() + { + return Err( + "modeled vector lacks originalBytes, rawStoredBytes or escapedHex".to_string(), + ); + } + } + if let Some(original) = vector.get("originalBytes").filter(|value| !value.is_null()) { + let original = original.as_u64().ok_or("originalBytes is not a number")? as usize; + if got.original_bytes != original { + return Err(assertion_mismatch(original, got.original_bytes)); + } + } + if let Some(raw_stored) = vector + .get("rawStoredBytes") + .filter(|value| !value.is_null()) + { + let raw_stored = raw_stored + .as_u64() + .ok_or("rawStoredBytes is not a number")? as usize; + if escaped.len() != raw_stored { + return Err(assertion_mismatch(raw_stored, escaped.len())); + } + } + if let Some(escaped_hex) = vector.get("escapedHex").and_then(Value::as_str) { + if escaped.bytes != unhex(escaped_hex)? { + return Err(assertion_mismatch(escaped_hex, hex::encode(&escaped.bytes))); + } + } + + let decoded = decompress_payload(got.payload.clone(), MAX_DECOMPRESSED_BYTES); + if decoded.payload != raw { + return Err(assertion_mismatch(&raw, &decoded)); + } + if got.outcome == CompressionOutcome::Compressed { + if !got.payload.binary + || got.stored_bytes >= escaped.len() + || got.payload.len() != got.stored_bytes + { + return Err(assertion_mismatch( + "binary payload shorter than escaped raw with accurate stored size", + &got, + )); + } + let mut marker = if raw.binary { + MARKER_ZSTD_BINARY + } else { + MARKER_ZSTD_UTF8 + }; + if let Some(expected) = modeled { + marker = u8::try_from( + field(expected, "marker")? + .as_i64() + .ok_or("marker is not an integer")?, + ) + .map_err(|_| "modeled marker is not a byte".to_string())?; + } + if got.payload.bytes.first() != Some(&marker) { + return Err(assertion_mismatch(marker, got.payload.bytes.first())); + } + } else if got.payload != escaped { + return Err(assertion_mismatch(&escaped, &got)); + } + Ok(()) +} + +/// The native single-frame, checksum-free zstd length at the fixture level, +/// computed independently of the crate's wrapper. +fn native_zstd_len(bytes: &[u8]) -> Result { + let describe = |code: usize| format!("native zstd: {}", zstd_safe::get_error_name(code)); + let mut context = zstd_safe::CCtx::create(); + context + .set_parameter(zstd_safe::CParameter::CompressionLevel(FIXTURE_ZSTD_LEVEL)) + .map_err(describe)?; + context + .set_parameter(zstd_safe::CParameter::ChecksumFlag(false)) + .map_err(describe)?; + let mut out = vec![0u8; zstd_safe::compress_bound(bytes.len())]; + context.compress2(&mut out[..], bytes).map_err(describe) +} diff --git a/rust/tests/formal/gate.rs b/rust/tests/formal/gate.rs new file mode 100644 index 00000000..0f499856 --- /dev/null +++ b/rust/tests/formal/gate.rs @@ -0,0 +1,88 @@ +//! A settle-once gate the driver holds and library callbacks await. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll, Waker}; + +use parking_lot::Mutex; + +struct GateState { + value: Option, + wakers: Vec, +} + +/// Driver-owned gate: external work blocks on it until the history releases it. +pub struct Gate { + inner: Arc>>, +} + +impl Clone for Gate { + fn clone(&self) -> Self { + Gate { + inner: self.inner.clone(), + } + } +} + +impl Default for Gate { + fn default() -> Self { + Gate::new() + } +} + +impl Gate { + pub fn new() -> Self { + Gate { + inner: Arc::new(Mutex::new(GateState { + value: None, + wakers: Vec::new(), + })), + } + } + + /// Release every waiter with `value`. Returns false if already settled. + pub fn settle(&self, value: T) -> bool { + let wakers = { + let mut state = self.inner.lock(); + if state.value.is_some() { + return false; + } + state.value = Some(value); + std::mem::take(&mut state.wakers) + }; + for waker in wakers { + waker.wake(); + } + true + } + + pub fn is_settled(&self) -> bool { + self.inner.lock().value.is_some() + } + + pub fn wait(&self) -> GateWait { + GateWait { gate: self.clone() } + } +} + +pub struct GateWait { + gate: Gate, +} + +impl Unpin for GateWait {} + +impl Future for GateWait { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut state = self.gate.inner.lock(); + if let Some(value) = &state.value { + return Poll::Ready(value.clone()); + } + if !state.wakers.iter().any(|w| w.will_wake(cx.waker())) { + state.wakers.push(cx.waker().clone()); + } + Poll::Pending + } +} diff --git a/rust/tests/formal/invalidation_vectors.rs b/rust/tests/formal/invalidation_vectors.rs new file mode 100644 index 00000000..08faadf0 --- /dev/null +++ b/rust/tests/formal/invalidation_vectors.rs @@ -0,0 +1,327 @@ +//! Real-server replay of the invalidation transition vectors +//! (`formal/PROTOCOL.md`, "Invalidation vector schema 2"). +//! +//! Mirrors `go/redis_integration_test.go` `testInvalidationVectors`: the 49 +//! fixed vectors of `formal/invalidation-vectors.json` are merged with the +//! 288 Quint-generated vectors of `formal/quint-invalidation-vectors.json` +//! after the generated corpus's provenance fingerprints are verified. Each +//! vector installs `existing` atomically, runs the adapter's raw decimal +//! invalidation, then observes type, content and retention atomically and +//! compares them with `expected.state`. TTL comparison subtracts only the +//! server time measured between setup and observation; persistence (`-1`) +//! and absence (`-2`) are exact. +//! +//! Every function takes the adapter and a connection so the including test +//! binary owns Docker orchestration and stays small. Include with +//! `#[path = "formal/invalidation_vectors.rs"] mod invalidation_vectors;`. + +#![allow(dead_code)] + +use std::fs; +use std::path::Path; + +use dialcache::redis::{RedisAdapter, RedisConnection}; +use redis::Value; +use serde::Deserialize; + +/// Fixed corpus size pinned by every port. +pub const FIXED_VECTORS: usize = 49; +/// Generated corpus size pinned by every port. +pub const GENERATED_VECTORS: usize = 288; +const SCHEMA_VERSION: u64 = 2; +const MODEL: &str = "formal/dialcache-invalidation-transition.qnt"; +const GENERATOR: &str = "formal/generate-invalidation-vectors.mjs"; + +/// One tagged Redis key state: `absent`, `string` with `value`, or `list` +/// with ordered `values`; `ttl_ms` is `-2` absent, `-1` persistent or positive. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct State { + pub kind: String, + #[serde(default)] + pub value: Option, + #[serde(default)] + pub values: Option>, + pub ttl_ms: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Expected { + #[serde(default)] + pub error: bool, + pub state: State, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct Vector { + pub name: String, + pub existing: State, + /// Raw decimal argument text, passed to the script unparsed. + pub future_buffer_ms: String, + /// Raw decimal argument text, passed to the script unparsed. + pub invalidated_at_ms: String, + pub expected: Expected, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Corpus { + schema_version: u64, + #[serde(default)] + provenance: Option, + vectors: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Provenance { + model: String, + source_sha256: std::collections::BTreeMap, +} + +fn read(repo_root: &Path, relative: &str) -> Vec { + let path = repo_root.join(relative); + fs::read(&path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())) +} + +/// Load and merge both corpora exactly as Go does. `sha256_hex` fingerprints +/// the generator's model and script sources so stale generated vectors are +/// rejected rather than replayed. +pub fn load_corpus(repo_root: &Path, sha256_hex: impl Fn(&[u8]) -> String) -> Vec { + let fixed: Corpus = + serde_json::from_slice(&read(repo_root, "formal/invalidation-vectors.json")) + .expect("parse fixed invalidation corpus"); + assert_eq!( + fixed.schema_version, SCHEMA_VERSION, + "unsupported fixed corpus schema" + ); + assert_eq!( + fixed.vectors.len(), + FIXED_VECTORS, + "unsupported fixed corpus size" + ); + + let generated: Corpus = + serde_json::from_slice(&read(repo_root, "formal/quint-invalidation-vectors.json")) + .expect("parse Quint invalidation corpus"); + assert_eq!( + generated.schema_version, SCHEMA_VERSION, + "unsupported Quint corpus schema" + ); + assert_eq!( + generated.vectors.len(), + GENERATED_VECTORS, + "unsupported Quint corpus size" + ); + let provenance = generated + .provenance + .expect("Quint corpus records provenance"); + assert_eq!( + provenance.model, MODEL, + "invalid Quint invalidation provenance" + ); + assert_eq!( + provenance.source_sha256.len(), + 2, + "invalid Quint invalidation provenance" + ); + for source in [MODEL, GENERATOR] { + let actual = sha256_hex(&read(repo_root, source)); + assert_eq!( + provenance.source_sha256.get(source), + Some(&actual), + "stale Quint invalidation vectors for {source}; regenerate and review" + ); + } + for (index, vector) in generated.vectors.iter().enumerate() { + assert!( + vector.name.starts_with(&format!("Quint {index:03}: ")), + "incomplete or reordered Quint invalidation combinations at {index}: {}", + vector.name + ); + } + + let mut vectors = fixed.vectors; + vectors.extend(generated.vectors); + vectors +} + +/// Install `existing` atomically and return the server clock in epoch ms. +const SETUP: &str = r#"redis.replicate_commands() +local now=redis.call("TIME") +redis.call("DEL",KEYS[1]) +if ARGV[1]=="string" then redis.call("SET",KEYS[1],ARGV[2]) end +if ARGV[1]=="list" then for _,value in ipairs(cjson.decode(ARGV[2])) do redis.call("RPUSH",KEYS[1],value) end end +if tonumber(ARGV[3])>0 then redis.call("PEXPIRE",KEYS[1],ARGV[3]) end +return tonumber(now[1])*1000+math.floor(tonumber(now[2])/1000)"#; + +/// Observe type, content, retention and the server clock atomically. +const OBSERVE: &str = r#"local kind=redis.call("TYPE",KEYS[1]).ok +local content={} +if kind=="string" then content=redis.call("GET",KEYS[1]) end +if kind=="list" then content=redis.call("LRANGE",KEYS[1],0,-1) end +if kind=="none" then kind="absent" end +local ttl=redis.call("PTTL",KEYS[1]);local now=redis.call("TIME") +return {kind,content,ttl,tonumber(now[1])*1000+math.floor(tonumber(now[2])/1000)}"#; + +fn text(value: &Value) -> Result { + match value { + Value::BulkString(bytes) => { + String::from_utf8(bytes.clone()).map_err(|error| format!("non-UTF-8 reply: {error}")) + } + Value::SimpleString(text) => Ok(text.clone()), + other => Err(format!("expected a string reply, got {other:?}")), + } +} + +fn integer(value: &Value) -> Result { + match value { + Value::Int(integer) => Ok(*integer), + other => Err(format!("expected an integer reply, got {other:?}")), + } +} + +/// What the server holds after the transition. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Observed { + pub kind: String, + pub value: Option, + pub values: Option>, + pub ttl_ms: i64, + pub now_ms: i64, +} + +fn observed(reply: Value) -> Result { + let Value::Array(items) = reply else { + return Err(format!("observe returned {reply:?}")); + }; + let [kind, content, ttl, now]: [Value; 4] = items + .try_into() + .map_err(|items| format!("observe returned {items:?}"))?; + let kind = text(&kind)?; + let (value, values) = match kind.as_str() { + "string" => (Some(text(&content)?), None), + "list" => { + let Value::Array(entries) = content else { + return Err(format!("list content {content:?}")); + }; + let entries = entries.iter().map(text).collect::, _>>()?; + (None, Some(entries)) + } + _ => (None, None), + }; + Ok(Observed { + kind, + value, + values, + ttl_ms: integer(&ttl)?, + now_ms: integer(&now)?, + }) +} + +/// Compare an observed state with `want` under the PROTOCOL.md TTL rule. +pub fn compare(want: &State, got: &Observed, elapsed_ms: i64) -> Result<(), String> { + if elapsed_ms < 0 { + return Err("server time moved backwards".to_string()); + } + if got.kind != want.kind { + return Err(format!("kind {} want {}", got.kind, want.kind)); + } + match want.kind.as_str() { + "string" if got.value != want.value => { + return Err(format!("content {:?} want {:?}", got.value, want.value)); + } + "list" if got.values != want.values => { + return Err(format!("list {:?} want {:?}", got.values, want.values)); + } + _ => {} + } + if want.ttl_ms < 0 { + if got.ttl_ms != want.ttl_ms { + return Err(format!("TTL {} want {}", got.ttl_ms, want.ttl_ms)); + } + } else { + let minimum = (want.ttl_ms - elapsed_ms).max(0); + if got.ttl_ms < minimum || got.ttl_ms > want.ttl_ms { + return Err(format!( + "TTL {} outside [{minimum},{}], measured elapsed={elapsed_ms}", + got.ttl_ms, want.ttl_ms + )); + } + } + Ok(()) +} + +/// Replay one vector against `key` through `adapter`; fixture commands run on +/// the key's primary through `connection`. +pub async fn replay( + adapter: &RedisAdapter, + connection: &C, + key: &str, + vector: &Vector, +) -> Result<(), String> { + let content = match vector.existing.kind.as_str() { + "list" => serde_json::to_string(vector.existing.values.as_deref().unwrap_or_default()) + .map_err(|error| error.to_string())?, + _ => vector.existing.value.clone().unwrap_or_default(), + }; + let mut setup = redis::cmd("EVAL"); + setup + .arg(SETUP) + .arg(1) + .arg(key) + .arg(&vector.existing.kind) + .arg(content) + .arg(vector.existing.ttl_ms); + let start_ms = integer( + &connection + .run_on_primary(key, setup) + .await + .map_err(|error| format!("setup: {error}"))?, + )?; + + let result = adapter + .invalidate_decimal(key, &vector.future_buffer_ms, &vector.invalidated_at_ms) + .await; + if result.is_err() != vector.expected.error { + return Err(format!( + "error={:?} expected rejection={}", + result.err().map(|error| error.to_string()), + vector.expected.error + )); + } + + let mut observe = redis::cmd("EVAL"); + observe.arg(OBSERVE).arg(1).arg(key); + let got = observed( + connection + .run_on_primary(key, observe) + .await + .map_err(|error| format!("observe: {error}"))?, + )?; + compare(&vector.expected.state, &got, got.now_ms - start_ms) +} + +/// Replay every vector under `key_prefix`. Returns the number replayed, or +/// every failure as `name: reason`. +pub async fn replay_all( + adapter: &RedisAdapter, + connection: &C, + key_prefix: &str, + vectors: &[Vector], +) -> Result> { + let mut failures = Vec::new(); + for vector in vectors { + let key = format!("{key_prefix}{}", vector.name); + if let Err(reason) = replay(adapter, connection, &key, vector).await { + failures.push(format!("{}: {reason}", vector.name)); + } + } + if failures.is_empty() { + Ok(vectors.len()) + } else { + Err(failures) + } +} diff --git a/rust/tests/formal/inventory.rs b/rust/tests/formal/inventory.rs new file mode 100644 index 00000000..84f88ed7 --- /dev/null +++ b/rust/tests/formal/inventory.rs @@ -0,0 +1,476 @@ +//! Corpus discovery and the language-neutral case inventory. +//! +//! Ports the trace selection of the Go harness (`TestCoreConformance`, +//! `effectsPaths`, `featurePaths`, `featureRegressionPaths`), its registry +//! checks (`validateRegistry`, `validateBehaviorProfileRegistry`) and the id +//! scheme of `formal/conformance.mjs` `conformanceInventory`, so the Rust +//! replay reports the same `sampled/…`, `regression/…`, `scenario/…`, +//! `protocol/…` and `witness/…` ids the completion checker requires. + +use super::json::strict_parse; +use serde_json::Value; +use std::path::{Path, PathBuf}; + +/// Absolute repository root (the parent of the `rust/` crate directory). +pub fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("crate lives in /rust") + .to_path_buf() +} + +/// Resolves a repository-relative path such as `formal/profiles.json`. +pub fn repo_path(relative: &str) -> PathBuf { + repo_root().join(relative) +} + +/// Where one replay corpus comes from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TraceSource { + /// The committed smoke history under `formal/`. + Smoke, + /// One explicit history file. + File(PathBuf), + /// A generated corpus directory plus its scheduled regressions. + Directory(PathBuf), +} + +impl TraceSource { + fn from_env(file: &str, directory: &str, conflict: &str) -> Result { + let file = std::env::var(file).unwrap_or_default(); + let directory = std::env::var(directory).unwrap_or_default(); + match (file.is_empty(), directory.is_empty()) { + (false, false) => Err(conflict.to_string()), + (false, true) => Ok(TraceSource::File(PathBuf::from(file))), + (true, false) => Ok(TraceSource::Directory(PathBuf::from(directory))), + (true, true) => Ok(TraceSource::Smoke), + } + } + + /// Whether this source is a full generated corpus. + pub fn is_directory(&self) -> bool { + matches!(self, TraceSource::Directory(_)) + } +} + +/// Which protocol vector rows a run replays. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProtocolCorpus { + /// Fixed and generated rows (the default and `all`). + All, + /// Quint-generated rows only. + Generated, + /// Checked-in rows only. + Fixed, +} + +/// Which halves of the suite a run executes (`DIALCACHE_RUST_SUITE`). +/// +/// The mutation measurement runs the Quint-generated evidence and the fixed +/// supplement as separate cohorts; a normal run executes both. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Suite { + /// Generated histories, witness evidence and fixed scenarios (the default and `all`). + All, + /// Generated histories and witness evidence only: no fixed scenarios. + Generated, + /// Fixed scenarios only: no histories and no witness evidence. + Fixed, +} + +impl Suite { + /// Whether generated histories and witness evidence run. + pub fn runs_generated(self) -> bool { + self != Suite::Fixed + } + + /// Whether the fixed scenarios run. + pub fn runs_fixed(self) -> bool { + self != Suite::Generated + } +} + +/// The environment-driven selection of what one conformance run replays. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Selection { + /// `DIALCACHE_MBT_TRACE_FILE` / `DIALCACHE_MBT_TRACE_DIR`. + pub core: TraceSource, + /// `DIALCACHE_EFFECTS_TRACE_FILE` / `DIALCACHE_EFFECTS_TRACE_DIR`. + pub effects: TraceSource, + /// `DIALCACHE_FEATURE_TRACE_FILE` / `DIALCACHE_FEATURE_TRACE_DIR`. + pub features: TraceSource, + /// `DIALCACHE_FEATURE_PROFILE`: replay one feature profile only. + pub feature_profile: Option, + /// `DIALCACHE_WITNESS_EVIDENCE_DIR`: evaluated witness evidence files. + pub witness_evidence_dir: Option, + /// `DIALCACHE_PROTOCOL_CORPUS`. + pub protocol_corpus: ProtocolCorpus, + /// `DIALCACHE_RUST_SUITE`. + pub suite: Suite, + /// `DIALCACHE_RUST_REPORT`: JSONL assertion report path. + pub report: Option, + /// `DIALCACHE_BEHAVIOR_SCENARIO`: substring filter on scenario names. + pub behavior_scenario: Option, +} + +fn optional(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} + +impl Selection { + /// Reads the selection from the environment. + pub fn from_env() -> Result { + let protocol_corpus = match optional("DIALCACHE_PROTOCOL_CORPUS").as_deref() { + None | Some("all") => ProtocolCorpus::All, + Some("generated") => ProtocolCorpus::Generated, + Some("fixed") => ProtocolCorpus::Fixed, + Some(other) => return Err(format!("unknown protocol corpus selection {other}")), + }; + let suite = match optional("DIALCACHE_RUST_SUITE").as_deref() { + None | Some("all") => Suite::All, + Some("generated") => Suite::Generated, + Some("fixed") => Suite::Fixed, + Some(other) => return Err(format!("unknown suite selection {other}")), + }; + Ok(Selection { + core: TraceSource::from_env( + "DIALCACHE_MBT_TRACE_FILE", + "DIALCACHE_MBT_TRACE_DIR", + "select either a trace file or directory", + )?, + effects: TraceSource::from_env( + "DIALCACHE_EFFECTS_TRACE_FILE", + "DIALCACHE_EFFECTS_TRACE_DIR", + "select either an effects trace file or directory", + )?, + features: TraceSource::from_env( + "DIALCACHE_FEATURE_TRACE_FILE", + "DIALCACHE_FEATURE_TRACE_DIR", + "select either a feature trace file or directory", + )?, + feature_profile: optional("DIALCACHE_FEATURE_PROFILE"), + witness_evidence_dir: optional("DIALCACHE_WITNESS_EVIDENCE_DIR").map(PathBuf::from), + protocol_corpus, + suite, + report: optional("DIALCACHE_RUST_REPORT").map(PathBuf::from), + behavior_scenario: optional("DIALCACHE_BEHAVIOR_SCENARIO"), + }) + } + + /// Core histories: the smoke history, one file, or a directory's `*.itf.json` + /// plus the scheduled core regressions. + pub fn core_paths(&self) -> Result, String> { + match &self.core { + TraceSource::Smoke => Ok(vec![repo_path("formal/conformance-smoke.itf.json")]), + TraceSource::File(file) => Ok(vec![file.clone()]), + TraceSource::Directory(directory) => { + let mut paths = glob_itf(directory)?; + if paths.is_empty() { + return Err("empty trace corpus".to_string()); + } + paths.extend(regression_paths("core", directory)?); + Ok(paths) + } + } + } + + /// Effects histories, including scheduled regressions for a directory corpus. + pub fn effects_paths(&self) -> Result, String> { + let paths = match &self.effects { + TraceSource::Smoke => vec![repo_path("formal/effects-smoke.itf.json")], + TraceSource::File(file) => vec![file.clone()], + TraceSource::Directory(directory) => { + let mut paths = glob_itf(directory)?; + paths.extend(regression_paths("effects", directory)?); + paths + } + }; + if paths.is_empty() { + return Err("empty effects corpus".to_string()); + } + Ok(paths) + } + + /// Histories of one feature profile (including `local-clock`). An explicit + /// file is used only when it belongs to the profile; a directory corpus + /// reads `

//*.itf.json` plus the scheduled regressions. + pub fn feature_paths(&self, profile: &str) -> Result, String> { + match &self.features { + TraceSource::File(file) => { + let parent = file + .parent() + .and_then(Path::file_name) + .map(|name| name.to_string_lossy().into_owned()); + let name = file_name(file); + if parent.as_deref() == Some(profile) || name.contains(&format!("{profile}-smoke")) + { + Ok(vec![file.clone()]) + } else { + Ok(Vec::new()) + } + } + TraceSource::Directory(directory) => { + let mut paths = glob_itf(&directory.join(profile))?; + if paths.is_empty() { + return Err(format!("empty feature corpus {profile}")); + } + paths.extend(regression_paths(profile, directory)?); + Ok(paths) + } + TraceSource::Smoke => Ok(vec![repo_path(&format!("formal/{profile}-smoke.itf.json"))]), + } + } +} + +fn file_name(path: &Path) -> String { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Sorted `*.itf.json` files directly inside `directory`. +pub fn glob_itf(directory: &Path) -> Result, String> { + let entries = match std::fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(format!("{}: {error}", directory.display())), + }; + let mut names = Vec::new(); + for entry in entries { + let entry = entry.map_err(|error| format!("{}: {error}", directory.display()))?; + let name = entry.file_name().to_string_lossy().into_owned(); + if name.ends_with(".itf.json") { + names.push(name); + } + } + names.sort(); + Ok(names.into_iter().map(|name| directory.join(name)).collect()) +} + +/// Lexically resolves `/..` the way Go's `filepath.Join` does. +fn parent_of(directory: &Path) -> PathBuf { + let mut parent = directory.to_path_buf(); + match directory.components().next_back() { + Some(std::path::Component::Normal(_)) => { + parent.pop(); + } + _ => parent.push(".."), + } + parent +} + +/// Exported Quint regressions of `profile`, read from +/// `/../regressions//`. The shared inventory derives which +/// runs export from the Quint source and the full-report gate requires exactly +/// those histories; the native driver only discovers their generated files. +pub fn regression_paths(profile: &str, directory: &Path) -> Result, String> { + let regressions = parent_of(directory).join("regressions").join(profile); + let paths = glob_itf(®ressions)?; + if paths.is_empty() { + return Err(format!("missing Quint regressions for {profile}")); + } + Ok(paths) +} + +fn read_json(relative: &str) -> Result { + let path = repo_path(relative); + let text = + std::fs::read_to_string(&path).map_err(|error| format!("{}: {error}", path.display()))?; + strict_parse(&text).map_err(|error| format!("{relative}: {error}")) +} + +/// A history's kind follows the corpus layout the shared evaluator classifies +/// by: exported regressions live under `regressions//`, every other +/// replayed history is a sampled one. +pub fn trace_kind(path: &Path) -> &'static str { + let grandparent = path + .parent() + .and_then(Path::parent) + .and_then(Path::file_name); + if grandparent.is_some_and(|name| name == "regressions") { + "regression" + } else { + "sampled" + } +} + +/// Inventory id of one replayed history: `regression//` for an +/// exported regression, `sampled//` for `trace_.itf.json` of a +/// full generated corpus, otherwise `smoke//`. +pub fn trace_case_id(profile: &str, path: &Path, full: bool) -> String { + let name = file_name(path); + let stem = name.strip_suffix(".itf.json").unwrap_or(&name); + if trace_kind(path) == "regression" { + return format!("regression/{profile}/{stem}"); + } + if full { + if let Some(index) = stem.strip_prefix("trace_") { + if !index.is_empty() + && index.bytes().all(|byte| byte.is_ascii_digit()) + && (index == "0" || !index.starts_with('0')) + { + return format!("sampled/{profile}/{index}"); + } + } + } + format!("smoke/{profile}/{name}") +} + +/// `scenario//` with percent-encoded components. +pub fn scenario_case_id(feature: &str, name: &str) -> String { + format!( + "scenario/{}/{}", + percent_encode_component(feature), + percent_encode_component(name) + ) +} + +/// `protocol//` with a percent-encoded vector name. +pub fn protocol_case_id(group: &str, name: &str) -> String { + format!("protocol/{group}/{}", percent_encode_component(name)) +} + +/// `witness/`. +pub fn witness_case_id(profile: &str) -> String { + format!("witness/{profile}") +} + +/// Exactly JavaScript's `encodeURIComponent`: every byte of the UTF-8 encoding +/// outside `A-Z a-z 0-9 - _ . ! ~ * ' ( )` becomes `%XX` with uppercase hex. +pub fn percent_encode_component(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || b"-_.!~*'()".contains(&byte) { + out.push(byte as char); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +/// Behavior profile versions this port implements, from +/// `go/behavior_registry_test.go`. +pub const BEHAVIOR_PROFILE_VERSIONS: [(&str, i64); 16] = [ + ("recovery-read", 1), + ("local-failure", 1), + ("runtime-boundaries", 1), + ("shadow-layers", 1), + ("local-clock", 1), + ("source-budgets", 1), + ("effects", 3), + ("scope", 2), + ("policy", 3), + ("layers", 2), + ("recovery", 1), + ("independent", 2), + ("shadow", 3), + ("admission", 1), + ("dark-layers", 2), + ("shadow-read-deadlines", 1), +]; + +/// The version this port implements of a behavior profile, if it knows it. +pub fn behavior_profile_version(profile: &str) -> Option { + BEHAVIOR_PROFILE_VERSIONS + .iter() + .find(|(name, _)| *name == profile) + .map(|(_, version)| *version) +} + +/// Ports `validateRegistry`: `formal/profiles.json` must declare schema 1, +/// specification 0.1.0, protocol schema 3 and exactly one core profile at version 1. +pub fn registry_check() -> Result<(), String> { + registry_check_text(&read_registry_text()?) +} + +fn read_registry_text() -> Result { + let path = repo_path("formal/profiles.json"); + std::fs::read_to_string(&path).map_err(|error| format!("{}: {error}", path.display())) +} + +/// [`registry_check`] over explicit registry text. +pub fn registry_check_text(raw: &str) -> Result<(), String> { + let registry = strict_parse(raw)?; + if registry.get("schemaVersion").and_then(Value::as_f64) != Some(1.0) + || registry.get("specificationVersion").and_then(Value::as_str) != Some("0.1.0") + || registry + .get("protocolSchemaVersion") + .and_then(Value::as_f64) + != Some(3.0) + { + return Err("unsupported specification/profile/protocol registry version".to_string()); + } + let mut core = 0; + for profile in registry + .get("profiles") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + if profile.get("id").and_then(Value::as_str) == Some("core") { + core += 1; + if profile.get("version").and_then(Value::as_f64) != Some(1.0) { + return Err("unsupported core profile version".to_string()); + } + } + } + if core != 1 { + return Err("registry requires exactly one core profile".to_string()); + } + Ok(()) +} + +/// Ports `validateBehaviorProfileRegistry` against `formal/profiles.json`. +pub fn profile_registry_check(profile: &str, version: i64) -> Result<(), String> { + profile_registry_check_text(&read_registry_text()?, profile, version) +} + +/// [`profile_registry_check`] over explicit registry text. +pub fn profile_registry_check_text(raw: &str, name: &str, version: i64) -> Result<(), String> { + let registry = strict_parse(raw)?; + if registry.get("schemaVersion").and_then(Value::as_f64) != Some(1.0) + || registry.get("specificationVersion").and_then(Value::as_str) != Some("0.1.0") + || registry + .get("behavioralSchemaVersion") + .and_then(Value::as_f64) + != Some(2.0) + || registry + .get("protocolSchemaVersion") + .and_then(Value::as_f64) + != Some(3.0) + { + return Err("unsupported specification/behavioral registry".to_string()); + } + let mut count = 0; + for profile in registry + .get("profiles") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + if profile.get("id").and_then(Value::as_str) != Some(name) { + continue; + } + count += 1; + if profile.get("version").and_then(Value::as_f64) != Some(version as f64) + || profile.get("model").and_then(Value::as_str) + != Some(format!("formal/dialcache-{name}-conformance.qnt").as_str()) + || profile.get("smoke").and_then(Value::as_str) + != Some(format!("formal/{name}-smoke.itf.json").as_str()) + { + return Err(format!("unsupported {name} profile definition/version")); + } + } + if count != 1 { + return Err(format!("registry needs exactly one {name} profile")); + } + Ok(()) +} + +/// Checks the registry entry of a behavior profile at the version this port implements. +pub fn require_behavior_profile(profile: &str) -> Result<(), String> { + let version = behavior_profile_version(profile) + .ok_or_else(|| format!("unknown behavior profile {profile}"))?; + profile_registry_check(profile, version) +} diff --git a/rust/tests/formal/json.rs b/rust/tests/formal/json.rs new file mode 100644 index 00000000..80228483 --- /dev/null +++ b/rust/tests/formal/json.rs @@ -0,0 +1,223 @@ +//! Strict JSON handling for the replay harness. +//! +//! Ports the Go harness helpers `validateBehaviorJSON`, `behaviorITF`, `bequal` +//! and `behaviorKeys`. Generated histories, coordinator replies and evidence +//! files are trust boundaries: an ambiguous document (duplicate member names, +//! non-finite or unsafe integers, trailing content) is rejected instead of being +//! decoded with last-key-wins or silent rounding. + +use serde_json::{Map, Value}; +use std::collections::HashSet; + +/// Largest integer JavaScript represents exactly; ITF integers outside +/// `[-MAX_SAFE_INTEGER, MAX_SAFE_INTEGER]` are rejected. +pub const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991; + +/// Parses `text` as JSON, rejecting duplicate object member names (compared +/// after unescaping), non-finite numbers and trailing content. +pub fn strict_parse(text: &str) -> Result { + let value: Value = + serde_json::from_str(text).map_err(|error| format!("invalid JSON: {error}"))?; + check_duplicate_keys(text.as_bytes())?; + check_finite(&value)?; + Ok(value) +} + +/// Decodes every `{"#bigint":"..."}` object into a JSON number after checking +/// it is a safe integer, and rejects NaN, infinities and unsafe plain numbers. +pub fn decode_itf(value: Value) -> Result { + match value { + Value::Object(map) => { + if let Some(raw) = map.get("#bigint") { + if map.len() != 1 { + return Err("malformed ITF integer".to_string()); + } + let parsed = raw.as_str().and_then(|text| text.parse::().ok()); + return match parsed { + Some(n) if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&n) => { + Ok(Value::from(n)) + } + _ => Err("unsafe ITF integer".to_string()), + }; + } + let mut out = Map::new(); + for (key, item) in map { + out.insert(key, decode_itf(item)?); + } + Ok(Value::Object(out)) + } + Value::Array(items) => items + .into_iter() + .map(decode_itf) + .collect::, _>>() + .map(Value::Array), + Value::Number(ref number) => match number.as_f64() { + Some(n) if n.is_finite() && n.abs() <= MAX_SAFE_INTEGER as f64 => Ok(value), + _ => Err("unsafe JSON number".to_string()), + }, + other => Ok(other), + } +} + +/// Compares two JSON values, treating numbers numerically (`1 == 1.0`) and +/// everything else structurally. `null`, `false`, `0`, `""` and `[]` are all +/// distinct from each other. +pub fn json_equal(a: &Value, b: &Value) -> bool { + match (a, b) { + (Value::Null, Value::Null) => true, + (Value::Bool(x), Value::Bool(y)) => x == y, + (Value::Number(x), Value::Number(y)) => match (x.as_f64(), y.as_f64()) { + (Some(x), Some(y)) => x == y, + _ => false, + }, + (Value::String(x), Value::String(y)) => x == y, + (Value::Array(x), Value::Array(y)) => { + x.len() == y.len() && x.iter().zip(y).all(|(a, b)| json_equal(a, b)) + } + (Value::Object(x), Value::Object(y)) => { + x.len() == y.len() + && x.iter() + .all(|(key, value)| y.get(key).is_some_and(|other| json_equal(value, other))) + } + _ => false, + } +} + +/// Joins an object's member names in sorted order with commas, like the Go +/// harness's `behaviorKeys`; used for exact envelope and reply shape checks. +pub fn sorted_keys(object: &Map) -> String { + let mut keys: Vec<&str> = object.keys().map(String::as_str).collect(); + keys.sort_unstable(); + keys.join(",") +} + +/// Sorted keys of `value` when it is an object, otherwise the empty string. +pub fn keys_of(value: &Value) -> String { + value.as_object().map(sorted_keys).unwrap_or_default() +} + +fn check_finite(value: &Value) -> Result<(), String> { + match value { + Value::Number(number) => match number.as_f64() { + Some(n) if n.is_finite() => Ok(()), + _ => Err("non-finite JSON number".to_string()), + }, + Value::Array(items) => items.iter().try_for_each(check_finite), + Value::Object(map) => map.values().try_for_each(check_finite), + _ => Ok(()), + } +} + +/// Second walk over syntactically valid JSON that checks duplicate object member +/// names without allocating every scalar. Escaped names are decoded before +/// comparison so `"a"` and its `\u0061` escape spell the same member name. +fn check_duplicate_keys(raw: &[u8]) -> Result<(), String> { + let mut walker = Walker { raw, at: 0 }; + walker.walk() +} + +struct Walker<'a> { + raw: &'a [u8], + at: usize, +} + +impl Walker<'_> { + fn peek(&self) -> Result { + self.raw + .get(self.at) + .copied() + .ok_or_else(|| "truncated JSON".to_string()) + } + + fn space(&mut self) { + while self.at < self.raw.len() && matches!(self.raw[self.at], b' ' | b'\n' | b'\r' | b'\t') + { + self.at += 1; + } + } + + fn quoted(&mut self) -> Result { + let start = self.at; + self.at += 1; + let mut escaped = false; + loop { + match self.peek()? { + b'"' => break, + b'\\' => { + escaped = true; + self.at += 2; + } + _ => self.at += 1, + } + } + self.at += 1; + let slice = &self.raw[start..self.at]; + if !escaped { + return std::str::from_utf8(&slice[1..slice.len() - 1]) + .map(str::to_owned) + .map_err(|error| error.to_string()); + } + let text = std::str::from_utf8(slice).map_err(|error| error.to_string())?; + serde_json::from_str::(text).map_err(|error| error.to_string()) + } + + fn walk(&mut self) -> Result<(), String> { + self.space(); + match self.peek()? { + b'{' => { + self.at += 1; + self.space(); + if self.peek()? == b'}' { + self.at += 1; + return Ok(()); + } + let mut seen: HashSet = HashSet::new(); + loop { + self.space(); + let key = self.quoted()?; + if !seen.insert(key.clone()) { + return Err(format!("duplicate JSON key {key:?}")); + } + self.space(); + self.at += 1; // ':' + self.walk()?; + self.space(); + if self.peek()? == b'}' { + self.at += 1; + return Ok(()); + } + self.at += 1; // ',' + } + } + b'[' => { + self.at += 1; + self.space(); + if self.peek()? == b']' { + self.at += 1; + return Ok(()); + } + loop { + self.walk()?; + self.space(); + if self.peek()? == b']' { + self.at += 1; + return Ok(()); + } + self.at += 1; // ',' + } + } + b'"' => self.quoted().map(|_| ()), + _ => { + while self.at < self.raw.len() + && !matches!( + self.raw[self.at], + b',' | b'}' | b']' | b' ' | b'\n' | b'\r' | b'\t' + ) + { + self.at += 1; + } + Ok(()) + } + } + } +} diff --git a/rust/tests/formal/key_vectors.rs b/rust/tests/formal/key_vectors.rs new file mode 100644 index 00000000..7575b373 --- /dev/null +++ b/rust/tests/formal/key_vectors.rs @@ -0,0 +1,239 @@ +//! Replay of the key, invalid-key, argument-normalization and rollout vector +//! groups against `dialcache::identity`. +//! +//! Include from a test binary alongside `fixtures.rs`: +//! `#[path = "formal/fixtures.rs"] mod fixtures;` +//! `#[path = "formal/key_vectors.rs"] mod key_vectors;` + +#![allow(dead_code)] + +use std::collections::BTreeMap; + +use dialcache::identity::{cohort, cohort_hash, normalize_args, ArgValue, Identity}; +use serde_json::Value; + +use super::fixtures::{assertion_mismatch, LONE_SURROGATE_MARKER}; + +fn identity_from(value: &Value) -> Result { + serde_json::from_value(value.clone()) + .map_err(|error| format!("identity input does not parse: {error}")) +} + +fn components(identity: &Identity) -> impl Iterator { + [ + identity.namespace.as_str(), + identity.key_type.as_str(), + identity.id.as_str(), + identity.use_case.as_str(), + ] + .into_iter() + .chain( + identity + .args + .iter() + .flat_map(|(name, value)| [name.as_str(), value.as_str()]), + ) +} + +fn expected_str(value: &Value, field: &str) -> Result, String> { + match value.get(field) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(text)) => Ok(Some(text.clone())), + Some(other) => Err(format!("{field} is neither a string nor null: {other}")), + } +} + +/// `keyVectors`: `keys()` must succeed and equal `logicalKey`, `valueKey` and `watermarkKey`. +pub fn check_key_vector(vector: &Value) -> Result<(), String> { + let identity = identity_from(&vector["input"])?; + let logical = expected_str(vector, "logicalKey")?.ok_or("logicalKey missing")?; + let value = expected_str(vector, "valueKey")?.ok_or("valueKey missing")?; + let watermark = expected_str(vector, "watermarkKey")?; + let keys = identity + .keys() + .map_err(|error| assertion_mismatch((&logical, &value, &watermark), error))?; + if keys.logical != logical { + return Err(assertion_mismatch(logical, keys.logical)); + } + if keys.value != value { + return Err(assertion_mismatch(value, keys.value)); + } + if keys.watermark != watermark { + return Err(assertion_mismatch(watermark, keys.watermark)); + } + Ok(()) +} + +/// Decode one `inputUtf16` field: `Ok(None)` when the units are not valid UTF-16. +fn utf16_field(value: &Value, what: &str) -> Result, String> { + let units = value + .as_array() + .ok_or_else(|| format!("inputUtf16 {what} is not an array"))?; + let units: Vec = units + .iter() + .map(|unit| { + unit.as_u64() + .and_then(|unit| u16::try_from(unit).ok()) + .ok_or_else(|| format!("inputUtf16 {what} has an invalid unit {unit}")) + }) + .collect::>()?; + Ok(String::from_utf16(&units).ok()) +} + +/// Rebuild the identity from `inputUtf16` code units, keeping the tracking +/// flag of the parsed input; `Ok(None)` when any field is not valid UTF-16. +fn identity_from_utf16(raw: &Value, tracked: bool) -> Result, String> { + let Some(namespace) = utf16_field(&raw["namespace"], "namespace")? else { + return Ok(None); + }; + let Some(key_type) = utf16_field(&raw["keyType"], "keyType")? else { + return Ok(None); + }; + let Some(id) = utf16_field(&raw["id"], "id")? else { + return Ok(None); + }; + let Some(use_case) = utf16_field(&raw["useCase"], "useCase")? else { + return Ok(None); + }; + let mut args = Vec::new(); + for pair in raw["args"] + .as_array() + .ok_or("inputUtf16 args is not an array")? + { + let pair = pair + .as_array() + .filter(|pair| pair.len() == 2) + .ok_or("inputUtf16 args entry is not a pair")?; + let Some(name) = utf16_field(&pair[0], "argument name")? else { + return Ok(None); + }; + let Some(value) = utf16_field(&pair[1], "argument value")? else { + return Ok(None); + }; + args.push((name, value)); + } + Ok(Some(Identity { + namespace, + key_type, + id, + use_case, + tracked, + args, + })) +} + +/// `invalidKeyVectors`: the identity must be rejected. A component carrying +/// [`LONE_SURROGATE_MARKER`] was already rejected while decoding the fixture, +/// which the protocol allows for scalar-only string types; every other input +/// must make `keys()` fail. When `inputUtf16` is present the identity is also +/// rebuilt from code units and must fail either UTF-16 decoding or `keys()`. +pub fn check_invalid_key_vector(vector: &Value) -> Result<(), String> { + let identity = identity_from(&vector["input"])?; + let unrepresentable = + components(&identity).any(|component| component.contains(LONE_SURROGATE_MARKER)); + if !unrepresentable && identity.keys().is_ok() { + return Err(assertion_mismatch("invalid key rejected", "accepted")); + } + if let Some(raw) = vector.get("inputUtf16") { + if let Some(rebuilt) = identity_from_utf16(raw, identity.tracked)? { + if rebuilt.keys().is_ok() { + return Err(assertion_mismatch( + "invalid UTF-16 key rejected", + "accepted", + )); + } + } + } + Ok(()) +} + +fn arg_from_json(value: &Value) -> Result { + Ok(match value { + Value::Null => ArgValue::Null, + Value::Bool(flag) => ArgValue::Bool(*flag), + Value::Number(number) => match number.as_i64() { + Some(integer) => ArgValue::Int(integer), + None => ArgValue::Number( + number + .as_f64() + .ok_or_else(|| format!("number {number} is not representable"))?, + ), + }, + Value::String(text) => ArgValue::Str(text.clone()), + other => return Err(format!("unsupported normalizeArgs input {other}")), + }) +} + +/// `normalizeArgsVectors`: build the host record (sentinel strings become +/// `Absent`, `bigintArgs` become `BigInt`, `specialArgs` become `Number`) and +/// require exactly the expected ordered pairs. +pub fn check_normalize_args_vector(vector: &Value) -> Result<(), String> { + let sentinel = vector.get("undefinedSentinel").and_then(Value::as_str); + let mut record: BTreeMap = BTreeMap::new(); + for (name, value) in vector["input"] + .as_object() + .ok_or("input is not an object")? + { + let arg = match (value.as_str(), sentinel) { + (Some(text), Some(sentinel)) if text == sentinel => ArgValue::Absent, + _ => arg_from_json(value)?, + }; + record.insert(name.clone(), arg); + } + if let Some(bigints) = vector.get("bigintArgs") { + for (name, text) in bigints.as_object().ok_or("bigintArgs is not an object")? { + let text = text + .as_str() + .ok_or_else(|| format!("bigintArgs {name} is not a string"))?; + record.insert(name.clone(), ArgValue::BigInt(text.to_owned())); + } + } + if let Some(specials) = vector.get("specialArgs") { + for (name, text) in specials.as_object().ok_or("specialArgs is not an object")? { + let text = text + .as_str() + .ok_or_else(|| format!("specialArgs {name} is not a string"))?; + let number: f64 = text + .parse() + .map_err(|error| format!("specialArgs {name} {text:?} is not a number: {error}"))?; + record.insert(name.clone(), ArgValue::Number(number)); + } + } + let expected: Vec<(String, String)> = serde_json::from_value(vector["expected"].clone()) + .map_err(|error| format!("expected pairs do not parse: {error}"))?; + let actual = normalize_args(record).map_err(|error| assertion_mismatch(&expected, error))?; + if actual != expected { + return Err(assertion_mismatch(expected, actual)); + } + Ok(()) +} + +/// `rampVectors`: the cohort of the logical key and `layer` must equal `sample` +/// exactly, and the FNV-1a numerator must equal `hashNumerator` when present. +pub fn check_ramp_vector(vector: &Value) -> Result<(), String> { + let identity = identity_from(&vector["input"])?; + let layer = vector["layer"].as_str().ok_or("layer missing")?; + let sample = vector["sample"].as_f64().ok_or("sample missing")?; + let numerator = vector + .get("hashNumerator") + .map(|value| { + value + .as_u64() + .ok_or("hashNumerator is not an unsigned integer") + }) + .transpose()?; + let keys = identity + .keys() + .map_err(|error| assertion_mismatch("valid rollout identity", error))?; + let actual = cohort(&keys.logical, layer); + if actual != sample { + return Err(assertion_mismatch(sample, actual)); + } + if let Some(numerator) = numerator { + let hash = u64::from(cohort_hash(&keys.logical, layer)); + if hash != numerator { + return Err(assertion_mismatch(numerator, hash)); + } + } + Ok(()) +} diff --git a/rust/tests/formal/local_clock.rs b/rust/tests/formal/local_clock.rs new file mode 100644 index 00000000..97d6b6ba --- /dev/null +++ b/rust/tests/formal/local_clock.rs @@ -0,0 +1,123 @@ +//! The local-clock profile driver: default instances whose production clock +//! alignment runs over a shared virtual grid, fractional environment ticks, +//! immediate healthy sources. + +use std::sync::Arc; + +use dialcache::testing::{grid_clock, TestExecutor}; +use dialcache::{DialCache, Identity, Operation, Policy}; +use parking_lot::Mutex; +use serde_json::{json, Map, Value}; + +use super::driver::WALL_EPOCH_MS; + +pub struct LocalClockDriver { + pub exec: TestExecutor, + caches: [Option; 2], + sources: Arc>, + calls: Vec, +} + +impl std::fmt::Debug for LocalClockDriver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LocalClockDriver").finish() + } +} + +impl LocalClockDriver { + pub fn new() -> LocalClockDriver { + LocalClockDriver { + exec: TestExecutor::new(WALL_EPOCH_MS), + caches: [None, None], + sources: Arc::new(Mutex::new(0)), + calls: Vec::new(), + } + } + + pub fn apply(&mut self, input: &Value) -> Result<(), String> { + let instance = input.get("instance").and_then(Value::as_i64).unwrap_or(0); + match input.get("op").and_then(Value::as_str).unwrap_or("") { + "constructInstance" => { + if !(0..2).contains(&instance) || self.caches[instance as usize].is_some() { + return Err("invalid/duplicate instance".to_string()); + } + let cache = DialCache::builder() + .clock(grid_clock(&self.exec.clock)) + .runtime_arc(self.exec.runtime.clone()) + .build() + .map_err(|e| e.to_string())?; + self.caches[instance as usize] = Some(cache); + Ok(()) + } + "advanceTicks" => { + let ticks = input.get("ticks").and_then(Value::as_i64).unwrap_or(0); + if ticks <= 0 { + return Err("invalid clock advance".to_string()); + } + self.exec.advance_micros(ticks); + Ok(()) + } + "call" => { + if !(0..2).contains(&instance) { + return Err("call before instance construction".to_string()); + } + let Some(cache) = self.caches[instance as usize].clone() else { + return Err("call before instance construction".to_string()); + }; + let offered = input.get("offered").and_then(Value::as_i64).unwrap_or(0); + let sources = self.sources.clone(); + let operation = + Operation::::new(Identity::new("clock", "one", "QuintLocalGrid")) + .policy(Policy::default().local_ttl_sec(1)); + let value = self.exec.block_on(async move { + let inner = cache.clone(); + cache + .enable(|scope| async move { + inner + .get_or_load(&scope, operation, move |_| { + let sources = sources.clone(); + async move { + *sources.lock() += 1; + Ok(offered) + } + }) + .await + }) + .await + }); + let value = value.map_err(|e| e.to_string())?; + self.calls.push(*value); + Ok(()) + } + other => Err(format!("unknown local-clock command {other}")), + } + } + + pub fn observation(&self) -> Value { + let mut o = Map::new(); + o.insert("calls".to_string(), json!(self.calls)); + o.insert("loaders".to_string(), json!(*self.sources.lock())); + for key in [ + "reads", + "writes", + "invalidations", + "loads", + "dumps", + "policyCalls", + "classifications", + "comparisons", + ] { + o.insert(key.to_string(), json!(0)); + } + for key in [ + "maintenance", + "sourceScopes", + "writeTtls", + "shadow", + "recovery", + ] { + o.insert(key.to_string(), json!([])); + } + Value::Object(o) + } +} diff --git a/rust/tests/formal/mod.rs b/rust/tests/formal/mod.rs new file mode 100644 index 00000000..d7c2111b --- /dev/null +++ b/rust/tests/formal/mod.rs @@ -0,0 +1,22 @@ +//! Language-neutral replay infrastructure shared by the Rust conformance +//! binaries: strict JSON, the protocol schema interpreter, the coordinator +//! transport, corpus/inventory discovery, the JSONL report and the witness +//! evidence check. Cache drivers plug into [`transport::Coordinator::execute`]. +//! +//! Each `tests/*.rs` binary includes this directory as a module and uses a +//! different subset of it, so unused-item lints are silenced here rather than +//! per binary. +#![allow(dead_code)] + +pub mod causal; +pub mod core_driver; +pub mod driver; +pub mod gate; +pub mod inventory; +pub mod json; +pub mod local_clock; +pub mod report; +pub mod scenarios; +pub mod schema; +pub mod transport; +pub mod witness; diff --git a/rust/tests/formal/report.rs b/rust/tests/formal/report.rs new file mode 100644 index 00000000..8da4bf7f --- /dev/null +++ b/rust/tests/formal/report.rs @@ -0,0 +1,168 @@ +//! JSONL assertion report of one Rust conformance run. +//! +//! The Node completion adapter reads this file to credit each inventory id, so +//! every record is one JSON object per line: a `start` header, one `case` per +//! required id, and a `finish` footer with the totals. The path comes from +//! `DIALCACHE_RUST_REPORT`; without it the writer is a no-op that still counts. + +use serde_json::{json, Map, Value}; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Current wall time in epoch milliseconds. +pub fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis() as i64) + .unwrap_or(0) +} + +/// Totals of one run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Summary { + /// Cases reported. + pub cases: u64, + /// Cases that failed. + pub failed: u64, +} + +impl Summary { + /// `passed` when nothing failed, otherwise `failed`. + pub fn status(&self) -> &'static str { + if self.failed == 0 { + "passed" + } else { + "failed" + } + } +} + +/// JSONL report writer. +pub struct Report { + output: Option>, + summary: Summary, + failures: Vec<(String, String)>, +} + +impl std::fmt::Debug for Report { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Report") + .field("enabled", &self.output.is_some()) + .field("summary", &self.summary) + .finish() + } +} + +impl Report { + /// Creates the report at `DIALCACHE_RUST_REPORT`, or a counting no-op when unset. + pub fn from_env() -> Result { + let path = std::env::var("DIALCACHE_RUST_REPORT") + .ok() + .filter(|value| !value.is_empty()); + Report::create(path.as_deref().map(Path::new)) + } + + /// Creates (truncates) the report file, or a counting no-op for `None`. + pub fn create(path: Option<&Path>) -> Result { + let output = match path { + Some(path) => Some(BufWriter::new( + File::create(path).map_err(|error| format!("{}: {error}", path.display()))?, + )), + None => None, + }; + Ok(Report { + output, + summary: Summary::default(), + failures: Vec::new(), + }) + } + + /// Whether records are written anywhere. + pub fn enabled(&self) -> bool { + self.output.is_some() + } + + fn write(&mut self, record: Value) -> Result<(), String> { + if let Some(output) = self.output.as_mut() { + serde_json::to_writer(&mut *output, &record).map_err(|error| error.to_string())?; + output.write_all(b"\n").map_err(|error| error.to_string())?; + } + Ok(()) + } + + /// Writes the `start` header. + pub fn start(&mut self) -> Result<(), String> { + self.write(json!({"schemaVersion": 1, "kind": "start", "implementation": "rust", "startedAt": now_ms()})) + } + + /// Records one case with its outcome and timestamps. + pub fn case( + &mut self, + id: &str, + result: &Result<(), String>, + started_ms: i64, + finished_ms: i64, + ) -> Result<(), String> { + self.summary.cases += 1; + let mut record = Map::new(); + record.insert("kind".to_string(), Value::from("case")); + record.insert("id".to_string(), Value::from(id)); + record.insert( + "status".to_string(), + Value::from(if result.is_ok() { "passed" } else { "failed" }), + ); + record.insert("startedAt".to_string(), Value::from(started_ms)); + record.insert("finishedAt".to_string(), Value::from(finished_ms)); + if let Err(message) = result { + self.summary.failed += 1; + self.failures.push((id.to_string(), message.clone())); + record.insert("message".to_string(), Value::from(message.as_str())); + } + self.write(Value::Object(record)) + } + + /// Runs `body` as one timed case and records it. + pub fn run_case( + &mut self, + id: &str, + body: impl FnOnce() -> Result<(), String>, + ) -> Result<(), String> { + let started = now_ms(); + let result = body(); + let finished = now_ms(); + self.case(id, &result, started, finished)?; + result + } + + /// Totals so far. + pub fn summary(&self) -> Summary { + self.summary + } + + /// Writes the `finish` footer, flushes, and prints a human summary. + pub fn finish(&mut self) -> Result { + let summary = self.summary; + self.write(json!({ + "kind": "finish", + "status": summary.status(), + "finishedAt": now_ms(), + "cases": summary.cases, + "failed": summary.failed, + }))?; + if let Some(output) = self.output.as_mut() { + output.flush().map_err(|error| error.to_string())?; + } + for (id, message) in &self.failures { + println!("FAILED {id}: {message}"); + } + println!( + "rust conformance: {} cases, {} failed, status {}", + summary.cases, + summary.failed, + summary.status() + ); + Ok(summary) + } +} diff --git a/rust/tests/formal/scenarios.rs b/rust/tests/formal/scenarios.rs new file mode 100644 index 00000000..01f01624 --- /dev/null +++ b/rust/tests/formal/scenarios.rs @@ -0,0 +1,88 @@ +//! Fixed behavioral scenarios: handwritten command/expectation sequences +//! replayed through the behavior driver. + +use serde_json::{Map, Value}; + +use super::driver::{empty_observation, Driver}; +use super::inventory::repo_path; +use super::json::{json_equal, strict_parse}; + +/// Load and validate `formal/behavioral-scenarios.json`. +pub fn load_scenarios() -> Result, String> { + let text = std::fs::read_to_string(repo_path("formal/behavioral-scenarios.json")) + .map_err(|e| e.to_string())?; + let corpus = strict_parse(&text)?; + if corpus.get("schemaVersion").and_then(Value::as_i64) != Some(2) { + return Err("unsupported behavioral schema".to_string()); + } + let scenarios = corpus + .get("scenarios") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + if scenarios.is_empty() { + return Err("empty behavioral corpus".to_string()); + } + let mut seen = std::collections::HashSet::new(); + let shape = empty_observation(&serde_json::json!({ "observe": [] })); + for scenario in &scenarios { + let name = scenario.get("name").and_then(Value::as_str).unwrap_or(""); + let steps = scenario + .get("steps") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + if name.is_empty() || !seen.insert(name.to_string()) || steps.is_empty() { + return Err(format!("invalid/duplicate/empty scenario {name:?}")); + } + for step in &steps { + if !step.get("input").is_some_and(Value::is_object) { + return Err(format!("missing input in {name}")); + } + let Some(patch) = step.get("expect").and_then(Value::as_object) else { + return Err(format!("missing expected patch in {name}")); + }; + for key in patch.keys() { + if !shape.contains_key(key) { + return Err(format!("unknown expected field {key}")); + } + } + } + } + Ok(scenarios) +} + +/// Replay one scenario; every step's observation must equal the accumulated expectation. +pub fn replay_scenario(scenario: &Value) -> Result<(), String> { + let fixture = scenario.get("fixture").cloned().unwrap_or(Value::Null); + let name = scenario.get("name").and_then(Value::as_str).unwrap_or(""); + let mut driver = Driver::new(fixture.clone()); + let mut expected: Map = empty_observation(&fixture); + let result = (|| { + for (index, step) in scenario + .get("steps") + .and_then(Value::as_array) + .into_iter() + .flatten() + .enumerate() + { + if let Some(patch) = step.get("expect").and_then(Value::as_object) { + for (key, value) in patch { + expected.insert(key.clone(), value.clone()); + } + } + let input = step.get("input").cloned().unwrap_or(Value::Null); + driver + .apply(&input) + .map_err(|e| format!("{name} step {index} input {input}: {e}"))?; + let actual = driver.observation(); + let expected_value = Value::Object(expected.clone()); + if !json_equal(&expected_value, &actual) { + return Err(format!("{name} step {index} input {input}\nexpected: {expected_value}\nactual: {actual}")); + } + } + Ok(()) + })(); + driver.close(); + result +} diff --git a/rust/tests/formal/schema.rs b/rust/tests/formal/schema.rs new file mode 100644 index 00000000..edd44e1c --- /dev/null +++ b/rust/tests/formal/schema.rs @@ -0,0 +1,380 @@ +//! Local interpreter for `formal/replay/protocol.schema.json`. +//! +//! Ports `readReplaySchema`, `validateReplaySchema`, `matchesReplaySchema`, +//! `replaySchemaType` and `replayObservationError` from the Go harness. Each +//! port interprets the schema mechanics; the shared schema owns the command and +//! observation field lists, so no second copy of them lives here. + +use super::json::{json_equal, keys_of, strict_parse, MAX_SAFE_INTEGER}; +use serde_json::{Map, Value}; +use std::path::PathBuf; + +/// Repository-relative location of the shared protocol schema. +pub const SCHEMA_PATH: &str = "formal/replay/protocol.schema.json"; + +/// Observation definitions a prepare result may name. Every observation the +/// driver reports is validated against the named definition before it is sent. +pub const OBSERVATION_DEFINITIONS: [&str; 3] = [ + "behaviorObservation", + "coreObservation", + "localClockObservation", +]; + +/// Schema keywords the interpreter understands; any other keyword is rejected at +/// load so an unknown constraint cannot be silently ignored. +const KEYWORDS: [&str; 20] = [ + "$schema", + "$id", + "$defs", + "$ref", + "title", + "description", + "oneOf", + "anyOf", + "const", + "enum", + "type", + "properties", + "required", + "additionalProperties", + "items", + "minItems", + "minimum", + "maximum", + "minLength", + "pattern", +]; + +/// The loaded protocol schema. +#[derive(Debug, Clone)] +pub struct Schema { + root: Map, +} + +/// Absolute path of the shared schema, resolved from this crate's manifest. +pub fn schema_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(SCHEMA_PATH) +} + +impl Schema { + /// Reads and validates `../formal/replay/protocol.schema.json`. + pub fn load() -> Result { + let path = schema_path(); + let text = std::fs::read_to_string(&path) + .map_err(|error| format!("{}: {error}", path.display()))?; + Schema::parse(&text) + } + + /// Parses schema text, rejecting unsupported keywords and patterns. + pub fn parse(text: &str) -> Result { + let root = match strict_parse(text)? { + Value::Object(root) => root, + _ => return Err("replay schema is not an object".to_string()), + }; + validate_schema_keywords(&root)?; + validate_patterns(&root)?; + if !root.get("$defs").is_some_and(Value::is_object) { + return Err("replay schema lacks $defs".to_string()); + } + Ok(Schema { root }) + } + + /// The `$defs` table every `$ref` resolves against. + pub fn defs(&self) -> &Map { + self.root + .get("$defs") + .and_then(Value::as_object) + .expect("validated at load") + } + + /// Looks up one `$defs` definition. + pub fn definition(&self, name: &str) -> Option<&Map> { + self.defs().get(name).and_then(Value::as_object) + } + + /// Whether `value` satisfies the named `$defs` definition. + pub fn matches_definition(&self, value: &Value, name: &str) -> bool { + self.definition(name) + .is_some_and(|rule| matches(value, rule, self.defs())) + } + + /// Validates one driver observation against the definition `prepare` named. + pub fn observation_error(&self, observed: &Value, definition: &str) -> Result<(), String> { + observation_error(observed, definition, self.defs()) + } + + /// Whether every input is a well-formed `$defs/command`. + pub fn commands_valid(&self, inputs: &[Value]) -> bool { + commands_valid(inputs, self.defs()) + } +} + +/// Rejects any keyword outside the supported list, recursively through +/// `$defs`, `properties`, `oneOf`, `anyOf`, `items` and `additionalProperties`. +pub fn validate_schema_keywords(rule: &Map) -> Result<(), String> { + for key in rule.keys() { + if !KEYWORDS.contains(&key.as_str()) { + return Err(format!("unsupported replay schema keyword: {key}")); + } + } + for child in children(rule) { + validate_schema_keywords(child)?; + } + Ok(()) +} + +fn validate_patterns(rule: &Map) -> Result<(), String> { + if let Some(pattern) = rule.get("pattern") { + match pattern.as_str() { + Some(text) if pattern_supported(text) => {} + _ => return Err(format!("unsupported replay schema pattern: {pattern}")), + } + } + for child in children(rule) { + validate_patterns(child)?; + } + Ok(()) +} + +fn children(rule: &Map) -> Vec<&Map> { + let mut out = Vec::new(); + for key in ["$defs", "properties"] { + if let Some(table) = rule.get(key).and_then(Value::as_object) { + out.extend(table.values().filter_map(Value::as_object)); + } + } + for key in ["oneOf", "anyOf"] { + if let Some(options) = rule.get(key).and_then(Value::as_array) { + out.extend(options.iter().filter_map(Value::as_object)); + } + } + for key in ["items", "additionalProperties"] { + if let Some(child) = rule.get(key).and_then(Value::as_object) { + out.push(child); + } + } + out +} + +/// Whether `value` satisfies `rule`, resolving `#/$defs/` references in `defs`. +pub fn matches(value: &Value, rule: &Map, defs: &Map) -> bool { + if let Some(reference) = rule.get("$ref").and_then(Value::as_str) { + let name = reference.strip_prefix("#/$defs/").unwrap_or(reference); + return match defs.get(name) { + Some(Value::Object(target)) => matches(value, target, defs), + Some(_) => matches(value, &Map::new(), defs), + None => false, + }; + } + if let Some(options) = rule.get("oneOf").and_then(Value::as_array) { + let count = options + .iter() + .filter(|option| matches(value, as_rule(option), defs)) + .count(); + if count != 1 { + return false; + } + } + if let Some(options) = rule.get("anyOf").and_then(Value::as_array) { + if !options + .iter() + .any(|option| matches(value, as_rule(option), defs)) + { + return false; + } + } + if let Some(constant) = rule.get("const") { + if !json_equal(value, constant) { + return false; + } + } + if let Some(options) = rule.get("enum").and_then(Value::as_array) { + if !options.iter().any(|option| json_equal(value, option)) { + return false; + } + } + if let Some(expected) = rule.get("type") { + let matched = match expected { + Value::String(name) => schema_type(value, name), + Value::Array(names) => names + .iter() + .any(|name| schema_type(value, name.as_str().unwrap_or(""))), + _ => false, + }; + if !matched { + return false; + } + } + match value { + Value::Number(number) => { + let n = number.as_f64().unwrap_or(f64::NAN); + if let Some(minimum) = rule.get("minimum").and_then(Value::as_f64) { + if n < minimum { + return false; + } + } + if let Some(maximum) = rule.get("maximum").and_then(Value::as_f64) { + if n > maximum { + return false; + } + } + } + Value::String(text) => { + if let Some(minimum) = rule.get("minLength").and_then(Value::as_f64) { + if (text.chars().count() as f64) < minimum.trunc() { + return false; + } + } + if let Some(pattern) = rule.get("pattern") { + if !pattern + .as_str() + .is_some_and(|pattern| pattern_matches(pattern, text)) + { + return false; + } + } + } + Value::Array(items) => { + if let Some(minimum) = rule.get("minItems").and_then(Value::as_f64) { + if (items.len() as f64) < minimum.trunc() { + return false; + } + } + if let Some(schema) = rule.get("items").and_then(Value::as_object) { + if !items.iter().all(|item| matches(item, schema, defs)) { + return false; + } + } + } + Value::Object(members) => { + if let Some(required) = rule.get("required").and_then(Value::as_array) { + if !required + .iter() + .all(|key| members.contains_key(key.as_str().unwrap_or(""))) + { + return false; + } + } + let empty = Map::new(); + let properties = rule + .get("properties") + .and_then(Value::as_object) + .unwrap_or(&empty); + let additional = rule.get("additionalProperties"); + for (key, item) in members { + if let Some(property) = properties.get(key) { + if !matches(item, as_rule(property), defs) { + return false; + } + } else if additional == Some(&Value::Bool(false)) { + return false; + } else if let Some(schema) = additional.and_then(Value::as_object) { + if !matches(item, schema, defs) { + return false; + } + } + } + } + _ => {} + } + true +} + +fn as_rule(value: &Value) -> &Map { + static EMPTY: std::sync::OnceLock> = std::sync::OnceLock::new(); + value + .as_object() + .unwrap_or_else(|| EMPTY.get_or_init(Map::new)) +} + +/// JSON Schema primitive type check; `integer` means a finite integral number. +pub fn schema_type(value: &Value, expected: &str) -> bool { + match expected { + "null" => value.is_null(), + "object" => value.is_object(), + "array" => value.is_array(), + "string" => value.is_string(), + "boolean" => value.is_boolean(), + "integer" | "number" => match value.as_f64() { + Some(n) => n.is_finite() && (expected == "number" || n.trunc() == n), + None => false, + }, + _ => false, + } +} + +/// Validates one driver observation against the `$defs` definition the prepare +/// result named. The diagnostic names the record's keys, never its values, and +/// carries no `expected:`/`actual:` markers: a shape defect is infrastructure +/// evidence, not a mutation detection. +pub fn observation_error( + observed: &Value, + definition: &str, + defs: &Map, +) -> Result<(), String> { + let target = match defs.get(definition) { + Some(Value::Object(target)) if OBSERVATION_DEFINITIONS.contains(&definition) => target, + _ => { + return Err(format!( + "unknown replay observation definition {definition:?}" + )) + } + }; + if !matches(observed, target, defs) { + return Err(format!( + "driver produced a malformed {definition} observation: keys [{}]", + keys_of(observed) + )); + } + Ok(()) +} + +/// Whether every input is a well-formed `$defs/command`. +pub fn commands_valid(inputs: &[Value], defs: &Map) -> bool { + match defs.get("command") { + Some(Value::Object(command)) => inputs.iter().all(|input| matches(input, command, defs)), + _ => false, + } +} + +/// Whether `value` is a whole number in `[minimum, MAX_SAFE_INTEGER]`, the +/// check the transport applies to step counts and indices. +pub fn safe_index(value: Option<&Value>, minimum: i64) -> bool { + match value.and_then(Value::as_f64) { + Some(n) => n >= minimum as f64 && n <= MAX_SAFE_INTEGER as f64 && n.trunc() == n, + None => false, + } +} + +// The schema uses exactly two regular expressions. They are matched by hand so +// the harness needs no regex dependency; `validate_patterns` fails loading if +// the schema ever gains another one. +const POSITIVE_INTEGER: &str = "^[1-9][0-9]*$"; +const CALL_ERROR: &str = "^(source|timeout):(0|[1-9][0-9]*)$|^unexpected:"; + +fn pattern_supported(pattern: &str) -> bool { + pattern == POSITIVE_INTEGER || pattern == CALL_ERROR +} + +fn pattern_matches(pattern: &str, text: &str) -> bool { + match pattern { + POSITIVE_INTEGER => positive_integer(text), + CALL_ERROR => { + if text.starts_with("unexpected:") { + return true; + } + ["source:", "timeout:"] + .iter() + .filter_map(|prefix| text.strip_prefix(prefix)) + .any(|rest| rest == "0" || positive_integer(rest)) + } + _ => false, + } +} + +fn positive_integer(text: &str) -> bool { + let mut chars = text.chars(); + matches!(chars.next(), Some('1'..='9')) && chars.all(|c| c.is_ascii_digit()) +} diff --git a/rust/tests/formal/transport.rs b/rust/tests/formal/transport.rs new file mode 100644 index 00000000..234b044e --- /dev/null +++ b/rust/tests/formal/transport.rs @@ -0,0 +1,526 @@ +//! JSON-lines client for the shared Node replay coordinator. +//! +//! Ports `replayCoordinator` from `go/replay_coordinator_test.go`: one request +//! per line on the child's stdin, one reply per line on its stdout, a strictly +//! increasing request id, exact envelope checks and a real-time watchdog that +//! kills a child which leaves a request pending longer than the timeout. +//! +//! The coordinator is `node /formal/replay/coordinator.mjs`, started with +//! the `node` found on `PATH`. **It must be Node 24** (the shared replay code +//! relies on it); when running these tests on a machine whose default `node` +//! is another major version, prepend the Node 24 `bin` directory to `PATH` +//! (for example `/opt/homebrew/opt/node@24/bin`) before invoking `cargo test`. +//! Node is test tooling only; the cache library itself has no Node dependency. +//! +//! Everything here is synchronous (`std::process`, `std::io`, threads). The +//! watchdog observes real process time only; replay IO never consumes a driver +//! deadline or releases a cache gate. + +use super::json::{sorted_keys, strict_parse}; +use super::schema::{safe_index, Schema, OBSERVATION_DEFINITIONS}; +use serde_json::{Map, Value}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// The settlement contract every observe request declares. +pub const SETTLEMENT: &str = "causally-ready-v1"; + +/// Largest reply frame the transport buffers. +pub const FRAME_LIMIT: usize = 64 * 1024 * 1024; + +/// Default real-time bound on one coordinator round trip. +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); + +/// Repository-relative location of the shared coordinator program. +pub const COORDINATOR_PATH: &str = "formal/replay/coordinator.mjs"; + +/// Absolute path of `formal/replay/coordinator.mjs`, resolved from this crate's manifest. +pub fn coordinator_program() -> Result { + let relative = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(COORDINATOR_PATH); + std::path::absolute(&relative).map_err(|error| format!("{}: {error}", relative.display())) +} + +/// A validated `prepare` result: the coordinator's session for one history. +#[derive(Debug, Clone, PartialEq)] +pub struct Prepared { + /// Session identifier for the following `observe`/`discard` requests. + pub session: String, + /// Number of steps, including the initial observation at index 0. + pub steps: i64, + /// `$defs` definition every observation of this session must satisfy. + pub observation: String, + /// Receipt definition for a behavior session; absent for core/local-clock. + pub receipt: Option, + /// Initialization fixture for the native driver (an object). + pub fixture: Value, + /// Commands to apply before the first observation. + pub setup: Vec, + /// Action name of every step; the first is always `init`. + pub actions: Vec, +} + +/// One running coordinator process and its JSON-lines connection. +pub struct Coordinator { + schema: Schema, + child: Arc>, + input: Option, + output: BufReader, + stderr: Option>>, + watchdog: Option>, + pending: Arc>>, + stopped: Arc, + sequence: i64, + expect_process_failure: bool, + finished: bool, +} + +impl std::fmt::Debug for Coordinator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Coordinator") + .field("sequence", &self.sequence) + .field("finished", &self.finished) + .finish_non_exhaustive() + } +} + +impl Coordinator { + /// Starts the shared coordinator (`node formal/replay/coordinator.mjs`) + /// with the default 30 s request timeout. + pub fn spawn() -> Result { + let mut command = Command::new("node"); + command.arg(coordinator_program()?); + Coordinator::start(command, DEFAULT_TIMEOUT, false) + } + + /// Starts an arbitrary child as the coordinator. Test controls substitute + /// a broken transport this way without replacing any cache implementation; + /// `expect_process_failure` silences the exit-status check in [`finish`]. + /// + /// [`finish`]: Coordinator::finish + pub fn start( + mut command: Command, + timeout: Duration, + expect_process_failure: bool, + ) -> Result { + let schema = Schema::load()?; + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = command + .spawn() + .map_err(|error| format!("cannot start replay coordinator: {error}"))?; + let input = child.stdin.take().ok_or("coordinator stdin unavailable")?; + let output = child + .stdout + .take() + .ok_or("coordinator stdout unavailable")?; + let mut errors = child + .stderr + .take() + .ok_or("coordinator stderr unavailable")?; + let stderr = std::thread::spawn(move || { + let mut captured = Vec::new(); + let _ = errors.read_to_end(&mut captured); + captured + }); + let child = Arc::new(Mutex::new(child)); + let pending: Arc>> = Arc::new(Mutex::new(None)); + let stopped = Arc::new(AtomicBool::new(false)); + let watchdog = { + let (child, pending, stopped) = ( + Arc::clone(&child), + Arc::clone(&pending), + Arc::clone(&stopped), + ); + std::thread::spawn(move || { + while !stopped.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(10)); + let expired = pending + .lock() + .map(|since| since.is_some_and(|since| since.elapsed() > timeout)) + .unwrap_or(false); + if expired { + if let Ok(mut child) = child.lock() { + let _ = child.kill(); + } + return; + } + } + }) + }; + Ok(Coordinator { + schema, + child, + input: Some(input), + output: BufReader::new(output), + stderr: Some(stderr), + watchdog: Some(watchdog), + pending, + stopped, + sequence: 0, + expect_process_failure, + finished: false, + }) + } + + /// The protocol schema this transport validates against. + pub fn schema(&self) -> &Schema { + &self.schema + } + + fn arm(&self, since: Option) { + if let Ok(mut pending) = self.pending.lock() { + *pending = since; + } + } + + /// Sends one request, adding `version: 1` and the next id, and returns the + /// validated `result` object of a successful reply. + pub fn call(&mut self, mut request: Map) -> Result { + self.sequence += 1; + request.insert("version".to_string(), Value::from(1)); + request.insert("id".to_string(), Value::from(self.sequence)); + let mut raw = + serde_json::to_vec(&Value::Object(request)).map_err(|error| error.to_string())?; + raw.push(b'\n'); + self.arm(Some(Instant::now())); + let exchanged = self.exchange(&raw); + self.arm(None); + let line = exchanged?; + let text = std::str::from_utf8(&line) + .map_err(|_| "coordinator response is not UTF-8".to_string())?; + let envelope = match strict_parse(text)? { + Value::Object(envelope) => envelope, + _ => return Err("malformed coordinator envelope".to_string()), + }; + let ok = match envelope.get("ok") { + Some(Value::Bool(ok)) => *ok, + Some(_) => return Err("malformed coordinator envelope".to_string()), + None => false, + }; + let keys = sorted_keys(&envelope); + if (ok && keys != "id,ok,result,version") || (!ok && keys != "error,id,ok,version") { + return Err("malformed coordinator envelope".to_string()); + } + if !safe_index(envelope.get("version"), 1) || !safe_index(envelope.get("id"), 1) { + return Err("malformed coordinator envelope".to_string()); + } + let version = envelope + .get("version") + .and_then(Value::as_f64) + .unwrap_or(0.0); + let id = envelope.get("id").and_then(Value::as_f64).unwrap_or(0.0); + if version != 1.0 || id != self.sequence as f64 { + return Err("unknown or out-of-sequence coordinator response".to_string()); + } + if !ok { + return match envelope.get("error") { + Some(Value::String(message)) if !message.is_empty() => { + Err(format!("shared replay: {message}")) + } + _ => Err("malformed coordinator failure".to_string()), + }; + } + match envelope.get("result") { + Some(Value::Object(_)) => Ok(envelope.get("result").cloned().unwrap_or(Value::Null)), + _ => Err("malformed coordinator success".to_string()), + } + } + + fn exchange(&mut self, raw: &[u8]) -> Result, String> { + let input = self + .input + .as_mut() + .ok_or("coordinator input already closed")?; + input + .write_all(raw) + .and_then(|()| input.flush()) + .map_err(|error| format!("coordinator write failed: {error}"))?; + read_frame(&mut self.output, FRAME_LIMIT).map_err(|error| { + format!("coordinator unavailable or exceeded real-time request limit: {error}") + }) + } + + /// Prepares one history and validates the coordinator's session description. + pub fn prepare( + &mut self, + profile: &str, + path: &Path, + raw: Option<&str>, + ) -> Result { + let absolute = + std::path::absolute(path).map_err(|error| format!("{}: {error}", path.display()))?; + let mut request = Map::new(); + request.insert("op".to_string(), Value::from("prepare")); + request.insert("profile".to_string(), Value::from(profile)); + request.insert( + "path".to_string(), + Value::from(absolute.to_string_lossy().into_owned()), + ); + if let Some(raw) = raw { + request.insert("raw".to_string(), Value::from(raw)); + } + let result = self.call(request)?; + let result = result.as_object().ok_or("malformed replay preparation")?; + let session = result.get("session").and_then(Value::as_str).unwrap_or(""); + if sorted_keys(result) + != "actions,fixture,observation,receipt,session,settlement,setup,steps" + || result.get("settlement").and_then(Value::as_str) != Some(SETTLEMENT) + || session.is_empty() + || !safe_index(result.get("steps"), 2) + { + return Err("malformed replay preparation".to_string()); + } + let definition = result + .get("observation") + .and_then(Value::as_str) + .unwrap_or(""); + if !OBSERVATION_DEFINITIONS.contains(&definition) + || self.schema.definition(definition).is_none() + { + return Err(format!( + "malformed replay observation definition {definition:?}" + )); + } + let receipt = match result.get("receipt") { + Some(Value::Null) if definition != "behaviorObservation" => None, + Some(Value::String(name)) + if name == "settlementReceipt" + && definition == "behaviorObservation" + && self.schema.definition(name).is_some() => + { + Some(name.clone()) + } + _ => return Err("malformed replay receipt definition".to_string()), + }; + let fixture = match result.get("fixture") { + Some(fixture @ Value::Object(_)) => fixture.clone(), + _ => return Err("malformed replay fixture".to_string()), + }; + let setup = match result.get("setup") { + Some(Value::Array(setup)) if self.schema.commands_valid(setup) => setup.clone(), + _ => return Err("malformed replay setup".to_string()), + }; + let steps = result.get("steps").and_then(Value::as_f64).unwrap_or(0.0) as i64; + let actions = match result.get("actions") { + Some(Value::Array(actions)) if actions.len() as i64 == steps => actions, + _ => return Err("malformed replay actions".to_string()), + }; + let mut names = Vec::with_capacity(actions.len()); + for (index, action) in actions.iter().enumerate() { + let name = action.as_str().unwrap_or(""); + if name.is_empty() || (index == 0) != (name == "init") { + return Err("malformed replay action".to_string()); + } + names.push(name.to_string()); + } + Ok(Prepared { + session: session.to_string(), + steps, + observation: definition.to_string(), + receipt, + fixture, + setup, + actions: names, + }) + } + + /// Replays one prepared history through a driver: applies setup, then for + /// every step runs the monitors, validates the observation locally, sends + /// it and applies the returned commands until the coordinator completes the + /// session. Any early exit discards the coordinator session. + pub fn execute( + &mut self, + prepared: &Prepared, + apply: &mut dyn FnMut(&Value) -> Result<(), String>, + observation: &mut dyn FnMut() -> Value, + wall_ms: &mut dyn FnMut() -> i64, + receipt: Option<&mut dyn FnMut() -> Value>, + monitors: &mut [&mut dyn FnMut() -> Result<(), String>], + ) -> Result<(), String> { + let outcome = self.run(prepared, apply, observation, wall_ms, receipt, monitors); + if outcome.is_err() { + let mut request = Map::new(); + request.insert("op".to_string(), Value::from("discard")); + request.insert( + "session".to_string(), + Value::from(prepared.session.as_str()), + ); + let _ = self.call(request); + } + outcome + } + + fn run( + &mut self, + prepared: &Prepared, + apply: &mut dyn FnMut(&Value) -> Result<(), String>, + observation: &mut dyn FnMut() -> Value, + wall_ms: &mut dyn FnMut() -> i64, + mut receipt: Option<&mut dyn FnMut() -> Value>, + monitors: &mut [&mut dyn FnMut() -> Result<(), String>], + ) -> Result<(), String> { + if prepared.receipt.is_some() != receipt.is_some() { + return Err(if prepared.receipt.is_some() { + "driver omitted its settlement receipt callback" + } else { + "driver supplied an unexpected settlement receipt callback" + } + .to_string()); + } + for input in &prepared.setup { + apply(input)?; + } + for index in 0..prepared.steps { + for monitor in monitors.iter_mut() { + monitor()?; + } + // A malformed record is a driver defect. Attribute it here, before + // the coordinator sees it, so no round trip or session state is + // spent on it. + let observed = observation(); + self.schema + .observation_error(&observed, &prepared.observation)?; + let mut environment = Map::new(); + environment.insert("wallMs".to_string(), Value::from(wall_ms())); + let mut request = Map::new(); + request.insert("op".to_string(), Value::from("observe")); + request.insert( + "session".to_string(), + Value::from(prepared.session.as_str()), + ); + request.insert("index".to_string(), Value::from(index)); + request.insert("settlement".to_string(), Value::from(SETTLEMENT)); + request.insert("observed".to_string(), observed); + request.insert("environment".to_string(), Value::Object(environment)); + if let Some(callback) = receipt.as_mut() { + let value = callback(); + let definition = prepared + .receipt + .as_deref() + .expect("validated receipt callback"); + if !self.schema.matches_definition(&value, definition) { + return Err(format!("driver produced a malformed {definition} receipt")); + } + request.insert("receipt".to_string(), value); + } + let result = self.call(request)?; + let result = result.as_object().ok_or("malformed next replay command")?; + let next = index + 1; + if result.get("complete") == Some(&Value::Bool(true)) { + let steps = result.get("steps"); + if sorted_keys(result) != "complete,steps" + || next != prepared.steps + || !safe_index(steps, 2) + || steps.and_then(Value::as_f64) != Some(next as f64) + { + return Err("premature or malformed replay completion".to_string()); + } + return Ok(()); + } + let inputs = result.get("inputs").and_then(Value::as_array); + if sorted_keys(result) != "complete,index,inputs" + || result.get("complete") != Some(&Value::Bool(false)) + || !safe_index(result.get("index"), 1) + || result.get("index").and_then(Value::as_f64) != Some(next as f64) + || !inputs + .is_some_and(|inputs| !inputs.is_empty() && self.schema.commands_valid(inputs)) + { + return Err("malformed next replay command".to_string()); + } + for input in inputs.unwrap_or(&Vec::new()) { + apply(input)?; + } + } + Err("replay ended without completion".to_string()) + } + + /// Closes the child's stdin, waits for it to exit and reports a failed exit + /// (with its captured stderr) unless the process was expected to fail. The + /// watchdog stays armed while closing so a child that ignores EOF cannot + /// hang the suite. + pub fn finish(mut self) -> Result<(), String> { + self.close() + } + + fn close(&mut self) -> Result<(), String> { + if self.finished { + return Ok(()); + } + self.finished = true; + self.arm(Some(Instant::now())); + drop(self.input.take()); + // Poll instead of blocking in wait() so the watchdog can take the child + // handle and kill a process that ignores EOF. + let status = loop { + let polled = match self.child.lock() { + Ok(mut child) => child.try_wait().map_err(|error| error.to_string()), + Err(_) => Err("coordinator handle poisoned".to_string()), + }; + match polled { + Ok(Some(status)) => break Ok(status), + Ok(None) => std::thread::sleep(Duration::from_millis(5)), + Err(error) => break Err(error), + } + }; + self.stopped.store(true, Ordering::SeqCst); + self.arm(None); + if let Some(watchdog) = self.watchdog.take() { + let _ = watchdog.join(); + } + let stderr = self + .stderr + .take() + .and_then(|thread| thread.join().ok()) + .unwrap_or_default(); + let status = status?; + if !status.success() && !self.expect_process_failure { + return Err(format!( + "shared replay process failed: {status}\n{}", + String::from_utf8_lossy(&stderr) + )); + } + Ok(()) + } +} + +impl Drop for Coordinator { + fn drop(&mut self) { + let _ = self.close(); + } +} + +/// Reads one newline-terminated frame, failing with `oversized coordinator +/// response` before buffering more than `limit` bytes. The reader's own buffer +/// bounds each fragment, so an unbounded line never accumulates in memory. +pub fn read_frame(reader: &mut R, limit: usize) -> Result, String> { + let mut line = Vec::new(); + loop { + let (fragment, done) = { + let available = reader.fill_buf().map_err(|error| error.to_string())?; + if available.is_empty() { + return Err("coordinator closed its output before completing a frame".to_string()); + } + match available.iter().position(|byte| *byte == b'\n') { + Some(at) => (available[..=at].to_vec(), true), + None => (available.to_vec(), false), + } + }; + if line.len() + fragment.len() > limit { + return Err("oversized coordinator response".to_string()); + } + reader.consume(fragment.len()); + line.extend_from_slice(&fragment); + if done { + return Ok(line); + } + } +} diff --git a/rust/tests/formal/witness.rs b/rust/tests/formal/witness.rs new file mode 100644 index 00000000..2ce08c83 --- /dev/null +++ b/rust/tests/formal/witness.rs @@ -0,0 +1,352 @@ +//! Witness-evidence check. +//! +//! Ports `checkWitnessEvidenceAt`, `witnessTraceKind` and `sharedReplaySources` +//! from `go/witness_evidence_test.go`. Required reachability witnesses have one +//! evaluator shared by the language drivers (`node formal/witnesses.mjs +//! evaluate`); reusing its result requires the exact corpus and definition +//! hashes recorded in `/.json` to match this checkout byte +//! for byte. It does not replace any native execution or observation assertion. + +use super::inventory::trace_kind; +use super::json::strict_parse; +use serde::{Deserialize, Deserializer}; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +/// Hex SHA-256 of a file's bytes. +pub fn file_sha256(path: &Path) -> Result { + let raw = std::fs::read(path).map_err(|error| format!("{}: {error}", path.display()))?; + Ok(sha256_hex(&raw)) +} + +pub use crate::digest::sha256_hex; + +fn null_default<'de, D: Deserializer<'de>, T: Default + Deserialize<'de>>( + deserializer: D, +) -> Result { + Option::::deserialize(deserializer).map(Option::unwrap_or_default) +} + +/// `{ path?, name?, sha256 }` fingerprint of one input or history. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(default)] +pub struct Digest { + /// Repository-relative definition path (inputs). + pub path: String, + /// History file name (corpus). + pub name: String, + /// Hex SHA-256. + pub sha256: String, +} + +/// One history that earned a label. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(default)] +pub struct Trace { + /// History file name. + pub name: String, + /// `sampled` or `regression`. + pub kind: String, + /// Step indices at which the classifier credited the label. + #[serde(deserialize_with = "null_default")] + pub checkpoints: Vec, +} + +/// Per-label provenance (schema 2). +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(default)] +pub struct Label { + /// Sampled histories that earned the label. + pub sampled: i64, + /// Exported regressions that earned the label. + pub regression: i64, + /// The citing histories. + #[serde(deserialize_with = "null_default")] + pub traces: Vec, +} + +/// The evidence document written by `node formal/witnesses.mjs evaluate`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Evidence { + pub schema_version: i64, + pub profile: String, + pub traces: i64, + #[serde(deserialize_with = "null_default")] + pub required: Vec, + #[serde(deserialize_with = "null_default")] + pub seen: Vec, + /// `None` when the document lacks per-label provenance entirely. + pub labels: Option>, + #[serde(deserialize_with = "null_default")] + pub inputs: Vec, + #[serde(deserialize_with = "null_default")] + pub corpus: Vec, +} + +fn read_strict(path: &Path) -> Result { + let text = + std::fs::read_to_string(path).map_err(|error| format!("{}: {error}", path.display()))?; + strict_parse(&text).map_err(|error| format!("{}: {error}", path.display())) +} + +fn decode Deserialize<'de>>(path: &Path) -> Result { + serde_json::from_value(read_strict(path)?) + .map_err(|error| format!("{}: {error}", path.display())) +} + +/// Checks `/.json` against the registry, definitions and +/// corpus of the checkout at `root`, where `paths` are the histories this run +/// replays for the profile. +pub fn check_witness_evidence( + root: &Path, + profile: &str, + directory: &Path, + paths: &[PathBuf], +) -> Result<(), String> { + let evidence: Evidence = decode(&directory.join(format!("{profile}.json")))?; + if evidence.schema_version != 2 + || evidence.profile != profile + || evidence.traces != paths.len() as i64 + || evidence.corpus.len() != paths.len() + { + return Err(format!("unsupported/incomplete {profile} witness evidence")); + } + let registry: HashMap> = + decode(&root.join("formal/coverage-witnesses.json"))?; + let required = registry.get(profile).cloned().unwrap_or_default(); + if required.is_empty() || required != evidence.required { + return Err(format!("{profile} required witness registry differs")); + } + let mut seen = std::collections::HashSet::new(); + for name in &evidence.seen { + if !seen.insert(name.as_str()) { + return Err(format!("duplicate witness {name}")); + } + } + for name in &required { + if !seen.contains(name.as_str()) { + return Err(format!("{profile} missing witness {name}")); + } + } + // Every required label names the histories that earned it: at least one + // sampled history or exported regression, each part of the bound corpus + // with the kind its directory gives it and cited at a checkpoint, and the + // split counts must agree with the cited histories. + let labels = evidence + .labels + .as_ref() + .ok_or_else(|| format!("{profile} witness evidence lacks per-label provenance"))?; + let corpus_kinds: HashMap = paths + .iter() + .map(|path| (base_name(path), trace_kind(path))) + .collect(); + for name in &required { + let label = match labels.get(name) { + Some(label) if label.sampled + label.regression >= 1 => label, + _ => return Err(format!("{profile} witness {name} lacks provenance")), + }; + let (mut sampled, mut regression) = (0i64, 0i64); + for trace in &label.traces { + let kind = *corpus_kinds.get(&trace.name).ok_or_else(|| { + format!( + "{profile} witness {name} cites an unknown history {}", + trace.name + ) + })?; + if trace.kind != kind { + return Err(format!( + "{profile} witness {name} reports {kind} history {} as {}", + trace.name, trace.kind + )); + } + if trace.checkpoints.is_empty() { + return Err(format!( + "{profile} witness {name} cites {} without a checkpoint", + trace.name + )); + } + if kind == "sampled" { + sampled += 1; + } else { + regression += 1; + } + } + if sampled != label.sampled || regression != label.regression { + return Err(format!( + "{profile} witness {name} counts {} sampled and {} regression hits but cites {sampled} and {regression}", + label.sampled, label.regression + )); + } + } + // The evidence binds language-neutral definitions only: the registry, the + // required witnesses, the execution manifest, the profile's model and the + // observation library, then every Quint library, the shared replay closure + // (which holds the witness classifiers) and the profile's witness sources. + let mut expected: Vec = [ + "formal/profiles.json".to_string(), + "formal/coverage-witnesses.json".to_string(), + "formal/execution.json".to_string(), + format!("formal/dialcache-{profile}-conformance.qnt"), + "formal/conformance-observations.qnt".to_string(), + ] + .to_vec(); + let execution = read_strict(&root.join("formal/execution.json"))?; + let definitions = read_strict(&root.join("formal/profiles.json"))?; + let shared = shared_replay_sources(root)?; + let claimed: HashSet = execution + .get("models") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .map(|model| { + model + .get("path") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| "execution model must name its path".to_string()) + }) + .collect::>()?; + let mut additional = quint_libraries(root, &claimed)?; + additional.extend(shared); + for definition in definitions + .get("profiles") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + { + if definition.get("id").and_then(serde_json::Value::as_str) == Some(profile) { + additional.extend(strings_at(definition, "witnessSources")?); + } + } + for path in additional { + if !expected.contains(&path) { + expected.push(path); + } + } + if evidence.inputs.len() != expected.len() { + return Err("incomplete witness definition fingerprints".to_string()); + } + for (item, path) in evidence.inputs.iter().zip(&expected) { + if &item.path != path { + return Err(format!("unexpected witness input {}", item.path)); + } + if file_sha256(&root.join(path))? != item.sha256 { + return Err(format!("stale witness definition {path}")); + } + } + let mut actual: HashMap = HashMap::new(); + for path in paths { + let name = base_name(path); + if actual.contains_key(&name) { + return Err(format!("duplicate trace name {name}")); + } + actual.insert(name, file_sha256(path)?); + } + for item in &evidence.corpus { + match actual.remove(&item.name) { + Some(hash) if hash == item.sha256 => {} + _ => return Err(format!("{profile} witness corpus differs at {}", item.name)), + } + } + if !actual.is_empty() { + return Err("unaccounted replay traces".to_string()); + } + Ok(()) +} + +/// Every Quint source in `formal/` and `formal/kernel/` not claimed by a +/// scheduled model, sorted as `formal/execution.mjs` derives library inputs. +pub fn quint_libraries(root: &Path, claimed: &HashSet) -> Result, String> { + let mut libraries = Vec::new(); + for folder in ["formal", "formal/kernel"] { + let directory = root.join(folder); + let entries = match std::fs::read_dir(&directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(format!("{}: {error}", directory.display())), + }; + for entry in entries { + let entry = entry.map_err(|error| format!("{}: {error}", directory.display()))?; + let name = entry.file_name().to_string_lossy().into_owned(); + if entry + .file_type() + .map_err(|error| format!("{}: {error}", entry.path().display()))? + .is_dir() + || !name.ends_with(".qnt") + { + continue; + } + let path = format!("{folder}/{name}"); + if !claimed.contains(&path) { + libraries.push(path); + } + } + } + libraries.sort(); + Ok(libraries) +} + +fn strings_at(value: &serde_json::Value, key: &str) -> Result, String> { + match value.get(key) { + None | Some(serde_json::Value::Null) => Ok(Vec::new()), + Some(serde_json::Value::Array(items)) => items + .iter() + .map(|item| { + item.as_str() + .map(str::to_owned) + .ok_or_else(|| format!("{key} must list strings")) + }) + .collect(), + Some(_) => Err(format!("{key} must be an array")), + } +} + +fn base_name(path: &Path) -> String { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Matches `replaySources` of `formal/profiles.json` against the `.mjs`, +/// `.mts` and `.json` files below `formal/replay`, sorted, and returns them. +pub fn shared_replay_sources(root: &Path) -> Result, String> { + let registry = read_strict(&root.join("formal/profiles.json"))?; + let declared = strings_at(®istry, "replaySources")?; + let mut actual = Vec::new(); + walk(root, &root.join("formal/replay"), &mut actual)?; + actual.sort(); + if actual.is_empty() || declared != actual { + return Err("shared replay source inventory differs from formal/replay".to_string()); + } + Ok(actual) +} + +fn walk(root: &Path, directory: &Path, out: &mut Vec) -> Result<(), String> { + let entries = std::fs::read_dir(directory) + .map_err(|error| format!("{}: {error}", directory.display()))?; + for entry in entries { + let entry = entry.map_err(|error| format!("{}: {error}", directory.display()))?; + let path = entry.path(); + if path.is_dir() { + walk(root, &path, out)?; + continue; + } + if matches!( + path.extension().and_then(|extension| extension.to_str()), + Some("mjs" | "mts" | "json") + ) { + let relative = path + .strip_prefix(root) + .map_err(|error| format!("{}: {error}", path.display()))?; + out.push( + relative + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"), + ); + } + } + Ok(()) +} diff --git a/rust/tests/harness_infra.rs b/rust/tests/harness_infra.rs new file mode 100644 index 00000000..b8837195 --- /dev/null +++ b/rust/tests/harness_infra.rs @@ -0,0 +1,1413 @@ +//! Controls for the replay infrastructure itself: the coordinator transport, +//! strict JSON, schema validation, inventory ids and the witness-evidence check. +//! These port the harness tests of `go/replay_coordinator_test.go`, +//! `go/behavior_replay_test.go` and `go/witness_evidence_test.go`. +//! +//! The transport tests spawn `node`; `node` on `PATH` must be Node 24 (see +//! `formal::transport`). Prepend `/opt/homebrew/opt/node@24/bin` to `PATH` +//! when the default toolchain is another major version. + +#[path = "formal/digest.rs"] +mod digest; + +mod formal; + +use formal::inventory::{ + percent_encode_component, profile_registry_check_text, protocol_case_id, registry_check, + regression_paths, repo_root, require_behavior_profile, scenario_case_id, trace_case_id, + trace_kind, witness_case_id, BEHAVIOR_PROFILE_VERSIONS, +}; +use formal::json::{decode_itf, json_equal, sorted_keys, strict_parse}; +use formal::report::Report; +use formal::schema::{observation_error, Schema}; +use formal::transport::{read_frame, Coordinator, Prepared, SETTLEMENT}; +use formal::witness::{ + check_witness_evidence, file_sha256, sha256_hex, Digest, Evidence, Label, Trace, +}; +use serde_json::{json, Map, Value}; +use std::collections::HashMap; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +fn node(program: &str) -> Command { + let mut command = Command::new("node"); + command.arg("-e").arg(program); + command +} + +fn request(op: &str) -> Map { + let mut request = Map::new(); + request.insert("op".to_string(), Value::from(op)); + request +} + +/// The flat all-zero record `$defs/coreObservation` accepts. +fn healthy_core_observation() -> Value { + let mut observed = Map::new(); + for field in "sourceVersion lastResult outsideLoaderCalls requestLoaderCalls localLoaderCalls coalescedLoaderCalls remoteLoaderCalls redisReads redisWrites".split(' ') { + observed.insert(field.to_string(), Value::from(0)); + } + Value::Object(observed) +} + +/// The empty behavior record; `events` is present exactly when the fixture observes. +fn empty_behavior_observation(observe: bool) -> Value { + let mut observed = Map::new(); + for field in [ + "loaders", + "reads", + "writes", + "invalidations", + "loads", + "dumps", + "policyCalls", + "classifications", + "comparisons", + ] { + observed.insert(field.to_string(), Value::from(0)); + } + for field in [ + "calls", + "maintenance", + "sourceScopes", + "writeTtls", + "shadow", + "recovery", + ] { + observed.insert(field.to_string(), Value::Array(Vec::new())); + } + if observe { + observed.insert("events".to_string(), Value::Array(Vec::new())); + } + Value::Object(observed) +} + +fn with(base: &Value, field: &str, value: Value) -> Value { + let mut changed = base.as_object().cloned().unwrap_or_default(); + changed.insert(field.to_string(), value); + Value::Object(changed) +} + +fn without(base: &Value, field: &str) -> Value { + let mut changed = base.as_object().cloned().unwrap_or_default(); + changed.remove(field); + Value::Object(changed) +} + +#[test] +fn publication_causality_distinguishes_overlapping_source_owners_and_deadlines() { + use formal::causal::{assert_publication_causality, CausalEvent::*}; + let overlap = vec![ + SourceStart { + id: 0, + owner: 10, + at_ms: 0, + budget_ms: Some(10), + }, + SourceStart { + id: 1, + owner: 20, + at_ms: 10, + budget_ms: Some(10), + }, + SourceSettlement { + id: 0, + at_ms: 15, + outcome: "resolve", + }, + SourceSettlement { + id: 1, + at_ms: 16, + outcome: "resolve", + }, + ]; + for (source, owner, expected) in [ + (0, 10, "late raw settlement"), + (1, 10, "different invocation"), + ] { + let mut history = overlap.clone(); + history.push(WriteDispatch { + source: Some(source), + owner: Some(owner), + at_ms: 17, + }); + let error = assert_publication_causality(&history).unwrap_err(); + assert!(error.starts_with("CAUSAL_PROPERTY_FAILURE"), "{error}"); + assert!(error.contains(expected), "{error}"); + } + let mut accepted = overlap.clone(); + accepted.push(WriteDispatch { + source: Some(1), + owner: Some(20), + at_ms: 30, + }); + assert_publication_causality(&accepted) + .expect("publication can outlive an accepted source's deadline"); + assert_publication_causality(&overlap[..2]).expect("pending prefixes are allowed"); + assert_publication_causality(&[ + SourceStart { + id: 0, + owner: 0, + at_ms: 0, + budget_ms: None, + }, + SourceSettlement { + id: 0, + at_ms: 100_000, + outcome: "resolve", + }, + WriteDispatch { + source: Some(0), + owner: Some(0), + at_ms: 100_000, + }, + ]) + .expect("unbounded sources stay unbounded"); + for outcome in [None, Some("reject")] { + let mut history = vec![SourceStart { + id: 0, + owner: 0, + at_ms: 0, + budget_ms: Some(10), + }]; + if let Some(outcome) = outcome { + history.push(SourceSettlement { + id: 0, + at_ms: 1, + outcome, + }); + } + history.push(WriteDispatch { + source: Some(0), + owner: Some(0), + at_ms: 1, + }); + assert!(assert_publication_causality(&history) + .unwrap_err() + .contains("exact source's successful settlement")); + } +} + +#[test] +fn malformed_monitor_inputs_never_become_mutation_evidence() { + use formal::causal::{assert_publication_causality, CausalEvent::*}; + use formal::driver::{Driver, HistoryEvent}; + for history in [ + vec![SourceStart { + id: 0, + owner: 0, + at_ms: 0, + budget_ms: Some(0), + }], + vec![SourceSettlement { + id: 0, + at_ms: 0, + outcome: "resolve", + }], + vec![WriteDispatch { + source: None, + owner: None, + at_ms: 0, + }], + vec![SourceStart { + id: 0, + owner: 0, + at_ms: -1, + budget_ms: None, + }], + vec![ + SourceStart { + id: 0, + owner: 0, + at_ms: 0, + budget_ms: None, + }, + SourceSettlement { + id: 0, + at_ms: 1, + outcome: "unknown", + }, + ], + ] { + let error = assert_publication_causality(&history).unwrap_err(); + assert!(!error.contains("CAUSAL_PROPERTY_FAILURE"), "{error}"); + } + let start = HistoryEvent { + event: "sourceStart", + id: 0, + at: 0, + outcome: "", + duration_ms: 0.0, + failed: false, + }; + let settled = HistoryEvent { + event: "sourceSettlement", + at: 10, + outcome: "resolve", + ..start.clone() + }; + let completed = HistoryEvent { + event: "fallbackCompletion", + at: 10, + duration_ms: 10.0, + ..start.clone() + }; + for history in [ + vec![completed.clone()], + vec![settled.clone()], + vec![ + start.clone(), + HistoryEvent { + outcome: "unknown", + ..settled.clone() + }, + ], + ] { + let error = Driver::assert_effects_events(&history).unwrap_err(); + assert!(!error.contains("CAUSAL_PROPERTY_FAILURE"), "{error}"); + } + let error = Driver::assert_effects_events(&[start, settled, completed]).unwrap_err(); + assert!( + error.starts_with("CAUSAL_PROPERTY_FAILURE rule=C25 event="), + "{error}" + ); +} + +#[test] +fn invocation_context_follows_spawn_and_defer_without_leaking_between_polls() { + use dialcache::{testing::TestExecutor, Runtime}; + use formal::causal::{current_invocation, InvocationFuture, InvocationRuntime}; + use formal::gate::Gate; + use parking_lot::Mutex; + use std::sync::Arc; + let mut exec = TestExecutor::new(0); + let runtime = Arc::new(InvocationRuntime(exec.runtime.clone())); + let seen = Arc::new(Mutex::new(Vec::new())); + let gate = Gate::<()>::new(); + for owner in [10, 20] { + let runtime = runtime.clone(); + let seen = seen.clone(); + let gate = gate.clone(); + exec.spawn(InvocationFuture::new(Some(owner), async move { + seen.lock().push(current_invocation()); + let deferred_seen = seen.clone(); + runtime.defer(Box::pin(async move { + deferred_seen.lock().push(current_invocation()); + })); + runtime.spawn(Box::pin(async move { + gate.wait().await; + seen.lock().push(current_invocation()); + })); + })); + } + exec.drain(); + assert_eq!(current_invocation(), None); + gate.settle(()); + exec.drain(); + assert_eq!(current_invocation(), None); + let mut seen = seen.lock().clone(); + seen.sort(); + assert_eq!( + seen, + [Some(10), Some(10), Some(10), Some(20), Some(20), Some(20)] + ); + let panicking = InvocationFuture::new(Some(30), async { panic!("controlled context unwind") }); + assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe( + || futures::executor::block_on(panicking) + )) + .is_err()); + assert_eq!(current_invocation(), None); +} + +#[test] +fn behavior_driver_attributes_detached_writes_to_their_actual_sources() { + use formal::causal::CausalEvent; + use formal::driver::Driver; + let mut driver = Driver::new(json!({"policy":{"ttlSec":{"remote":10},"coalesce":false}})); + for input in [ + json!({"op":"begin", "key":"same"}), + json!({"op":"begin", "key":"same"}), + json!({"op":"resolve", "loader":1, "value":2}), + json!({"op":"resolve", "loader":0, "value":1}), + ] { + driver.apply(&input).unwrap(); + } + let writes: Vec<_> = driver + .causal_history() + .into_iter() + .filter_map(|event| match event { + CausalEvent::WriteDispatch { source, owner, .. } => Some((source, owner)), + _ => None, + }) + .collect(); + assert_eq!(writes, [(Some(1), Some(1)), (Some(0), Some(0))]); + driver.close(); +} + +fn prepared_control() -> Prepared { + Prepared { + session: "1".to_string(), + steps: 2, + observation: "coreObservation".to_string(), + receipt: None, + fixture: Value::Object(Map::new()), + setup: Vec::new(), + actions: vec!["init".to_string(), "outsideCall".to_string()], + } +} + +struct TempDir(PathBuf); + +impl TempDir { + fn new(name: &str) -> TempDir { + let path = std::env::temp_dir().join(format!( + "dialcache-rust-harness-{}-{name}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).expect("temp dir"); + TempDir(path) + } + + fn write(&self, relative: &str, content: &str) { + let path = self.0.join(relative); + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(&path, content).expect("write"); + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[test] +fn coordinator_answers_repeated_profile_requests() { + let mut coordinator = Coordinator::spawn().expect("shared replay requires Node 24 on PATH"); + let mut profiles = Map::new(); + for _ in 0..20 { + let result = coordinator.call(request("profiles")).expect("profiles"); + assert_eq!( + result.get("settlement").and_then(Value::as_str), + Some(SETTLEMENT) + ); + profiles = result + .get("profiles") + .and_then(Value::as_object) + .cloned() + .expect("profiles object"); + } + assert_eq!(profiles.len(), 17, "profiles: {}", sorted_keys(&profiles)); + for name in [ + "core", + "effects", + "local-clock", + "scope", + "policy", + "shadow", + "recovery", + ] { + assert!(profiles.contains_key(name), "missing profile {name}"); + } + for (name, actions) in &profiles { + let actions = actions + .as_array() + .unwrap_or_else(|| panic!("{name} actions must be a list")); + assert!(!actions.is_empty(), "{name} lists no actions"); + assert!( + actions + .iter() + .all(|action| action.as_str().is_some_and(|action| !action.is_empty())), + "{name} has a malformed action" + ); + } + coordinator.finish().expect("coordinator exits cleanly"); +} + +#[test] +fn coordinator_rejects_broken_responses() { + let cases = [ + ("malformed JSON", "not-json"), + ( + "unknown version", + r#"{"version":2,"id":1,"ok":true,"result":{}}"#, + ), + ( + "wrong sequence", + r#"{"version":1,"id":2,"ok":true,"result":{}}"#, + ), + ( + "unknown member", + r#"{"version":1,"id":1,"ok":true,"result":{},"extra":true}"#, + ), + ( + "duplicate member", + r#"{"version":1,"id":1,"ok":true,"result":{},"result":{}}"#, + ), + ( + "missing status", + r#"{"version":1,"id":1,"error":"missing status"}"#, + ), + ( + "mixed success", + r#"{"version":1,"id":1,"ok":true,"result":{},"error":"hidden"}"#, + ), + ]; + for (name, response) in cases { + let encoded = serde_json::to_string(&format!("{response}\n")).expect("encode"); + let program = format!("process.stdin.once('data', () => {{ process.stdout.write({encoded}); process.stdin.resume(); }});"); + let mut coordinator = + Coordinator::start(node(&program), Duration::from_secs(1), false).expect("start"); + assert!( + coordinator.call(request("profiles")).is_err(), + "{name}: accepted malformed coordinator response" + ); + coordinator + .finish() + .unwrap_or_else(|error| panic!("{name}: {error}")); + } +} + +#[test] +fn coordinator_times_out_and_closes_broken_process() { + // Both a blocked RPC and a child ignoring EOF must be bounded by real time. + let started = Instant::now(); + let mut coordinator = Coordinator::start( + node("process.stdin.resume(); setInterval(() => {}, 1000);"), + Duration::from_millis(100), + true, + ) + .expect("start"); + let error = coordinator + .call(request("profiles")) + .expect_err("accepted a process that never acknowledged the request"); + assert!( + error.contains("coordinator unavailable or exceeded real-time request limit"), + "{error}" + ); + coordinator + .finish() + .expect("expected failure is not reported"); + assert!( + started.elapsed() < Duration::from_secs(10), + "watchdog did not bound the broken process" + ); +} + +#[test] +fn coordinator_rejects_premature_completion_and_empty_commands() { + let cases = [ + ("premature completion", r#"{"complete":true,"steps":2}"#), + ( + "empty commands", + r#"{"complete":false,"index":1,"inputs":[]}"#, + ), + ( + "fractional index", + r#"{"complete":false,"index":1.5,"inputs":[{"op":"begin"}]}"#, + ), + ( + "non-command input", + r#"{"complete":false,"index":1,"inputs":[null]}"#, + ), + ( + "missing argument", + r#"{"complete":false,"index":1,"inputs":[{"op":"resolve"}]}"#, + ), + ( + "wrong argument type", + r#"{"complete":false,"index":1,"inputs":[{"op":"resolve","loader":"0"}]}"#, + ), + ( + "unexpected argument", + r#"{"complete":false,"index":1,"inputs":[{"op":"begin","expected":0}]}"#, + ), + ( + "unknown operation", + r#"{"complete":false,"index":1,"inputs":[{"op":"invented"}]}"#, + ), + ]; + for (name, result) in cases { + let program = format!( + "require('readline').createInterface({{input:process.stdin}}).on('line', line => {{ const request=JSON.parse(line); process.stdout.write(JSON.stringify({{version:1,id:request.id,ok:true,result:{result}}})+'\\n'); }});" + ); + let mut coordinator = + Coordinator::start(node(&program), Duration::from_secs(1), false).expect("start"); + let mut applied = 0; + // A well-formed core observation passes the local shape check, so the + // coordinator's malformed reply is what execution must reject. + let error = coordinator.execute( + &prepared_control(), + &mut |_input| { + applied += 1; + Ok(()) + }, + &mut healthy_core_observation, + &mut || 0, + None, + &mut [], + ); + assert!( + error.is_err() && applied == 0, + "{name}: malformed reply advanced execution: {error:?}, commands={applied}" + ); + coordinator + .finish() + .unwrap_or_else(|error| panic!("{name}: {error}")); + } +} + +#[test] +fn transport_validates_observations_locally() { + let schema = Schema::load().expect("schema"); + let defs = schema.defs(); + let behavior = empty_behavior_observation(true); + let local = empty_behavior_observation(false); + for (definition, observed) in [ + ("behaviorObservation", &behavior), + ("coreObservation", &healthy_core_observation()), + ("localClockObservation", &local), + ] { + observation_error(observed, definition, defs) + .unwrap_or_else(|error| panic!("well-formed {definition} rejected: {error}")); + } + let core = healthy_core_observation(); + let controls: Vec<(&str, &str, Value)> = vec![ + ( + "string counter", + "behaviorObservation", + with(&behavior, "loaders", json!("1")), + ), + ( + "pending call value", + "behaviorObservation", + with(&behavior, "calls", json!([{"status": "value"}])), + ), + ( + "invented event", + "behaviorObservation", + with(&behavior, "events", json!([{"event": "invented"}])), + ), + ( + "unknown field", + "behaviorObservation", + with(&behavior, "extra", json!(1)), + ), + ( + "missing list", + "behaviorObservation", + without(&behavior, "maintenance"), + ), + ( + "negative counter", + "coreObservation", + with(&core, "redisReads", json!(-1)), + ), + ( + "missing counter", + "coreObservation", + without(&core, "redisWrites"), + ), + ( + "fractional counter", + "coreObservation", + with(&core, "redisReads", json!(0.5)), + ), + ( + "structured local call", + "localClockObservation", + with(&local, "calls", json!([{"status": "pending"}])), + ), + ( + "events on local clock", + "localClockObservation", + with(&local, "events", json!([])), + ), + ("non-object observation", "behaviorObservation", json!([])), + ("nil observation", "coreObservation", Value::Null), + ]; + for (name, definition, observed) in &controls { + let error = observation_error(observed, definition, defs) + .err() + .unwrap_or_else(|| panic!("{name}: malformed {definition} accepted: {observed}")); + assert!( + error.starts_with(&format!( + "driver produced a malformed {definition} observation: " + )), + "{name}: shape defect lacks the driver attribution: {error}" + ); + assert!( + !carries_comparison_markers(&error), + "{name}: shape defect acquired comparison markers: {error}" + ); + assert!( + !error.contains("\"1\"") && !error.contains("invented"), + "{name}: diagnostic leaked a value: {error}" + ); + } + for definition in ["", "invented", "observedEvent", "command"] { + let error = observation_error(&behavior, definition, defs) + .err() + .unwrap_or_else(|| panic!("definition {definition:?} accepted")); + assert!( + error.contains("unknown replay observation definition"), + "{definition:?}: {error}" + ); + } + + // Rejected before any request reaches the coordinator: the stand-in exits + // nonzero if an observe request ever arrives, which finish() reports. + let program = "require('readline').createInterface({input:process.stdin}).on('line', line => { const request=JSON.parse(line); if (request.op === 'observe') process.exit(3); process.stdout.write(JSON.stringify({version:1,id:request.id,ok:true,result:{complete:true,steps:2}})+'\\n'); });"; + let mut coordinator = + Coordinator::start(node(program), Duration::from_secs(1), false).expect("start"); + let mut applied = 0; + let mut prepared = prepared_control(); + prepared.setup = vec![json!({"op": "bumpSource"})]; + let error = coordinator + .execute( + &prepared, + &mut |_input| { + applied += 1; + Ok(()) + }, + &mut || with(&healthy_core_observation(), "redisReads", json!("many")), + &mut || 0, + None, + &mut [], + ) + .expect_err("malformed observation accepted"); + assert!( + error.contains("driver produced a malformed coreObservation observation"), + "not attributed to the driver: {error}" + ); + assert_eq!(applied, 1, "setup must run before the first observation"); + coordinator + .finish() + .expect("an observe request reached the coordinator"); + + // prepare rejects an unknown observation definition. + let program = "require('readline').createInterface({input:process.stdin}).on('line', line => { const request=JSON.parse(line); process.stdout.write(JSON.stringify({version:1,id:request.id,ok:true,result:{session:'1',settlement:'causally-ready-v1',observation:'observedEvent',receipt:null,fixture:{},setup:[],actions:['init','outsideCall'],steps:2}})+'\\n'); });"; + let mut coordinator = + Coordinator::start(node(program), Duration::from_secs(1), false).expect("start"); + let error = coordinator + .prepare("core", Path::new("control.itf.json"), None) + .expect_err("unknown observation definition accepted"); + assert!( + error.contains("malformed replay observation definition"), + "{error}" + ); + coordinator.finish().expect("clean exit"); +} + +fn carries_comparison_markers(text: &str) -> bool { + text.find("expected:") + .is_some_and(|at| text[at..].contains("actual:")) +} + +#[test] +fn transport_validates_settlement_receipts_before_sending() { + let receipt = json!({"elapsedMs": 0, "runnable": 0, "held": { + "loaders": 0, "reads": 0, "writes": 0, "dumps": 0, + "loads": 0, "policies": 0, "scopes": 0 + }}); + let schema = Schema::load().expect("schema"); + assert!(schema.matches_definition(&receipt, "settlementReceipt")); + let mut bad_held = receipt.clone(); + bad_held["held"]["reads"] = json!(-1); + let malformed = [ + without(&receipt, "held"), + with(&receipt, "runnable", json!(0.5)), + with(&receipt, "elapsedMs", json!("0")), + with(&receipt, "extra", json!(0)), + bad_held, + ]; + // The process rejects any observe request; a malformed receipt must be + // attributed to the driver before it spends a coordinator round trip. + let program = "require('readline').createInterface({input:process.stdin}).on('line', line => { const r=JSON.parse(line); if (r.op === 'observe') process.exit(3); process.stdout.write(JSON.stringify({version:1,id:r.id,ok:true,result:{discarded:true}})+'\\n'); });"; + let mut coordinator = + Coordinator::start(node(program), Duration::from_secs(1), false).expect("start"); + let mut prepared = prepared_control(); + prepared.observation = "behaviorObservation".to_string(); + prepared.receipt = Some("settlementReceipt".to_string()); + for malformed in malformed { + let error = coordinator + .execute( + &prepared, + &mut |_| Ok(()), + &mut || empty_behavior_observation(true), + &mut || 0, + Some(&mut || malformed.clone()), + &mut [], + ) + .expect_err("malformed receipt accepted"); + assert!( + error.contains("malformed settlementReceipt receipt"), + "{error}" + ); + assert!(!carries_comparison_markers(&error), "{error}"); + } + let missing = coordinator + .execute( + &prepared, + &mut |_| Ok(()), + &mut || empty_behavior_observation(true), + &mut || 0, + None, + &mut [], + ) + .expect_err("missing receipt callback accepted"); + assert!( + missing.contains("omitted its settlement receipt"), + "{missing}" + ); + coordinator + .finish() + .expect("an observe request reached the coordinator"); + + // prepare must reject invalid receipt definitions and prevent a core + // session from attaching a behavior driver's receipt. + for (observation, value) in [ + ("behaviorObservation", Value::Null), + ("behaviorObservation", json!("coreObservation")), + ("coreObservation", json!("settlementReceipt")), + ] { + let result = json!({"session":"1", "settlement":SETTLEMENT, + "observation":observation, "receipt":value, "fixture":{}, + "setup":[], "actions":["init","outsideCall"], "steps":2}); + let program = format!("require('readline').createInterface({{input:process.stdin}}).on('line', line => {{ const r=JSON.parse(line); process.stdout.write(JSON.stringify({{version:1,id:r.id,ok:true,result:{result}}})+'\\n'); }});"); + let mut coordinator = + Coordinator::start(node(&program), Duration::from_secs(1), false).expect("start"); + let error = coordinator + .prepare("core", Path::new("control.itf.json"), None) + .expect_err("invalid receipt definition accepted"); + assert!( + error.contains("malformed replay receipt definition"), + "{error}" + ); + coordinator.finish().expect("clean exit"); + } +} + +#[test] +fn coordinator_prepares_and_replays_the_smoke_history_shape() { + // The real coordinator's prepare result must satisfy every structural check. + let mut coordinator = Coordinator::spawn().expect("shared replay requires Node 24 on PATH"); + let path = repo_root().join("formal/conformance-smoke.itf.json"); + let raw = std::fs::read_to_string(&path).expect("smoke history"); + let prepared = coordinator + .prepare("core", &path, Some(&raw)) + .expect("prepare"); + assert_eq!(prepared.observation, "coreObservation"); + assert_eq!(prepared.actions.len() as i64, prepared.steps); + assert_eq!(prepared.actions[0], "init"); + assert!(prepared.fixture.is_object()); + // Discarding is the early-exit path execute() takes; a stale session is an error. + let mut discard = request("discard"); + discard.insert("session".to_string(), Value::from(prepared.session.clone())); + coordinator.call(discard.clone()).expect("discard"); + let error = coordinator + .call(discard) + .expect_err("second discard accepted"); + assert!(error.starts_with("shared replay: "), "{error}"); + let error = coordinator + .prepare("core", Path::new("missing-history"), Some("{")) + .expect_err("malformed history accepted"); + assert!(error.starts_with("shared replay: "), "{error}"); + coordinator.finish().expect("clean exit"); +} + +#[test] +fn frames_are_bounded_before_buffering() { + let mut reader = + BufReader::with_capacity(16, "12345678901234567890123456789012345\n".as_bytes()); + let error = read_frame(&mut reader, 32).expect_err("oversized frame accepted"); + assert!(error.contains("oversized"), "{error}"); + let mut reader = BufReader::with_capacity(16, "{\"value\":1}".as_bytes()); + let error = read_frame(&mut reader, 32).expect_err("unterminated frame accepted"); + assert!(!error.contains("oversized"), "{error}"); + let mut reader = BufReader::with_capacity(4, "{\"value\":1}\nnext\n".as_bytes()); + assert_eq!( + read_frame(&mut reader, 32).expect("frame"), + b"{\"value\":1}\n" + ); + assert_eq!(read_frame(&mut reader, 32).expect("frame"), b"next\n"); +} + +#[test] +fn strict_json_rejects_ambiguous_inputs() { + for raw in [ + r#"{"action":1,"action":2}"#, + r#"{"action":1,"action":2}"#, + r#"[{"s":{"calls":[],"calls":[]}}]"#, + r#"{"value":NaN}"#, + r#"{"value":1} trailing"#, + "{", + "1e999", + ] { + assert!( + strict_parse(raw).is_err(), + "ambiguous/malformed JSON accepted: {raw}" + ); + } + for raw in [ + r#"{"value":"escaped \\\" delimiter } ]","nested":[null,false,1,-2.5e3,{"empty":{}}]}"#, + r#"{"a":1,"b":2}"#, + r#"[{},[],true,false,null,"",1]"#, + " {\"a\" : { \"b\" : [ 1 , 2 ] } , \"c\" : \"x\" }\n", + ] { + strict_parse(raw).unwrap_or_else(|error| panic!("valid JSON rejected: {raw}: {error}")); + } +} + +#[test] +fn json_projection_distinguishes_values() { + assert!( + json_equal( + &json!({"n": 1, "a": [false, null, ""]}), + &json!({"n": 1.0, "a": [false, null, ""]}) + ), + "equivalent JSON numbers differed" + ); + for (a, b) in [ + (json!(false), json!(0)), + (Value::Null, json!(false)), + (json!("1"), json!(1)), + (json!([]), Value::Null), + (json!({"x": null}), json!({})), + ] { + assert!( + !json_equal(&a, &b), + "distinct observations compared equal: {a} {b}" + ); + } +} + +#[test] +fn itf_integers_decode_exactly_or_fail() { + let decoded = + decode_itf(json!({"s": {"calls": [{"#bigint": "1"}, {"#bigint": "-7"}]}, "n": 2.5})) + .expect("decode"); + assert!( + json_equal(&decoded, &json!({"s": {"calls": [1, -7]}, "n": 2.5})), + "{decoded}" + ); + for (raw, message) in [ + (json!({"#bigint": "9007199254740992"}), "unsafe ITF integer"), + ( + json!({"#bigint": "-9007199254740992"}), + "unsafe ITF integer", + ), + (json!({"#bigint": "1.5"}), "unsafe ITF integer"), + (json!({"#bigint": 1}), "unsafe ITF integer"), + ( + json!({"#bigint": "1", "extra": true}), + "malformed ITF integer", + ), + (json!([9007199254740992i64]), "unsafe JSON number"), + ] { + let error = decode_itf(raw.clone()) + .err() + .unwrap_or_else(|| panic!("accepted {raw}")); + assert_eq!(error, message, "{raw}"); + } + assert!(json_equal( + &decode_itf(json!({"#bigint": "9007199254740991"})).expect("boundary"), + &json!(9007199254740991i64) + )); +} + +#[test] +fn inventory_ids_follow_the_shared_scheme() { + assert_eq!(percent_encode_component("a b"), "a%20b"); + assert_eq!( + percent_encode_component("scope: nested/close"), + "scope%3A%20nested%2Fclose" + ); + assert_eq!(percent_encode_component("café ✓"), "caf%C3%A9%20%E2%9C%93"); + assert_eq!( + percent_encode_component("A-Z_a.z!~*'()09"), + "A-Z_a.z!~*'()09" + ); + assert_eq!(percent_encode_component("#?&=+"), "%23%3F%26%3D%2B"); + assert_eq!( + scenario_case_id("stale recovery", "adapter legacy null"), + "scenario/stale%20recovery/adapter%20legacy%20null" + ); + assert_eq!( + protocol_case_id("keyVectors", "user id: 1"), + "protocol/keyVectors/user%20id%3A%201" + ); + assert_eq!(witness_case_id("effects"), "witness/effects"); + + let sampled = Path::new(".formal-traces/features/scope/trace_12.itf.json"); + assert_eq!(trace_kind(sampled), "sampled"); + assert_eq!(trace_case_id("scope", sampled, true), "sampled/scope/12"); + assert_eq!( + trace_case_id("scope", sampled, false), + "smoke/scope/trace_12.itf.json" + ); + let regression = Path::new( + ".formal-traces/features/../regressions/scope/absentValueIsMemoizedTest.itf.json", + ); + assert_eq!(trace_kind(regression), "regression"); + assert_eq!( + trace_case_id("scope", regression, true), + "regression/scope/absentValueIsMemoizedTest" + ); + assert_eq!( + trace_case_id("scope", regression, false), + "regression/scope/absentValueIsMemoizedTest" + ); + let smoke = repo_root().join("formal/scope-smoke.itf.json"); + assert_eq!( + trace_case_id("scope", &smoke, false), + "smoke/scope/scope-smoke.itf.json" + ); + assert_eq!( + trace_case_id("core", Path::new("/tmp/custom.itf.json"), true), + "smoke/core/custom.itf.json" + ); + assert_eq!( + trace_case_id("core", Path::new("/x/conformance/trace_007.itf.json"), true), + "smoke/core/trace_007.itf.json" + ); +} + +#[test] +fn missing_scheduled_regressions_fail_corpus_selection() { + let corpus = TempDir::new("effects-corpus"); + let error = + regression_paths("effects", &corpus.0).expect_err("missing scheduled histories accepted"); + assert!(error.contains("missing Quint regression"), "{error}"); + assert!(regression_paths("not-a-profile", &corpus.0) + .expect_err("unknown profile without exports accepted") + .contains("missing Quint regression")); +} + +#[test] +fn registry_checks_match_go() { + registry_check().expect("core registry"); + for (profile, _) in BEHAVIOR_PROFILE_VERSIONS { + require_behavior_profile(profile).unwrap_or_else(|error| panic!("{profile}: {error}")); + } + assert!( + require_behavior_profile("core").is_err(), + "core is not a behavior profile" + ); + let raw = std::fs::read_to_string(repo_root().join("formal/profiles.json")).expect("registry"); + for mode in ["version", "missing", "duplicate", "model", "schema"] { + let mut registry = strict_parse(&raw).expect("parse"); + let profiles = registry["profiles"].as_array().cloned().expect("profiles"); + let at = profiles + .iter() + .position(|profile| profile["id"] == "scope") + .expect("scope profile"); + let mut profile = profiles[at].clone(); + match mode { + "version" => profile["version"] = json!(999), + "model" => profile["model"] = json!("formal/unknown.qnt"), + "schema" => registry["behavioralSchemaVersion"] = json!(999), + _ => {} + } + let mut changed = profiles.clone(); + match mode { + "missing" => { + changed.remove(at); + } + "duplicate" => changed.push(profile.clone()), + _ => changed[at] = profile, + } + registry["profiles"] = Value::Array(changed); + assert!( + profile_registry_check_text(®istry.to_string(), "scope", 2).is_err(), + "{mode}: unsupported profile registry accepted" + ); + } +} + +#[test] +fn report_writes_jsonl_records() { + let directory = TempDir::new("report"); + let path = directory.0.join("rust-replay.jsonl"); + let mut report = Report::create(Some(&path)).expect("create"); + report.start().expect("start"); + report + .case("sampled/core/0", &Ok(()), 10, 20) + .expect("case"); + report + .case( + "witness/scope", + &Err("stale witness definition x".to_string()), + 30, + 40, + ) + .expect("case"); + let summary = report.finish().expect("finish"); + assert_eq!( + (summary.cases, summary.failed, summary.status()), + (2, 1, "failed") + ); + let text = std::fs::read_to_string(&path).expect("report"); + let records: Vec = text + .lines() + .map(|line| strict_parse(line).expect("record")) + .collect(); + assert_eq!(records.len(), 4); + assert_eq!(records[0]["kind"], "start"); + assert_eq!(records[0]["schemaVersion"], 1); + assert_eq!(records[0]["implementation"], "rust"); + assert!(records[0]["startedAt"] + .as_i64() + .is_some_and(|ms| ms > 1_700_000_000_000)); + assert_eq!( + records[1], + json!({"kind": "case", "id": "sampled/core/0", "status": "passed", "startedAt": 10, "finishedAt": 20}) + ); + assert_eq!( + records[2], + json!({"kind": "case", "id": "witness/scope", "status": "failed", "startedAt": 30, "finishedAt": 40, "message": "stale witness definition x"}) + ); + assert_eq!(records[3]["kind"], "finish"); + assert_eq!(records[3]["status"], "failed"); + assert_eq!(records[3]["cases"], 2); + assert_eq!(records[3]["failed"], 1); + let mut silent = Report::create(None).expect("no-op"); + silent.start().expect("start"); + assert!(silent.run_case("x", || Ok(())).is_ok()); + assert_eq!(silent.finish().expect("finish").cases, 1); +} + +#[test] +fn sha256_matches_known_vectors() { + assert_eq!( + sha256_hex(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + sha256_hex(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert_eq!( + sha256_hex(b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ); + assert_eq!( + sha256_hex(&[b'a'; 1_000_000]), + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0" + ); + // 55, 56 and 64 byte messages straddle the padding boundary. + assert_eq!( + sha256_hex(&[0u8; 55]), + "02779466cdec163811d078815c633f21901413081449002f24aa3e80f0b88ef7" + ); + assert_eq!( + sha256_hex(&[0u8; 56]), + "d4817aa5497628e7c77e6b606107042bbba3130888c5f47a375e6179be789fbb" + ); + assert_eq!( + sha256_hex(&[0u8; 64]), + "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" + ); + let schema = repo_root().join("formal/replay/protocol.schema.json"); + let expected = Command::new("shasum") + .arg("-a") + .arg("256") + .arg(&schema) + .output() + .ok() + .filter(|output| output.status.success()); + if let Some(output) = expected { + let text = String::from_utf8_lossy(&output.stdout); + assert_eq!( + text.split_whitespace().next(), + Some(file_sha256(&schema).expect("hash").as_str()) + ); + } +} + +#[test] +fn witness_evidence_binds_shared_replay_sources() { + let root = TempDir::new("witness-root"); + let inputs = [ + "formal/profiles.json", + "formal/coverage-witnesses.json", + "formal/execution.json", + "formal/dialcache-effects-conformance.qnt", + "formal/conformance-observations.qnt", + "formal/replay/coordinator.mjs", + "formal/replay/mapping.mjs", + ]; + for path in inputs { + root.write(path, "reviewed input"); + } + root.write("formal/profiles.json", r#"{"profiles":[{"id":"effects"}],"replaySources":["formal/replay/coordinator.mjs","formal/replay/mapping.mjs"]}"#); + root.write( + "formal/coverage-witnesses.json", + r#"{"effects":["observed"]}"#, + ); + root.write( + "formal/execution.json", + r#"{"models":[{"path":"formal/dialcache-effects-conformance.qnt"}]}"#, + ); + root.write("trace.itf.json", "controlled trace"); + let trace = root.0.join("trace.itf.json"); + let cite = |name: &str, kind: &str, checkpoints: Vec| Trace { + name: name.to_string(), + kind: kind.to_string(), + checkpoints, + }; + let mut evidence = Evidence { + schema_version: 2, + profile: "effects".to_string(), + traces: 1, + required: vec!["observed".to_string()], + seen: vec!["observed".to_string()], + labels: Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 1, + regression: 0, + traces: vec![cite("trace.itf.json", "sampled", vec![1])], + }, + )])), + inputs: Vec::new(), + corpus: Vec::new(), + }; + for path in inputs { + evidence.inputs.push(Digest { + path: path.to_string(), + name: String::new(), + sha256: file_sha256(&root.0.join(path)).expect("hash"), + }); + } + evidence.corpus = vec![Digest { + path: String::new(), + name: "trace.itf.json".to_string(), + sha256: file_sha256(&trace).expect("hash"), + }]; + let write = |evidence: &Evidence| { + root.write( + "effects.json", + &serde_json::to_string(evidence).expect("encode"), + ) + }; + write(&evidence); + let check = + || check_witness_evidence(&root.0, "effects", &root.0, std::slice::from_ref(&trace)); + check().expect("valid evidence rejected"); + + let expect = |patch: &dyn Fn(&mut Evidence), fragment: &str| { + let mut copied = evidence.clone(); + patch(&mut copied); + write(&copied); + let error = check() + .err() + .unwrap_or_else(|| panic!("evidence accepted; expected {fragment:?}")); + assert!(error.contains(fragment), "expected {fragment:?} in {error}"); + }; + expect(&|e| e.schema_version = 1, "unsupported"); + expect(&|e| e.labels = Some(HashMap::new()), "lacks provenance"); + expect(&|e| e.labels = None, "lacks per-label provenance"); + expect( + &|e| { + e.labels = Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 1, + regression: 0, + traces: vec![cite("other.itf.json", "sampled", vec![1])], + }, + )])) + }, + "unknown history", + ); + expect( + &|e| { + e.labels = Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 0, + regression: 1, + traces: vec![cite("trace.itf.json", "regression", vec![1])], + }, + )])) + }, + "as regression", + ); + expect( + &|e| { + e.labels = Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 2, + regression: 0, + traces: vec![cite("trace.itf.json", "sampled", vec![1])], + }, + )])) + }, + "cites 1 and 0", + ); + expect( + &|e| { + e.labels = Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 1, + regression: 0, + traces: vec![cite("trace.itf.json", "sampled", Vec::new())], + }, + )])) + }, + "without a checkpoint", + ); + expect( + &|e| e.seen = vec!["observed".to_string(), "observed".to_string()], + "duplicate witness observed", + ); + expect(&|e| e.seen = Vec::new(), "missing witness observed"); + expect( + &|e| e.required = Vec::new(), + "required witness registry differs", + ); + expect( + &|e| e.inputs.pop().map(drop).unwrap_or_default(), + "incomplete witness definition fingerprints", + ); + expect( + &|e| e.inputs[6].path = "formal/replay/other.mjs".to_string(), + "unexpected witness input formal/replay/other.mjs", + ); + expect( + &|e| e.corpus[0].sha256 = "00".to_string(), + "witness corpus differs at trace.itf.json", + ); + expect( + &|e| e.corpus[0].name = "other.itf.json".to_string(), + "witness corpus differs at other.itf.json", + ); + + // Go-encoded evidence spells empty lists as null; the reader accepts that. + root.write( + "effects.json", + &serde_json::to_string(&evidence) + .expect("encode") + .replace(r#""regression":0"#, r#""regression":0,"extra":null"#), + ); + check().expect("unknown members are ignored"); + write(&evidence); + check().expect("restored evidence rejected"); + + // Library membership is discovered from actual Quint sources, so adding + // a kernel library must invalidate the old evidence even without a + // manifest edit; updating its digest then makes subsequent drift visible. + root.write("formal/kernel/new-library.qnt", "new reviewed library"); + let error = check().expect_err("new kernel library omitted from evidence"); + assert!( + error.contains("incomplete witness definition fingerprints"), + "{error}" + ); + let mut with_library = evidence.clone(); + with_library.inputs.insert( + 5, + Digest { + path: "formal/kernel/new-library.qnt".to_string(), + name: String::new(), + sha256: file_sha256(&root.0.join("formal/kernel/new-library.qnt")).expect("hash"), + }, + ); + write(&with_library); + check().expect("new library fingerprint rejected"); + root.write("formal/kernel/new-library.qnt", "changed library"); + let error = check().expect_err("stale kernel library digest accepted"); + assert!( + error.contains("stale witness definition formal/kernel/new-library.qnt"), + "{error}" + ); + std::fs::remove_file(root.0.join("formal/kernel/new-library.qnt")).expect("remove library"); + write(&evidence); + check().expect("restored library inventory rejected"); + + root.write("formal/replay/mapping.mjs", "changed input mapping"); + let error = check().expect_err("changed shared mapping accepted"); + assert!( + error.contains("stale witness definition formal/replay/mapping.mjs"), + "{error}" + ); + root.write("formal/replay/mapping.mjs", "reviewed input"); + root.write("formal/replay/new-helper.mjs", "unregistered helper"); + let error = check().expect_err("new dependency accepted"); + assert!(error.contains("inventory differs"), "{error}"); + std::fs::remove_file(root.0.join("formal/replay/new-helper.mjs")).expect("remove"); + std::fs::remove_file(root.0.join("formal/replay/mapping.mjs")).expect("remove"); + let error = check().expect_err("missing dependency accepted"); + assert!(error.contains("inventory differs"), "{error}"); +} + +#[test] +fn observe_requests_reach_the_coordinator_comparison() { + // A driver that reports zeros diverges from the smoke history, but only + // after the coordinator accepted the observe request shape and compared the + // record: the failure is an observation mismatch, not a protocol violation. + let mut coordinator = Coordinator::spawn().expect("shared replay requires Node 24 on PATH"); + let path = repo_root().join("formal/conformance-smoke.itf.json"); + let prepared = coordinator.prepare("core", &path, None).expect("prepare"); + let mut applied = Vec::new(); + let error = coordinator + .execute( + &prepared, + &mut |input| { + applied.push(input.clone()); + Ok(()) + }, + &mut healthy_core_observation, + &mut || 1_788_868_800_000, + None, + &mut [], + ) + .expect_err("a zero observation matched the smoke history"); + assert!( + error.starts_with("shared replay: ") && error.contains("Observation mismatch"), + "{error}" + ); + assert!( + !error.contains("Malformed") && !error.contains("schema"), + "observe request was rejected structurally: {error}" + ); + assert_eq!( + applied.len(), + prepared.setup.len(), + "commands applied before the first observation mismatch" + ); + // The failed session was discarded, so the coordinator no longer knows it. + let mut observe = request("observe"); + observe.insert("session".to_string(), Value::from(prepared.session.clone())); + observe.insert("index".to_string(), Value::from(0)); + observe.insert("settlement".to_string(), Value::from(SETTLEMENT)); + observe.insert("observed".to_string(), healthy_core_observation()); + observe.insert( + "environment".to_string(), + json!({"wallMs": 1_788_868_800_000i64}), + ); + let error = coordinator + .call(observe) + .expect_err("discarded session accepted"); + assert!(error.contains("Unknown replay session"), "{error}"); + coordinator.finish().expect("clean exit"); +} + +#[test] +fn checked_in_replay_sources_match_the_registry() { + // The real registry must agree with the real formal/replay tree, or no + // witness evidence produced from this checkout could ever be accepted. + let sources = + formal::witness::shared_replay_sources(&repo_root()).expect("shared replay sources"); + assert!(sources + .iter() + .any(|path| path == "formal/replay/coordinator.mjs")); + assert!(sources + .iter() + .any(|path| path == "formal/replay/protocol.schema.json")); +} diff --git a/rust/tests/key_args.rs b/rust/tests/key_args.rs new file mode 100644 index 00000000..0ea31d4c --- /dev/null +++ b/rust/tests/key_args.rs @@ -0,0 +1,118 @@ +//! Native scalar conversion at the secondary-key argument API boundary. + +// Borrowed inputs are part of the public contract exercised below. +#![allow(clippy::needless_borrows_for_generic_args)] + +use dialcache::{normalize_args, ArgValue, Identity, KeySpec}; + +fn normalized(value: impl Into) -> String { + let key = KeySpec::new("entity").arg("value", value); + normalize_args(key.args).unwrap().remove(0).1 +} + +#[test] +fn unsigned_and_wide_integer_arguments_preserve_every_decimal_digit() { + assert_eq!(normalized(9_007_199_254_740_993_u64), "9007199254740993"); + assert_eq!(normalized(u64::MAX), "18446744073709551615"); + assert_eq!( + normalized(i128::MIN), + "-170141183460469231731687303715884105728" + ); + assert_eq!( + normalized(i128::MAX), + "170141183460469231731687303715884105727" + ); + assert_eq!( + normalized(u128::MAX), + "340282366920938463463374607431768211455" + ); + assert_eq!(normalized(usize::MAX), usize::MAX.to_string()); + assert_eq!(normalized(isize::MIN), isize::MIN.to_string()); + assert_eq!(normalized(isize::MAX), isize::MAX.to_string()); +} + +#[test] +fn every_integer_primitive_accepts_zero_and_its_extremes() { + macro_rules! check { + ($($t:ty),*) => { $( + assert_eq!(normalized(0 as $t), "0", stringify!($t)); + assert_eq!(normalized(<$t>::MIN), <$t>::MIN.to_string(), stringify!($t)); + assert_eq!(normalized(<$t>::MAX), <$t>::MAX.to_string(), stringify!($t)); + )* }; + } + check!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); +} + +#[test] +fn arguments_keep_float_boolean_string_and_optional_scalar_semantics() { + for (value, expected) in [ + (0.1_f32, "0.10000000149011612"), + (-0.0_f32, "0"), + (f32::INFINITY, "Infinity"), + (f32::NEG_INFINITY, "-Infinity"), + (f32::NAN, "NaN"), + ] { + assert_eq!(normalized(value), expected); + } + assert_eq!( + normalized(f64::from_bits(0x430c6bf526340002)), + "1000000000000000.2" + ); + assert_eq!(normalized(false), "false"); + assert_eq!(normalized(true), "true"); + assert_eq!(normalized("001e+21"), "001e+21"); + assert_eq!(normalized(String::from("001e+21")), "001e+21"); + assert_eq!(normalized(ArgValue::Null), "null"); + + let key = KeySpec::new("entity") + .arg("present", Some(u128::MAX)) + .arg("omitted", None::) + .arg("absent", ArgValue::Absent); + assert_eq!( + normalize_args(key.args).unwrap(), + vec![( + "present".into(), + "340282366920938463463374607431768211455".into() + )] + ); +} + +#[test] +fn wide_arguments_form_distinct_keys_without_float_rounding() { + let key = |value: u128| { + let specification = KeySpec::new("id").arg("version", value); + Identity::new("entity", specification.id, "lookup") + .namespace("urn") + .args(normalize_args(specification.args).unwrap()) + .keys() + .unwrap() + .logical + }; + assert_eq!( + key(u128::MAX), + "urn:entity:id?version=340282366920938463463374607431768211455#lookup" + ); + assert_eq!( + key(u128::MAX - 1), + "urn:entity:id?version=340282366920938463463374607431768211454#lookup" + ); +} + +#[test] +fn borrowed_arguments_preserve_owned_scalar_spellings() { + let text = String::from("001e+21"); + let borrowed_text = text.as_str(); + assert_eq!(normalized(&text), text); + assert_eq!(normalized(&borrowed_text), text); + assert_eq!(normalized(&u64::MAX), "18446744073709551615"); + assert_eq!(normalized(&usize::MAX), usize::MAX.to_string()); + assert_eq!( + normalized(&u128::MAX), + "340282366920938463463374607431768211455" + ); + assert_eq!(normalized(&0.1_f32), "0.10000000149011612"); + assert_eq!(normalized(&false), "false"); + assert_eq!(normalized(&Some(u64::MAX)), "18446744073709551615"); + let key = KeySpec::new("entity").arg("omitted", &None::); + assert!(normalize_args(key.args).unwrap().is_empty()); +} diff --git a/rust/tests/key_spec.rs b/rust/tests/key_spec.rs new file mode 100644 index 00000000..5cdc8a6e --- /dev/null +++ b/rust/tests/key_spec.rs @@ -0,0 +1,295 @@ +//! Entity-ID conversion shared by registered use cases, inline calls and invalidation. + +// Borrowed input support is part of the public contract exercised below. +#![allow(clippy::needless_borrows_for_generic_args)] + +use std::borrow::Cow; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, DialCache, Identity, IntoKeyId, InvalidateRequest, KeySpec, MissReason, Operation, + Policy, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; + +#[test] +fn floating_ids_use_javascript_spelling_for_owned_and_borrowed_inputs() { + for (value, expected) in [ + (-0.0, "0"), + (1e-7, "1e-7"), + (1e21, "1e+21"), + (f64::from_bits(0x430c6bf526340002), "1000000000000000.2"), + (f64::from_bits(0xc30c6bf526340002), "-1000000000000000.2"), + (f64::INFINITY, "Infinity"), + (f64::NAN, "NaN"), + ] { + assert_eq!(KeySpec::new(value).id, expected); + assert_eq!(KeySpec::new(&value).id, expected); + assert_eq!(KeySpec::from(value).id, expected); + assert_eq!(Identity::new("thing", value, "byId").id, expected); + assert_eq!(Identity::new("thing", &value, "byId").id, expected); + } +} + +#[test] +fn single_precision_ids_are_promoted_to_javascript_numbers() { + for (value, expected) in [(-0.0_f32, "0"), (0.1_f32, "0.10000000149011612")] { + assert_eq!(KeySpec::new(value).id, expected); + assert_eq!(KeySpec::new(&value).id, expected); + assert_eq!(KeySpec::from(value).id, expected); + assert_eq!(Identity::new("thing", value, "byId").id, expected); + assert_eq!(Identity::new("thing", &value, "byId").id, expected); + } +} + +#[test] +fn string_and_integer_ids_keep_their_text_and_support_references() { + let text = String::from("001e+21"); + let borrowed_text = text.as_str(); + assert_eq!(KeySpec::new(text.as_str()).id, text); + assert_eq!(KeySpec::new(&borrowed_text).id, text); + assert_eq!(KeySpec::new(&text).id, text); + assert_eq!(KeySpec::new(text.clone()).id, text); + assert_eq!(KeySpec::from(text.clone()).id, text); + assert_eq!(KeySpec::new(&42_u64).id, "42"); + assert_eq!(KeySpec::new(&i128::MIN).id, i128::MIN.to_string()); + assert_eq!(KeySpec::new(u128::MAX).id, u128::MAX.to_string()); + assert_eq!(KeySpec::from(u128::MAX).id, u128::MAX.to_string()); + + // A custom Display type retains the explicit text escape hatch. + assert_eq!( + KeySpec::new(std::net::Ipv4Addr::LOCALHOST.to_string()).id, + "127.0.0.1" + ); + assert_eq!(42_u64.into_key_id(), "42"); + assert_eq!(Identity::new("thing", text.as_str(), "byId").id, text); + assert_eq!(Identity::new("thing", &borrowed_text, "byId").id, text); + assert_eq!(Identity::new("thing", &text, "byId").id, text); + assert_eq!(Identity::new("thing", text.clone(), "byId").id, text); + assert_eq!( + Identity::new("thing", text.into_boxed_str(), "byId").id, + "001e+21" + ); + assert_eq!(Identity::new("thing", Cow::Borrowed("-0"), "byId").id, "-0"); + assert_eq!( + Identity::new("thing", Cow::<'_, str>::Owned("1e+21".to_owned()), "byId").id, + "1e+21" + ); + assert_eq!(Identity::new("thing", 'é', "byId").id, "é"); +} + +#[test] +fn direct_identities_preserve_every_primitive_integer_domain() { + macro_rules! check { + ($($kind:ty),*) => { $( + for value in [0 as $kind, <$kind>::MIN, <$kind>::MAX] { + let expected = value.to_string(); + assert_eq!(Identity::new("thing", value, "byId").id, expected); + assert_eq!(Identity::new("thing", &value, "byId").id, expected); + assert_eq!(KeySpec::new(value).id, expected); + } + )* }; + } + check!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); +} + +#[test] +fn registered_float_ids_share_the_javascript_zero_identity() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .build() + .unwrap(); + let sources = Arc::new(AtomicUsize::new(0)); + let calls = sources.clone(); + let lookup = cache + .use_case::("thing", "FloatId") + .policy(Policy::default().local_ttl_sec(60)) + .key(|id: &f64| KeySpec::new(id)) + .source(move |_, _| { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + async move { Ok(value) } + }) + .register() + .unwrap(); + let (negative, positive, inline) = executor.block_on(async move { + let request = cache.enable_guard(); + let negative = lookup.get(request.scope(), -0.0).await.unwrap(); + let positive = lookup.get(request.scope(), 0.0).await.unwrap(); + let operation = Operation::::new(Identity::new("thing", -0.0, "FloatId")) + .policy(Policy::default().local_ttl_sec(60)); + let inline = cache + .get_or_load(request.scope(), operation, |_| async { Ok(999) }) + .await + .unwrap(); + (negative, positive, inline) + }); + assert_eq!((*negative, *positive), (1, 1)); + assert!(Arc::ptr_eq(&negative, &inline)); + assert_eq!(sources.load(Ordering::SeqCst), 1); +} + +#[derive(Default)] +struct RecordingRemote { + reads: Mutex>, + invalidations: Mutex>, +} + +impl Remote for RecordingRemote { + fn read( + &self, + request: ReadRequest, + _: ReadContext, + ) -> BoxFuture<'_, Result> { + self.reads.lock().push(request); + Box::pin(async { Ok(ReadResult::miss(MissReason::ValueAbsent)) }) + } + + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(async { Ok(()) }) + } + + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.invalidations.lock().push(request); + Box::pin(async { Ok(()) }) + } +} + +fn assert_invalidation_matches_registered_id(id: T, escaped_id: &str) +where + T: IntoKeyId + Clone + Send + Sync + 'static, +{ + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(RecordingRemote::default()); + let cache = DialCache::builder() + .namespace("numeric") + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .build() + .unwrap(); + let lookup = cache + .use_case::("thing", "TrackedId") + .tracked(true) + .policy(Policy::default().remote_ttl_sec(60)) + .key(|id: &T| KeySpec::new(id)) + .source(|_, _| async { Ok(7) }) + .register() + .unwrap(); + let direct = Identity::new("thing", &id, "TrackedId") + .namespace("numeric") + .tracked(true); + let keys = direct.keys().unwrap(); + assert_eq!(direct.id, KeySpec::new(&id).id); + executor.block_on(async move { + let request = cache.enable_guard(); + assert_eq!(*lookup.get(request.scope(), id.clone()).await.unwrap(), 7); + let operation = Operation::::new(direct).policy(Policy::default().remote_ttl_sec(60)); + assert_eq!( + *cache + .get_or_load(request.scope(), operation, |_| async { Ok(8) }) + .await + .unwrap(), + 8 + ); + cache.invalidate("thing", &id, 17).await.unwrap(); + cache.invalidate("thing", id, 17).await.unwrap(); + }); + + let expected = format!("{{numeric:thing:{escaped_id}}}#watermark"); + let reads = remote.reads.lock(); + assert_eq!(reads.len(), 2); + assert_eq!(reads[0], reads[1]); + assert_eq!(reads[0].value_key, keys.value); + assert_eq!(reads[0].watermark_key, keys.watermark); + assert_eq!(reads[0].watermark_key.as_deref(), Some(expected.as_str())); + let invalidations = remote.invalidations.lock(); + assert_eq!(invalidations.len(), 2); + for request in invalidations.iter() { + assert_eq!(request.watermark_key, expected); + assert_eq!(request.invalidated_at_ms, WALL_EPOCH_MS as u64); + assert_eq!(request.future_buffer_ms, 17); + } +} + +#[test] +fn numeric_invalidation_uses_the_registered_entity_watermark() { + for (id, escaped_id) in [ + (-0.0, "0"), + (1e-7, "1e-7"), + (1e21, "1e%2B21"), + (f64::from_bits(0x430c6bf526340002), "1000000000000000.2"), + (f64::INFINITY, "Infinity"), + (f64::NAN, "NaN"), + ] { + assert_invalidation_matches_registered_id(id, escaped_id); + } + assert_invalidation_matches_registered_id(0.1_f32, "0.10000000149011612"); + assert_invalidation_matches_registered_id(u64::MAX, "18446744073709551615"); + assert_invalidation_matches_registered_id( + i128::MIN, + "-170141183460469231731687303715884105728", + ); + assert_invalidation_matches_registered_id(u128::MAX, "340282366920938463463374607431768211455"); +} + +#[test] +fn string_invalidation_preserves_explicit_entity_text() { + assert_invalidation_matches_registered_id("001e+21", "001e%2B21"); + assert_invalidation_matches_registered_id(String::from("-0"), "-0"); +} + +#[test] +fn identity_invalidation_matches_explicit_and_inherited_namespaces_and_all_variants() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(RecordingRemote::default()); + let cache = DialCache::builder() + .namespace("tenant-a") + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .build() + .unwrap(); + executor.block_on(async move { + let request = cache.enable_guard(); + for namespace in ["", "tenant-b"] { + for use_case in ["summary", "details"] { + let mut identity = Identity::new("thing", 42, use_case) + .namespace(namespace) + .tracked(true); + identity.args = vec![("variant".to_owned(), use_case.to_owned())]; + cache + .get_or_load( + request.scope(), + Operation::::new(identity.clone()) + .policy(Policy::default().remote_ttl_sec(60)), + |_| async { Ok(7) }, + ) + .await + .unwrap(); + cache.invalidate_identity(identity, 17).await.unwrap(); + } + } + }); + let reads = remote.reads.lock(); + let invalidations = remote.invalidations.lock(); + assert_eq!(reads.len(), 4); + assert_eq!(invalidations.len(), 4); + for (i, (read, invalidation)) in reads.iter().zip(invalidations.iter()).enumerate() { + assert_eq!( + read.watermark_key.as_ref(), + Some(&invalidation.watermark_key) + ); + let namespace = if i < 2 { "tenant-a" } else { "tenant-b" }; + assert_eq!( + invalidation.watermark_key, + format!("{{{namespace}:thing:42}}#watermark") + ); + assert_eq!(invalidation.future_buffer_ms, 17); + } + assert_ne!(reads[0].value_key, reads[1].value_key); +} diff --git a/rust/tests/local_storage.rs b/rust/tests/local_storage.rs new file mode 100644 index 00000000..f7ab41b8 --- /dev/null +++ b/rust/tests/local_storage.rs @@ -0,0 +1,73 @@ +//! Native allocation and eviction guarantees of the default local store. + +use std::num::NonZeroUsize; +use std::sync::Arc; + +use dialcache::limits::MAX_SAFE_INTEGER; +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + DialCache, Identity, LocalEntry, LocalRead, LocalStore, LruLocalStore, Operation, Policy, +}; + +#[test] +fn maximum_supported_capacity_allocates_only_as_entries_arrive() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let capacity = usize::try_from(MAX_SAFE_INTEGER).unwrap_or(usize::MAX); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .local_capacity(capacity) + .build() + .expect("the maximum supported capacity must remain constructible"); + let (first, second) = executor.block_on(async move { + let request = cache.enable_guard(); + let operation = Operation::::new(Identity::new("thing", "one", "LargeCapacity")) + .policy(Policy::default().local_ttl_sec(60)); + let first = cache + .get_or_load(request.scope(), operation.clone(), |_| async { Ok(7) }) + .await + .expect("source success"); + let second = cache + .get_or_load(request.scope(), operation, |_| async { Ok(8) }) + .await + .expect("local hit"); + (first, second) + }); + assert_eq!((*first, *second), (7, 7)); +} + +fn entry(value: u64) -> LocalEntry { + LocalEntry { + value: Arc::new(value), + inserted_ms: 0, + ttl_ms: 1_000, + } +} + +#[test] +fn sparse_storage_preserves_capacity_and_lru_eviction() { + let mut store = LruLocalStore::new(NonZeroUsize::new(2).unwrap()); + assert!(store.put("a".into(), entry(1)).unwrap().is_none()); + assert!(store.put("b".into(), entry(2)).unwrap().is_none()); + assert!(matches!(store.get("a", 0).unwrap(), LocalRead::Live(_))); + + let evicted = store + .put("c".into(), entry(3)) + .unwrap() + .expect("at capacity"); + assert_eq!(evicted.value.downcast_ref::(), Some(&2)); + assert_eq!(store.len(), 2); + assert!(matches!(store.get("b", 0).unwrap(), LocalRead::Absent)); + + let replaced = store + .put("a".into(), entry(4)) + .unwrap() + .expect("replacement"); + assert_eq!(replaced.value.downcast_ref::(), Some(&1)); + let evicted = store + .put("d".into(), entry(5)) + .unwrap() + .expect("at capacity"); + assert_eq!(evicted.value.downcast_ref::(), Some(&3)); + assert_eq!(store.len(), 2); +} diff --git a/rust/tests/metrics_exporters.rs b/rust/tests/metrics_exporters.rs new file mode 100644 index 00000000..392cc0e9 --- /dev/null +++ b/rust/tests/metrics_exporters.rs @@ -0,0 +1,818 @@ +//! The metric exporters' wire contract: Datadog names, units and tags; +//! Prometheus schemas, reuse and conflict isolation; and observer failure +//! isolation from cache results. + +use std::sync::Arc; + +use dialcache::datadog::{metric_suffix, DatadogError}; +use dialcache::observe::{ + CoalescingScope, CompressionOperation, CompressionOutcome, DisabledReason, ErrorKind, Labels, + Layer, MissReason, OutcomeLabels, RecoveryOutcome, SerializationOperation, ShadowOutcome, +}; +use dialcache::{ + DatadogObserver, DatadogOptions, DogStatsdClient, Event, MetricKind, ObservationMetricType, + Observer, +}; +use parking_lot::Mutex; + +fn base() -> Labels { + Labels { + namespace: Arc::from("logical"), + use_case: Arc::from("lookup"), + key_type: Arc::from("item"), + layer: Layer::Remote, + } +} + +fn outcome() -> OutcomeLabels { + OutcomeLabels { + namespace: Arc::from("logical"), + use_case: Arc::from("lookup"), + key_type: Arc::from("item"), + } +} + +/// One event per kind, mirroring Go's `metricTestEvent`: 0.25 s timers and +/// ages, 123-byte sizes, a 0.25 ratio. +fn metric_test_event(kind: MetricKind) -> Event { + match kind { + MetricKind::Request => Event::Request { labels: base() }, + MetricKind::Miss => Event::Miss { + labels: base(), + reason: MissReason::Expired, + }, + MetricKind::Disabled => Event::Disabled { + labels: base(), + reason: DisabledReason::RampedDown, + }, + MetricKind::Error => Event::Error { + labels: base(), + error: ErrorKind::Fallback, + in_fallback: true, + }, + MetricKind::Invalidation => Event::Invalidation { + namespace: Arc::from("logical"), + key_type: Arc::from("item"), + layer: Layer::Remote, + }, + MetricKind::Coalesced => Event::Coalesced { + labels: outcome(), + scope: CoalescingScope::Process, + }, + MetricKind::ShadowValidation => Event::ShadowValidation { + labels: outcome(), + outcome: ShadowOutcome::Mismatch, + }, + MetricKind::ShadowValueAge => Event::ShadowValueAge { + labels: outcome(), + outcome: ShadowOutcome::Mismatch, + seconds: 0.25, + }, + MetricKind::FutureTimestampOffset => Event::FutureTimestampOffset { + labels: base(), + seconds: 0.25, + }, + MetricKind::StaleRecovery => Event::StaleRecovery { + labels: outcome(), + outcome: RecoveryOutcome::Served, + }, + MetricKind::StaleRecoveryValueAge => Event::StaleRecoveryValueAge { + labels: outcome(), + outcome: RecoveryOutcome::Served, + seconds: 0.25, + }, + MetricKind::Compression => Event::Compression { + labels: base(), + outcome: CompressionOutcome::Compressed, + }, + MetricKind::Get => Event::Get { + labels: base(), + seconds: 0.25, + }, + MetricKind::Fallback => Event::Fallback { + labels: base(), + seconds: 0.25, + }, + MetricKind::Serialization => Event::Serialization { + labels: base(), + operation: SerializationOperation::Dump, + seconds: 0.25, + }, + MetricKind::Size => Event::Size { + labels: base(), + bytes: 123, + }, + MetricKind::StoredSize => Event::StoredSize { + labels: base(), + bytes: 123, + }, + MetricKind::CompressionRatio => Event::CompressionRatio { + labels: base(), + ratio: 0.25, + }, + MetricKind::CompressionDuration => Event::CompressionDuration { + labels: base(), + operation: CompressionOperation::Compress, + seconds: 0.25, + }, + } +} + +fn tags(items: &[(&str, &str)]) -> Vec<(String, String)> { + items + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[derive(Debug, Clone, PartialEq)] +struct Recorded { + method: &'static str, + name: String, + value: f64, + tags: Vec<(String, String)>, +} + +#[derive(Default)] +struct RecordingClient { + events: Arc>>, +} + +impl RecordingClient { + fn record(&self, method: &'static str, name: &str, value: f64, tags: &[(String, String)]) { + self.events.lock().push(Recorded { + method, + name: name.to_string(), + value, + tags: tags.to_vec(), + }); + } +} + +impl DogStatsdClient for RecordingClient { + fn increment(&self, name: &str, value: f64, tags: &[(String, String)]) { + self.record("increment", name, value, tags); + } + fn histogram(&self, name: &str, value: f64, tags: &[(String, String)]) { + self.record("histogram", name, value, tags); + } + fn distribution(&self, name: &str, value: f64, tags: &[(String, String)]) { + self.record("distribution", name, value, tags); + } +} + +fn recording() -> (RecordingClient, Arc>>) { + let client = RecordingClient::default(); + let events = client.events.clone(); + (client, events) +} + +/// Port of TestDatadogMetricNamesUnitsAndLabels. +#[test] +fn datadog_metric_names_units_and_labels() { + let (client, events) = recording(); + let observer = DatadogObserver::new( + client, + DatadogOptions::new(ObservationMetricType::Distribution).namespace("app.cache"), + ) + .expect("valid options"); + assert!(observer.observes_shadow_outcomes()); + + // (kind, suffix, method, value), copied from go/metrics_test.go. + let cases: [(MetricKind, &str, &str, f64); 19] = [ + (MetricKind::Request, "request.count", "increment", 1.0), + (MetricKind::Miss, "miss.count", "increment", 1.0), + (MetricKind::Disabled, "disabled.count", "increment", 1.0), + (MetricKind::Error, "error.count", "increment", 1.0), + ( + MetricKind::Invalidation, + "invalidation.count", + "increment", + 1.0, + ), + (MetricKind::Coalesced, "coalesced.count", "increment", 1.0), + ( + MetricKind::ShadowValidation, + "shadow.count", + "increment", + 1.0, + ), + ( + MetricKind::ShadowValueAge, + "shadow.value_age", + "distribution", + 0.25, + ), + ( + MetricKind::FutureTimestampOffset, + "future_timestamp_offset", + "distribution", + 0.25, + ), + ( + MetricKind::StaleRecovery, + "stale_recovery.count", + "increment", + 1.0, + ), + ( + MetricKind::StaleRecoveryValueAge, + "stale_recovery.value_age", + "distribution", + 0.25, + ), + ( + MetricKind::Compression, + "compression.count", + "increment", + 1.0, + ), + (MetricKind::Get, "get.duration", "distribution", 0.25), + ( + MetricKind::Fallback, + "fallback.duration", + "distribution", + 0.25, + ), + ( + MetricKind::Serialization, + "serialization.duration", + "distribution", + 0.25, + ), + ( + MetricKind::Size, + "serialization.size", + "distribution", + 123.0, + ), + (MetricKind::StoredSize, "stored.size", "distribution", 123.0), + ( + MetricKind::CompressionRatio, + "compression.ratio", + "distribution", + 0.25, + ), + ( + MetricKind::CompressionDuration, + "compression.duration", + "distribution", + 0.25, + ), + ]; + for (kind, suffix, method, value) in cases { + observer.observe(&metric_test_event(kind)); + let got = events.lock().last().cloned().expect("recorded"); + assert_eq!(got.name, format!("app.cache.{suffix}"), "{kind:?}"); + assert_eq!(got.method, method, "{kind:?}"); + assert_eq!(got.value, value, "{kind:?}"); + assert_eq!(metric_suffix(kind), suffix); + assert_eq!(observer.metric_name(kind), got.name); + // Every tag is one of the kind's declared labels: no logical key or + // identity ever reaches a metric. + let names: Vec<&str> = got.tags.iter().map(|(name, _)| name.as_str()).collect(); + assert_eq!(names, kind.label_names(), "{kind:?}"); + } + let recorded = events.lock().clone(); + assert_eq!( + recorded[3].tags, + tags(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("layer", "remote"), + ("error", "fallback"), + ("in_fallback", "true"), + ]), + "error tags" + ); + assert_eq!( + recorded[4].tags, + tags(&[ + ("cache_namespace", "logical"), + ("key_type", "item"), + ("layer", "remote"), + ]), + "invalidation acquired use_case" + ); + assert_eq!( + recorded[5].tags, + tags(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("scope", "process"), + ]), + "coalescing labels changed" + ); + assert!( + !recorded[6].tags.iter().any(|(name, _)| name == "layer"), + "shadow acquired layer" + ); + assert_eq!( + recorded[6] + .tags + .last() + .map(|(k, v)| (k.as_str(), v.as_str())), + Some(("outcome", "mismatch")) + ); + + let (client, events) = recording(); + let histogram = DatadogObserver::new( + client, + DatadogOptions::new(ObservationMetricType::Histogram), + ) + .expect("default namespace"); + histogram.observe(&metric_test_event(MetricKind::Get)); + let got = events.lock().last().cloned().expect("recorded"); + assert_eq!(got.method, "histogram", "histogram option ignored"); + assert_eq!(got.name, "dialcache.get.duration", "default namespace"); +} + +#[test] +fn datadog_rejects_invalid_namespaces_and_long_names() { + for namespace in ["1bad", "has-dash", "two..dots", ".lead", "trail."] { + let error = DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(namespace), + ) + .err() + .unwrap_or_else(|| panic!("accepted namespace {namespace:?}")); + assert_eq!(error, DatadogError::InvalidNamespace(namespace.to_string())); + } + let explicitly_empty = DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(""), + ); + assert_eq!( + explicitly_empty.err(), + Some(DatadogError::InvalidNamespace(String::new())), + "accepted explicitly empty namespace" + ); + // The longest suffix, "stale_recovery.value_age", is 24 characters; with + // the dot a 175-character namespace reaches exactly 200 and 176 exceeds it. + let longest = "stale_recovery.value_age"; + assert_eq!(longest.len(), 24); + let fits = "a".repeat(175); + DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(fits.clone()), + ) + .expect("200-character name fits"); + let overflow = "a".repeat(176); + let error = DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(overflow.clone()), + ) + .expect_err("201-character name rejected"); + assert_eq!( + error, + DatadogError::MetricNameTooLong(format!("{overflow}.{longest}")) + ); + let huge = "a".repeat(201); + assert!(DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(huge), + ) + .is_err()); +} + +#[cfg(feature = "prometheus")] +mod prometheus_exporter { + use super::*; + use dialcache::prometheus::{schemas, CollectorSchema}; + use dialcache::{PrometheusError, PrometheusObserver}; + use prometheus::proto::MetricType; + use prometheus::{Gauge, IntCounterVec, Opts, Registry}; + + struct Expected { + kind: &'static str, + name: &'static str, + help: &'static str, + labels: &'static [&'static str], + buckets: &'static [f64], + } + + const TIMER: &[f64] = &[ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + const SIZE: &[f64] = &[100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 10000000.0]; + const AGE: &[f64] = &[ + 1.0, 5.0, 15.0, 60.0, 300.0, 900.0, 3600.0, 10800.0, 43200.0, 86400.0, 259200.0, 604800.0, + ]; + const LAYER: &[&str] = &["cache_namespace", "use_case", "key_type", "layer"]; + const OUTCOME: &[&str] = &["cache_namespace", "use_case", "key_type", "outcome"]; + + /// Copied from go/metrics_prometheus.go PrometheusCollectorSchemas, in order. + const EXPECTED: [Expected; 19] = [ + Expected { + kind: "disabled", + name: "dialcache_disabled_counter", + help: "Requests where DialCache skipped a cache layer.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "reason"], + buckets: &[], + }, + Expected { + kind: "miss", + name: "dialcache_miss_counter", + help: "DialCache cache misses.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "reason"], + buckets: &[], + }, + Expected { + kind: "request", + name: "dialcache_request_counter", + help: "Total DialCache cache-layer requests.", + labels: LAYER, + buckets: &[], + }, + Expected { + kind: "error", + name: "dialcache_error_counter", + help: "Errors during DialCache cache operations or fallback execution.", + labels: &[ + "cache_namespace", + "use_case", + "key_type", + "layer", + "error", + "in_fallback", + ], + buckets: &[], + }, + Expected { + kind: "invalidation", + name: "dialcache_invalidation_counter", + help: "DialCache invalidation calls by key type and layer.", + labels: &["cache_namespace", "key_type", "layer"], + buckets: &[], + }, + Expected { + kind: "coalesced", + name: "dialcache_coalesced_counter", + help: "DialCache requests coalesced onto in-flight work by sharing scope.", + labels: &["cache_namespace", "use_case", "key_type", "scope"], + buckets: &[], + }, + Expected { + kind: "shadowValidation", + name: "dialcache_shadow_validation_counter", + help: "Sampled DialCache Redis shadow-validation outcomes.", + labels: OUTCOME, + buckets: &[], + }, + Expected { + kind: "shadowValueAge", + name: "dialcache_shadow_value_age_histogram", + help: "Age in seconds of the validated Redis value at DialCache shadow verdict time.", + labels: OUTCOME, + buckets: AGE, + }, + Expected { + kind: "futureTimestampOffset", + name: "dialcache_future_timestamp_offset_histogram", + help: "Positive offset in seconds of Redis frames dated after the observing DialCache process clock.", + labels: LAYER, + buckets: &[ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 15.0, 60.0, 300.0, + 900.0, 3600.0, 10800.0, 43200.0, + ], + }, + Expected { + kind: "staleRecovery", + name: "dialcache_stale_recovery_counter", + help: "DialCache stale-on-error Redis recovery outcomes.", + labels: OUTCOME, + buckets: &[], + }, + Expected { + kind: "staleRecoveryValueAge", + name: "dialcache_stale_recovery_value_age_histogram", + help: "Age in seconds of Redis values served by DialCache stale-on-error recovery.", + labels: OUTCOME, + buckets: AGE, + }, + Expected { + kind: "compression", + name: "dialcache_compression_counter", + help: "DialCache Redis payload compression and decompression outcomes.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "outcome"], + buckets: &[], + }, + Expected { + kind: "get", + name: "dialcache_get_timer", + help: "DialCache cache get latency in seconds.", + labels: LAYER, + buckets: TIMER, + }, + Expected { + kind: "fallback", + name: "dialcache_fallback_timer", + help: "Time DialCache waited for the fallback function in seconds.", + labels: LAYER, + buckets: TIMER, + }, + Expected { + kind: "serialization", + name: "dialcache_serialization_timer", + help: "DialCache serialization latency in seconds.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "operation"], + buckets: TIMER, + }, + Expected { + kind: "size", + name: "dialcache_size_histogram", + help: "Serialized DialCache value sizes in bytes.", + labels: LAYER, + buckets: SIZE, + }, + Expected { + kind: "storedSize", + name: "dialcache_stored_size_histogram", + help: "Stored DialCache payload sizes in bytes, after compression and escaping.", + labels: LAYER, + buckets: SIZE, + }, + Expected { + kind: "compressionRatio", + name: "dialcache_compression_ratio_histogram", + help: "Compressed-to-original DialCache payload size ratio for compressed writes.", + labels: LAYER, + buckets: &[0.05, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9, 1.0], + }, + Expected { + kind: "compressionDuration", + name: "dialcache_compression_timer", + help: "DialCache payload compression and decompression latency in seconds.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "operation"], + buckets: TIMER, + }, + ]; + + /// Port of TestPrometheusWireSchemaMatchesTypeScriptBinding against the + /// Go table, which that Go test pins to src/prometheus.ts. + #[test] + fn wire_schema_matches_the_reference_table() { + for prefix in ["", "svc_"] { + let actual: Vec = schemas(prefix); + assert_eq!(actual.len(), 19); + for (schema, expected) in actual.iter().zip(EXPECTED.iter()) { + assert_eq!(schema.kind.as_str(), expected.kind); + assert_eq!(schema.name, format!("{prefix}{}", expected.name)); + assert_eq!(schema.help, expected.help, "{} help drift", expected.name); + assert_eq!( + schema.labels, expected.labels, + "{} label order drift", + expected.name + ); + assert_eq!( + schema.buckets, expected.buckets, + "{} buckets drift", + expected.name + ); + assert_eq!(schema.is_counter(), expected.buckets.is_empty()); + } + } + let registry = Registry::new(); + let observer = PrometheusObserver::new(®istry, "svc_").expect("fresh registry"); + assert!(observer.observes_shadow_outcomes()); + assert_eq!(observer.schemas(), schemas("svc_")); + } + + /// The registered collectors expose the schema on a scrape: every kind + /// produces one family with the schema's name, help, type, label names + /// and bucket bounds. + #[test] + fn scrape_exposes_names_help_labels_and_buckets() { + let registry = Registry::new(); + let observer = PrometheusObserver::new(®istry, "scrape_").expect("fresh registry"); + for kind in MetricKind::ALL { + observer.observe(&metric_test_event(kind)); + } + let families = registry.gather(); + assert_eq!(families.len(), 19); + for expected in EXPECTED.iter() { + let name = format!("scrape_{}", expected.name); + let family = families + .iter() + .find(|f| f.name() == name) + .unwrap_or_else(|| panic!("{name} not scraped")); + assert_eq!(family.help(), expected.help); + let metric = &family.get_metric()[0]; + // The exposition sorts label pairs by name; wire order is pinned + // by the schema test above. + let mut label_names: Vec<&str> = metric.get_label().iter().map(|l| l.name()).collect(); + label_names.sort_unstable(); + let mut expected_labels = expected.labels.to_vec(); + expected_labels.sort_unstable(); + assert_eq!(label_names, expected_labels, "{name}"); + if expected.buckets.is_empty() { + assert_eq!(family.type_(), MetricType::COUNTER, "{name}"); + assert_eq!(metric.get_counter().value(), 1.0, "{name}"); + } else { + assert_eq!(family.type_(), MetricType::HISTOGRAM, "{name}"); + let histogram = metric.get_histogram(); + let bounds: Vec = histogram + .get_bucket() + .iter() + .map(|b| b.upper_bound()) + .collect(); + assert_eq!(bounds, expected.buckets, "{name}"); + assert_eq!(histogram.sample_count(), 1); + } + } + } + + /// Port of TestPrometheusReuseAndConflictIsolation. Rust shares one + /// observer by cloning it rather than by re-registering the same names. + #[test] + fn reuse_and_conflict_isolation() { + let registry = Registry::new(); + let first = PrometheusObserver::new(®istry, "test_").expect("first"); + let second = first.clone(); + first.observe(&metric_test_event(MetricKind::Request)); + second.observe(&metric_test_event(MetricKind::Request)); + first.observe(&metric_test_event(MetricKind::Get)); + let (mut saw_counter, mut saw_histogram) = (false, false); + for family in registry.gather() { + match family.name() { + "test_dialcache_request_counter" => { + saw_counter = true; + assert_eq!( + family.get_metric()[0].get_counter().value(), + 2.0, + "compatible observer did not reuse counter" + ); + } + "test_dialcache_get_timer" => { + saw_histogram = true; + let histogram = family.get_metric()[0].get_histogram(); + assert_eq!(histogram.sample_count(), 1); + assert_eq!(histogram.sample_sum(), 0.25); + assert_eq!(histogram.get_bucket().len(), 12); + } + _ => {} + } + } + assert!(saw_counter && saw_histogram, "metrics not exported"); + + // Clones survive dropping the original observer. + let third = second.clone(); + drop(first); + drop(second); + third.observe(&metric_test_event(MetricKind::Request)); + let requests = registry + .gather() + .into_iter() + .find(|f| f.name() == "test_dialcache_request_counter") + .expect("request family"); + assert_eq!(requests.get_metric()[0].get_counter().value(), 3.0); + + // A second registration of the same names is a conflict that leaves + // the registry, and the shared series, untouched. + let duplicate = PrometheusObserver::new(®istry, "test_") + .expect_err("re-registered the same collectors"); + match &duplicate { + PrometheusError::Conflict { name, .. } => { + assert_eq!(name, "test_dialcache_disabled_counter") + } + other => panic!("unexpected error {other}"), + } + third.observe(&metric_test_event(MetricKind::Request)); + let requests = registry + .gather() + .into_iter() + .find(|f| f.name() == "test_dialcache_request_counter") + .expect("request family"); + assert_eq!(requests.get_metric()[0].get_counter().value(), 4.0); + + // Another prefix on the same registry is an independent group. + let other = PrometheusObserver::new(®istry, "other_").expect("other prefix"); + other.observe(&metric_test_event(MetricKind::Request)); + let names: Vec = registry + .gather() + .iter() + .map(|f| f.name().to_string()) + .collect(); + assert!(names.contains(&"other_dialcache_request_counter".to_string())); + assert!(names.contains(&"test_dialcache_request_counter".to_string())); + + let conflict = Registry::new(); + conflict + .register(Box::new( + Gauge::new("dialcache_request_counter", "incompatible").expect("gauge"), + )) + .expect("gauge registered"); + let error = + PrometheusObserver::new(&conflict, "").expect_err("accepted conflicting collector"); + match &error { + PrometheusError::Conflict { name, .. } => { + assert_eq!(name, "dialcache_request_counter") + } + other => panic!("unexpected error {other}"), + } + assert!(error.to_string().contains("unique prefix or registry")); + // Only the gauge has an observed series... + assert_eq!( + conflict.gather().len(), + 1, + "failed observer partially registered collectors" + ); + // ...and the collectors registered before the conflict were rolled + // back (the name stays bound to the DialCache schema, so only that + // schema can reuse it) + // back, so their names are free again. + let disabled = schemas("") + .into_iter() + .find(|s| s.kind == MetricKind::Disabled) + .expect("disabled schema"); + conflict + .register(Box::new( + IntCounterVec::new(Opts::new(disabled.name, disabled.help), disabled.labels) + .expect("counter"), + )) + .expect("rolled-back collector name is free"); + } +} + +mod isolation { + use super::*; + use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; + use dialcache::{DialCache, KeySpec, Policy}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct PanickingObserver { + calls: Arc, + } + + impl Observer for PanickingObserver { + fn observe(&self, _event: &Event) { + self.calls.fetch_add(1, Ordering::SeqCst); + panic!("exporter failure"); + } + fn observes_shadow_outcomes(&self) -> bool { + panic!("exporter failure"); + } + } + + /// A panicking observer never changes the result of a cached call. + #[test] + fn panicking_observer_does_not_change_cached_results() { + let calls = Arc::new(AtomicUsize::new(0)); + let mut exec = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(exec.clock.clone()) + .runtime_arc(exec.runtime.clone()) + .observer(PanickingObserver { + calls: calls.clone(), + }) + .build() + .expect("configuration"); + let source_calls = Arc::new(AtomicUsize::new(0)); + let counting = source_calls.clone(); + let lookup = cache + .use_case::("item", "lookup") + .policy(Policy::default().local_ttl_sec(60)) + .key(|id: &i64| KeySpec::new(id.to_string())) + .source(move |_scope, id: i64| { + let counting = counting.clone(); + async move { + counting.fetch_add(1, Ordering::SeqCst); + Ok(id * 10) + } + }) + .register() + .expect("use case"); + + let first = { + let cache = cache.clone(); + let lookup = lookup.clone(); + exec.block_on(async move { + cache + .enable(|scope| async move { lookup.get(&scope, 7).await }) + .await + }) + }; + assert_eq!(*first.expect("source value survives observer panic"), 70); + let second = { + let cache = cache.clone(); + let lookup = lookup.clone(); + exec.block_on(async move { + cache + .enable(|scope| async move { lookup.get(&scope, 7).await }) + .await + }) + }; + assert_eq!(*second.expect("cached value survives observer panic"), 70); + assert_eq!( + source_calls.load(Ordering::SeqCst), + 1, + "second call was served from the local layer" + ); + assert!( + calls.load(Ordering::SeqCst) >= 2, + "the observer received the request events" + ); + } +} diff --git a/rust/tests/operation_api.rs b/rust/tests/operation_api.rs new file mode 100644 index 00000000..1a2d65f2 --- /dev/null +++ b/rust/tests/operation_api.rs @@ -0,0 +1,248 @@ +//! Inline operations remain reusable when the cached value cannot be cloned. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{DialCache, Identity, Operation, Policy}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct NonCloneValue { + value: usize, +} + +#[test] +fn cloned_operation_reuses_non_clone_cached_values() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .build() + .unwrap(); + let sources = Arc::new(AtomicUsize::new(0)); + let calls = sources.clone(); + let operation = Operation::::new(Identity::new("thing", "one", "NonClone")) + .policy(Policy::default().local_ttl_sec(60)); + let (first, second) = executor.block_on(async move { + let request = cache.enable_guard(); + let load = move |_| { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + async move { Ok(NonCloneValue { value }) } + }; + let first = cache + .get_or_load(request.scope(), operation.clone(), load.clone()) + .await + .unwrap(); + let second = cache + .get_or_load(request.scope(), operation, load) + .await + .unwrap(); + (first, second) + }); + + assert_eq!(first.value, 1); + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(sources.load(Ordering::SeqCst), 1); +} + +#[test] +fn incompatible_settled_memory_values_miss_and_are_replaced() { + use dialcache::observe::Layer; + use dialcache::{Event, MissReason, Observer}; + use parking_lot::Mutex; + #[derive(Default)] + struct Events(Mutex>); + impl Observer for Events { + fn observe(&self, event: &Event) { + self.0.lock().push(event.clone()); + } + } + for policy in [ + Policy::default().request_local(true), + Policy::default().local_ttl_sec(60), + ] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .observer_arc(events.clone()) + .build() + .unwrap(); + executor.block_on(async move { + let request = cache.enable_guard(); + let id = Identity::new("thing", "one", "MixedTypes"); + cache + .get_or_load( + request.scope(), + Operation::::new(id.clone()).policy(policy.clone()), + |_| async { Ok("old".to_owned()) }, + ) + .await + .unwrap(); + let op = Operation::::new(id).policy(policy); + let value = cache + .get_or_load(request.scope(), op.clone(), |_| async { + Ok(serde_json::json!("new")) + }) + .await + .unwrap(); + assert_eq!(*value, serde_json::json!("new")); + let hit = cache + .get_or_load(request.scope(), op, |_| async { + panic!("typed entry should hit") + }) + .await + .unwrap(); + assert!(Arc::ptr_eq(&value, &hit)); + }); + assert!(events.0.lock().iter().any(|event| matches!(event, + Event::Miss { labels, reason: MissReason::Unclassified } + if matches!(labels.layer, Layer::Local | Layer::RequestLocal)))); + } +} + +#[test] +fn incompatible_memory_hit_can_decode_compatible_remote_json() { + use dialcache::{ + BoxError, Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, + Remote, WriteRequest, + }; + use futures::future::BoxFuture; + use parking_lot::Mutex; + #[derive(Default)] + struct MemoryRemote(Mutex>); + impl Remote for MemoryRemote { + fn read( + &self, + _: ReadRequest, + _: ReadContext, + ) -> BoxFuture<'_, Result> { + Box::pin(std::future::ready(Ok(self + .0 + .lock() + .clone() + .map(ReadResult::Hit) + .unwrap_or(ReadResult::miss(MissReason::ValueAbsent))))) + } + fn write(&self, value: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + *self.0.lock() = Some(value.frame); + Box::pin(std::future::ready(Ok(()))) + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(std::future::ready(Ok(()))) + } + } + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote(MemoryRemote::default()) + .build() + .unwrap(); + executor.block_on(async move { + let request = cache.enable_guard(); + let id = Identity::new("thing", "one", "CompatibleJson"); + let policy = Policy::enabled(60).request_local(true); + cache + .get_or_load( + request.scope(), + Operation::::new(id.clone()).policy(policy.clone()), + |_| async { Ok("stored".to_owned()) }, + ) + .await + .unwrap(); + let value = cache + .get_or_load( + request.scope(), + Operation::::new(id).policy(policy), + |_| async { panic!("compatible remote representation must be decoded") }, + ) + .await + .unwrap(); + assert_eq!(*value, serde_json::json!("stored")); + }); +} + +#[test] +fn registration_and_disabled_calls_do_not_evaluate_keys_or_runtime_policy() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .policy_provider(|_| async { panic!("disabled policy resolution") }) + .build() + .unwrap(); + let lookup = cache + .use_case::<(), String>("thing", "LazyRegistration") + .key(|_| panic!("disabled key construction")) + .source(|_, _| async { Ok("source".to_owned()) }) + .register() + .unwrap(); + executor.block_on(async move { + assert_eq!(*lookup.get_uncached(()).await.unwrap(), "source"); + }); +} + +#[test] +fn incompatible_coalesced_follower_errors_without_retrying_the_source() { + use futures::channel::oneshot; + use parking_lot::Mutex; + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .build() + .unwrap(); + let guard = cache.enable_guard(); + let (send, receive) = oneshot::channel(); + let receive = Arc::new(Mutex::new(Some(receive))); + let first = cache.clone(); + let scope = guard.scope().clone(); + executor.spawn(async move { + first + .get_or_load( + &scope, + Operation::::new(Identity::new("thing", "one", "MixedFlight")) + .policy(Policy::default().local_ttl_sec(60)), + move |_| { + let receive = receive.lock().take().unwrap(); + async move { + receive.await.unwrap(); + Ok("first".to_owned()) + } + }, + ) + .await + .unwrap(); + }); + executor.drain(); + let result = Arc::new(Mutex::new(None)); + let sink = result.clone(); + let scope = guard.scope().clone(); + executor.spawn(async move { + *sink.lock() = Some( + cache + .get_or_load( + &scope, + Operation::::new(Identity::new( + "thing", + "one", + "MixedFlight", + )) + .policy(Policy::default().local_ttl_sec(60)), + |_| async { panic!("follower retried source") }, + ) + .await, + ); + }); + executor.drain(); + assert!(result.lock().is_none()); + send.send(()).unwrap(); + executor.drain(); + assert!(matches!( + result.lock().take().unwrap(), + Err(dialcache::Error::Config(_)) + )); +} diff --git a/rust/tests/owned_codec.rs b/rust/tests/owned_codec.rs new file mode 100644 index 00000000..001410c9 --- /dev/null +++ b/rust/tests/owned_codec.rs @@ -0,0 +1,338 @@ +//! Shared ownership at the public codec boundary, without Clone or Serde bounds. + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use dialcache::observe::{ErrorKind, SerializationOperation}; +use dialcache::testing::{StepRuntime, TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, Codec, DialCache, Event, Frame, Identity, InvalidateRequest, KeySpec, MissReason, + Observer, Operation, Payload, Policy, ReadContext, ReadRequest, ReadResult, Remote, Runtime, + Scope, UseCase, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; + +#[derive(Debug, PartialEq)] +struct Value(u64); + +fn payload(value: &Value) -> Payload { + Payload::text(format!("custom:{}", value.0)) +} + +#[derive(Default)] +struct RecordingRemote(Mutex>); +impl Remote for RecordingRemote { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(async { Ok(ReadResult::miss(MissReason::ValueAbsent)) }) + } + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.0.lock().push(request.frame); + Box::pin(async { Ok(()) }) + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(async { Ok(()) }) + } +} + +enum Lookup { + Inline(DialCache, Box>, Arc), + Registered(UseCase<(), Value>), +} +impl Lookup { + fn new( + cache: &DialCache, + codec: Arc>, + registered: bool, + calls: Arc, + ) -> Self { + let policy = Policy::enabled(60).request_local(false); + if registered { + Self::Registered( + cache + .use_case("thing", "OwnedCodec") + .policy(policy) + .key(|_: &()| KeySpec::new("one")) + .codec(codec) + .comparator(|a, b| a == b) + .source(move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value(7)) } + }) + .register_custom() + .unwrap(), + ) + } else { + Self::Inline( + cache.clone(), + Box::new( + Operation::with_codec( + Identity::new("thing", "one", "OwnedCodec"), + codec, + |a, b| a == b, + ) + .policy(policy), + ), + calls, + ) + } + } + + async fn get(&self, scope: &Scope) -> Arc { + match self { + Self::Inline(cache, operation, calls) => { + let calls = calls.clone(); + cache + .get_or_load(scope, operation.as_ref().clone(), move |_| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value(7)) } + }) + .await + } + Self::Registered(lookup) => lookup.get(scope, ()).await, + } + .expect("cache plumbing preserves the source result") + } +} + +struct BorrowedCodec(Arc); +impl Codec for BorrowedCodec { + fn encode<'a>(&'a self, value: &'a Value) -> BoxFuture<'a, Result> { + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { Ok(payload(value)) }) + } + fn decode(&self, _: Payload) -> BoxFuture<'_, Result> { + panic!("remote always misses") + } +} + +#[test] +fn borrowed_only_codecs_keep_working_through_both_apis() { + for registered in [false, true] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(RecordingRemote::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .build() + .unwrap(); + let encoded = Arc::new(AtomicUsize::new(0)); + let lookup = Lookup::new( + &cache, + Arc::new(BorrowedCodec(encoded.clone())), + registered, + Arc::new(AtomicUsize::new(0)), + ); + executor.block_on(async move { + let request = cache.enable_guard(); + assert_eq!(*lookup.get(request.scope()).await, Value(7)); + }); + assert_eq!(encoded.load(Ordering::SeqCst), 1); + let writes = remote.0.lock(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].payload, payload(&Value(7))); + } +} + +#[cfg(feature = "tokio")] +#[tokio::test(flavor = "current_thread")] +async fn owned_encoders_move_non_clone_values_off_thread_through_both_apis() { + struct BackgroundCodec { + started: tokio::sync::mpsc::UnboundedSender<(std::thread::ThreadId, Arc)>, + release: Arc>>, + } + impl Codec for BackgroundCodec { + fn encode<'a>(&'a self, _: &'a Value) -> BoxFuture<'a, Result> { + panic!("the engine must select the owned hook") + } + fn encode_owned(&self, value: Arc) -> BoxFuture<'_, Result> { + let (started, release) = (self.started.clone(), self.release.clone()); + let job = tokio::task::spawn_blocking(move || -> Result { + started.send((std::thread::current().id(), value.clone()))?; + release.lock().recv_timeout(Duration::from_secs(5))?; + Ok(payload(&value)) + }); + Box::pin(async move { job.await? }) + } + fn decode(&self, _: Payload) -> BoxFuture<'_, Result> { + panic!("remote always misses") + } + } + + for registered in [false, true] { + let remote = Arc::new(RecordingRemote::default()); + let cache = DialCache::builder() + .remote_arc(remote.clone()) + .build() + .unwrap(); + let request = cache.enable_guard(); + let (started, mut started_rx) = tokio::sync::mpsc::unbounded_channel(); + let (release, gate) = std::sync::mpsc::channel(); + let lookup = Lookup::new( + &cache, + Arc::new(BackgroundCodec { + started, + release: Arc::new(Mutex::new(gate)), + }), + registered, + Arc::new(AtomicUsize::new(0)), + ); + let scope = request.scope().clone(); + let pending = tokio::spawn(async move { lookup.get(&scope).await }); + let (worker, observed) = tokio::time::timeout(Duration::from_secs(2), started_rx.recv()) + .await + .unwrap() + .expect("owned encoder started"); + assert_ne!(worker, std::thread::current().id()); + tokio::time::sleep(Duration::from_millis(1)).await; + assert!( + !pending.is_finished(), + "encoding remains gated while the timer progresses" + ); + release.send(()).unwrap(); + let value = pending.await.unwrap(); + assert_eq!(*value, Value(7)); + assert!(Arc::ptr_eq(&value, &observed)); + let writes = remote.0.lock(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].payload, payload(&value)); + } +} + +#[derive(Default)] +struct Events(Mutex>); +impl Observer for Events { + fn observe(&self, event: &Event) { + self.0.lock().push(event.clone()); + } +} + +struct RejectCpu { + step: Arc, + attempts: AtomicUsize, + ran: Arc, +} +impl Runtime for RejectCpu { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.step.spawn(task); + } + fn sleep(&self, delay: Duration) -> BoxFuture<'static, ()> { + self.step.sleep(delay) + } + fn spawn_blocking(&self, _: Box) -> Result<(), BoxError> { + self.attempts.fetch_add(1, Ordering::SeqCst); + Err("custom CPU admission rejected".into()) + } +} + +#[derive(Clone, Copy)] +enum Failure { + Construction, + Poll, + Error, + Rejection, +} +struct FailingCodec { + mode: Failure, + cpu: Arc, + calls: Arc, +} +impl Codec for FailingCodec { + fn encode<'a>(&'a self, _: &'a Value) -> BoxFuture<'a, Result> { + panic!("borrowed encode must not be selected") + } + fn encode_owned(&self, _: Arc) -> BoxFuture<'_, Result> { + self.calls.fetch_add(1, Ordering::SeqCst); + if matches!(self.mode, Failure::Construction) { + panic!("owned hook construction"); + } + Box::pin(async move { + match self.mode { + Failure::Construction => unreachable!(), + Failure::Poll => panic!("owned hook polling"), + Failure::Error => Err("owned hook error".into()), + Failure::Rejection => { + let ran = self.cpu.ran.clone(); + self.cpu.spawn_blocking(Box::new(move || { + ran.store(true, Ordering::SeqCst); + }))?; + Ok(Payload::text("unexpected admission")) + } + } + }) + } + fn decode(&self, _: Payload) -> BoxFuture<'_, Result> { + panic!("remote always misses") + } +} + +#[test] +fn owned_hook_failures_preserve_source_and_local_publication() { + for registered in [false, true] { + for mode in [ + Failure::Construction, + Failure::Poll, + Failure::Error, + Failure::Rejection, + ] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(RecordingRemote::default()); + let events = Arc::new(Events::default()); + let cpu = Arc::new(RejectCpu { + step: executor.runtime.clone(), + attempts: AtomicUsize::new(0), + ran: Arc::new(AtomicBool::new(false)), + }); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .observer_arc(events.clone()) + .build() + .unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let encoded = Arc::new(AtomicUsize::new(0)); + let lookup = Lookup::new( + &cache, + Arc::new(FailingCodec { + mode, + cpu: cpu.clone(), + calls: encoded.clone(), + }), + registered, + calls.clone(), + ); + executor.block_on(async move { + let request = cache.enable_guard(); + let first = lookup.get(request.scope()).await; + assert_eq!(*first, Value(7)); + assert!(Arc::ptr_eq(&first, &lookup.get(request.scope()).await)); + }); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(encoded.load(Ordering::SeqCst), 1); + assert!(remote.0.lock().is_empty()); + assert_eq!( + cpu.attempts.load(Ordering::SeqCst), + usize::from(matches!(mode, Failure::Rejection)) + ); + assert!(!cpu.ran.load(Ordering::SeqCst)); + let events = events.0.lock(); + assert!(events.iter().any(|e| matches!( + e, + Event::Error { + error: ErrorKind::SerializationDump, + .. + } + ))); + assert!(events.iter().any(|e| matches!( + e, + Event::Serialization { + operation: SerializationOperation::Dump, + .. + } + ))); + } + } +} diff --git a/rust/tests/protocol_frames.rs b/rust/tests/protocol_frames.rs new file mode 100644 index 00000000..5f913f0d --- /dev/null +++ b/rust/tests/protocol_frames.rs @@ -0,0 +1,102 @@ +//! Frame, decode, timestamp, duration and envelope conformance against the +//! portable protocol corpus (`formal/protocol-vectors.json` plus the +//! Quint-generated frame and envelope artifacts). + +#[path = "formal/digest.rs"] +mod digest; + +// TODO(integrator): switch to `formal/fixtures.rs` and delete `formal/fixtures_tmp.rs`. +#[path = "formal/fixtures.rs"] +mod fixtures; +#[path = "formal/frame_vectors.rs"] +mod frame_vectors; + +use serde_json::{json, Value}; + +type Check = fn(&Value) -> Result<(), String>; + +#[test] +fn frame_assertions_distinguish_library_rejections_from_bad_fixtures() { + let valid_shape = json!({"createdAtMs":0, "payloadType":"string", "payloadUtf8":"value", "frameHex":"expected"}); + let error = frame_vectors::check_frame_vector(&valid_shape).unwrap_err(); + assert!( + error.starts_with("PROTOCOL_ASSERTION_FAILURE expected:"), + "{error}" + ); + let mut malformed = valid_shape; + malformed["frameHex"] = json!(5); + let error = frame_vectors::check_frame_vector(&malformed).unwrap_err(); + assert!(!error.contains("PROTOCOL_ASSERTION_FAILURE"), "{error}"); +} + +/// The eight groups this test owns, with their fixed-corpus sizes. +const GROUPS: [(&str, usize, Check); 8] = [ + ("frameVectors", 8, frame_vectors::check_frame_vector), + ( + "trackedDecodeVectors", + 38, + frame_vectors::check_tracked_decode_vector, + ), + ( + "untrackedDecodeVectors", + 18, + frame_vectors::check_untracked_decode_vector, + ), + ( + "invalidTimestampVectors", + 3, + frame_vectors::check_invalid_timestamp_vector, + ), + ("durationVectors", 7, frame_vectors::check_duration_vector), + ("envelopeVectors", 7, frame_vectors::check_envelope_vector), + ( + "compressedDecodeVectors", + 12, + frame_vectors::check_compressed_decode_vector, + ), + ( + "compressionWriteVectors", + 6, + frame_vectors::check_compression_write_vector, + ), +]; + +#[test] +fn protocol_frame_groups_conform() { + let groups = fixtures::protocol_groups("all"); + let generated = fixtures::protocol_groups("generated"); + let mut failures = Vec::new(); + let mut total = 0; + let mut expected_total = 0; + for (name, fixed_count, check) in GROUPS { + let rows = groups + .get(name) + .unwrap_or_else(|| panic!("protocol corpus lacks group {name}")); + let generated_rows = generated.get(name).map_or(0, Vec::len); + expected_total += fixed_count + generated_rows; + let mut passed = 0; + for vector in rows { + total += 1; + let vector_name = vector + .get("name") + .and_then(Value::as_str) + .unwrap_or(""); + match check(vector) { + Ok(()) => passed += 1, + Err(reason) => failures.push(format!("{name} / {vector_name}: {reason}")), + } + } + println!( + "{name}: {passed}/{} ({fixed_count} fixed + {generated_rows} generated)", + rows.len() + ); + } + assert!( + failures.is_empty(), + "{} protocol vector(s) failed:\n{}", + failures.len(), + failures.join("\n") + ); + assert_eq!(total, expected_total, "review protocol vector coverage"); + println!("frame protocol groups exercised: {total} vectors"); +} diff --git a/rust/tests/protocol_keys.rs b/rust/tests/protocol_keys.rs new file mode 100644 index 00000000..5afd540e --- /dev/null +++ b/rust/tests/protocol_keys.rs @@ -0,0 +1,71 @@ +//! Replays every key, invalid-key, argument-normalization and rollout vector +//! of the portable protocol corpus (fixed plus generated) through +//! `dialcache::identity`. +//! +//! Set `DIALCACHE_PROTOCOL_CORPUS=fixed|generated` to narrow the corpus. + +#[path = "formal/digest.rs"] +mod digest; + +#[path = "formal/fixtures.rs"] +mod fixtures; +#[path = "formal/key_vectors.rs"] +mod key_vectors; + +use serde_json::{json, Value}; + +type Check = fn(&Value) -> Result<(), String>; + +#[test] +fn key_assertions_distinguish_library_rejections_from_bad_fixtures() { + let valid_shape = json!({ + "input":{"namespace":"bad{namespace", "keyType":"id", "id":"1", "useCase":"Example", "tracked":false, "args":[]}, + "logicalKey":"expected", "valueKey":"expected", "watermarkKey":null, + }); + let error = key_vectors::check_key_vector(&valid_shape).unwrap_err(); + assert!( + error.starts_with("PROTOCOL_ASSERTION_FAILURE expected:"), + "{error}" + ); + let mut malformed = valid_shape; + malformed["logicalKey"] = json!(5); + let error = key_vectors::check_key_vector(&malformed).unwrap_err(); + assert!(!error.contains("PROTOCOL_ASSERTION_FAILURE"), "{error}"); +} + +const GROUPS: [(&str, Check); 4] = [ + ("keyVectors", key_vectors::check_key_vector), + ("invalidKeyVectors", key_vectors::check_invalid_key_vector), + ( + "normalizeArgsVectors", + key_vectors::check_normalize_args_vector, + ), + ("rampVectors", key_vectors::check_ramp_vector), +]; + +#[test] +fn key_protocol_vectors_replay() { + let groups = fixtures::protocol_groups("all"); + let mut failures = Vec::new(); + for (group, check) in GROUPS { + let rows = groups + .get(group) + .unwrap_or_else(|| panic!("protocol corpus has no {group} group")); + assert!(!rows.is_empty(), "{group}: no vectors selected"); + let mut passed = 0; + for row in rows { + let name = row["name"].as_str().unwrap_or(""); + match check(row) { + Ok(()) => passed += 1, + Err(reason) => failures.push(format!("{group} {name:?}: {reason}")), + } + } + println!("{group}: {passed}/{} passed", rows.len()); + } + assert!( + failures.is_empty(), + "{} vector(s) failed:\n{}", + failures.len(), + failures.join("\n") + ); +} diff --git a/rust/tests/recovery_logging.rs b/rust/tests/recovery_logging.rs new file mode 100644 index 00000000..1b0e43e2 --- /dev/null +++ b/rust/tests/recovery_logging.rs @@ -0,0 +1,125 @@ +//! Recovery diagnostics must not disclose retained values through the default logger. +use dialcache::observe::LogFacadeLogger; +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, DialCache, Frame, Identity, InvalidateRequest, LogEvent, Logger, Operation, Payload, + Policy, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest, +}; +use futures::future::BoxFuture; +use std::future::ready; +use std::sync::{Arc, Mutex}; + +const SECRET: &str = "private-retained-token-7f61"; +struct Capture(Mutex>); +impl log::Log for Capture { + fn enabled(&self, _: &log::Metadata<'_>) -> bool { + true + } + fn log(&self, record: &log::Record<'_>) { + self.0.lock().unwrap().push(record.args().to_string()); + } + fn flush(&self) {} +} +static CAPTURE: Capture = Capture(Mutex::new(Vec::new())); + +#[derive(Default)] +struct DetailedLogger(Mutex>); +impl Logger for DetailedLogger { + fn log(&self, event: &LogEvent) { + if let LogEvent::RecoveryDecodeFailed(error) = event { + let error = error + .downcast_ref::() + .expect("original codec error"); + self.0.lock().unwrap().push(error.to_string()); + } + } +} +struct RetainedString; +impl Remote for RetainedString { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(ready(Ok(ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64 - 1500, + payload: Payload::text(serde_json::to_string(SECRET).unwrap()), + })))) + } + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + panic!("failed recovery must not write a value") + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + panic!("no invalidation requested") + } +} +#[derive(Debug, thiserror::Error)] +#[error("source unavailable")] +struct SourceFailure(Arc<()>); + +#[test] +fn recovery_warning_omits_cached_values_and_preserves_structured_errors() { + log::set_logger(&CAPTURE).unwrap(); + log::set_max_level(log::LevelFilter::Warn); + for custom_logger in [false, true] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let detailed = Arc::new(DetailedLogger::default()); + let mut builder = DialCache::builder() + .runtime_arc(executor.runtime.clone()) + .clock_arc(executor.clock.clone()) + .remote(RetainedString); + if custom_logger { + builder = builder.logger_arc(detailed.clone()); + } + let cache = builder.build().unwrap(); + let marker = Arc::new(()); + let source_marker = marker.clone(); + let error = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load( + request.scope(), + Operation::::new(Identity::new("thing", "one", "PrivateRecovery")) + .policy( + Policy::default() + .remote_ttl_sec(1) + .stale_on_error_max_age_sec(10), + ) + .should_recover(|_| true), + move |_| { + ready(Err( + Box::new(SourceFailure(source_marker.clone())) as BoxError + )) + }, + ) + .await + .expect_err("incompatible retained value must preserve source failure") + }); + let original = error + .source_error() + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(Arc::ptr_eq(&original.0, &marker)); + if custom_logger { + let details = detailed.0.lock().unwrap(); + assert_eq!(details.len(), 1); + assert!( + details[0].contains(SECRET), + "explicit logger retains original diagnostic" + ); + } else { + let messages = CAPTURE.0.lock().unwrap(); + assert_eq!(messages.len(), 1, "actual default warning must be emitted"); + assert!( + !messages[0].contains(SECRET), + "cached value leaked: {}", + messages[0] + ); + assert!(messages[0].contains("JSON Data")); + assert!(messages[0].contains("line 1")); + } + } + // Arbitrary codec errors may also contain values: the default representation + // cannot trust their Display implementation even without JSON metadata. + LogFacadeLogger.log(&LogEvent::RecoveryDecodeFailed(SECRET.into())); + let messages = CAPTURE.0.lock().unwrap(); + assert_eq!(messages.len(), 2); + assert!(!messages[1].contains(SECRET)); +} diff --git a/rust/tests/redis_integration.rs b/rust/tests/redis_integration.rs new file mode 100644 index 00000000..2a29b611 --- /dev/null +++ b/rust/tests/redis_integration.rs @@ -0,0 +1,909 @@ +//! Real-server tests of the `redis` adapter, mirroring +//! `go/redis_integration_test.go`: Docker-started Redis 6.2, Redis 7 and +//! Valkey 8 standalone servers plus a single-node Redis 7 cluster, each +//! exercised for complete-frame round trips, watermark fencing, the +//! `EVALSHA` to `EVAL` recovery after `SCRIPT FLUSH`, and the full +//! invalidation vector corpus (`formal/invalidation_vectors.rs`), plus actual +//! TypeScript/Rust key, payload and invalidation interoperability; and a +//! replicated Redis 7 cluster that pins tracked reads to the slot primary. +//! +//! Every test here is `#[ignore]`d: `cargo test` reports it as ignored, and +//! `make integration-rust` runs the file with `--ignored` where Docker exists. +//! The `docker` CLI and the images must be available. Containers and the +//! cluster network are removed by drop guards on every exit path. +//! +//! Cluster coverage has two shapes. `redis_7_single_node_cluster` is one +//! primary owning every slot, announced on loopback, and runs the whole +//! corpus through a real `ClusterConnection`. With no replica present it +//! cannot tell primary routing from replica routing, so +//! `redis_7_replicated_cluster_primary_read` adds one primary plus one +//! replica of the same slots on a dedicated Docker bridge network. The +//! nodes gossip their container addresses, which Docker Desktop does not +//! route from the host; `ClusterClientBuilder::node_address_map` remaps +//! each announced address to the node's published loopback port at connect +//! time (the slot map keeps the announced addresses, so `MOVED` redirects +//! still resolve). With replica reads enabled on the client +//! (`read_routing_strategy(RandomReplicaStrategy)`, the successor of +//! `read_from_replicas`), each node's `INFO commandstats` proves every +//! tracked `MGET` executed on the primary and none on the replica: the +//! property `testPrimaryRead` in the Go test checks with per-node route +//! hooks on a six-node cluster. + +#![cfg(feature = "redis")] + +#[path = "formal/digest.rs"] +mod digest; + +mod formal; +#[path = "formal/invalidation_vectors.rs"] +mod invalidation_vectors; +mod redis_interop; + +use std::collections::HashMap; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use dialcache::protocol::encode_frame; +use dialcache::redis::{RedisAdapter, RedisConnection}; +use dialcache::{ + CancelToken, Frame, InvalidateRequest, MissReason, Payload, ReadContext, ReadRequest, + ReadResult, Remote, WriteRequest, +}; +use formal::witness::sha256_hex; +use redis::aio::{ConnectionManager, ConnectionManagerConfig, MultiplexedConnection}; +use redis::cluster::{ClusterClientBuilder, NodeAddress}; +use redis::cluster_async::ClusterConnection; +use redis::cluster_read_routing::RandomReplicaStrategy; +use redis::cluster_routing::{Route, RoutingInfo, SingleNodeRoutingInfo, Slot, SlotAddr}; +use redis::{Client, Value}; + +const COMMAND_BUDGET: Duration = Duration::from_secs(2); +const READY_BUDGET: Duration = Duration::from_secs(15); + +/// The Go test's server flags: no persistence, no protected mode. +const SERVER_FLAGS: [&str; 6] = ["--save", "", "--appendonly", "no", "--protected-mode", "no"]; +/// Cluster mode without announce overrides: a node announces the address +/// its peers see it from. +const CLUSTER_FLAGS: [&str; 6] = [ + "--cluster-enabled", + "yes", + "--cluster-config-file", + "/tmp/nodes.conf", + "--cluster-node-timeout", + "5000", +]; + +fn server_binary(image: &str) -> &'static str { + if image.contains("valkey") { + "valkey-server" + } else { + "redis-server" + } +} + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("..") +} + +/// Run one `docker` command and return its trimmed stdout. A cold `docker +/// run` writes pull progress to stderr and only the container id to stdout. +fn docker(args: &[&str]) -> Result { + let output = Command::new("docker") + .args(args) + .output() + .map_err(|error| format!("docker {args:?}: {error}"))?; + if !output.status.success() { + return Err(format!( + "docker {args:?}: {}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +/// A started container, force-removed when dropped. +struct Container { + id: String, +} + +impl Drop for Container { + fn drop(&mut self) { + let _ = Command::new("docker").args(["rm", "-f", &self.id]).output(); + } +} + +/// A free loopback port; the listener is dropped so Docker can bind it. +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .and_then(|listener| listener.local_addr()) + .expect("bind an ephemeral loopback port") + .port() +} + +/// `docker run` a Redis-compatible image with the Go test's server flags. +/// For a cluster node the host port is chosen first so the node can announce +/// the loopback address a host-side cluster client can reach. +fn start_server(image: &str, cluster: bool) -> (Container, String) { + let announce_port = cluster.then(free_port); + let publish = match announce_port { + Some(port) => format!("127.0.0.1:{port}:6379"), + None => "127.0.0.1::6379".to_string(), + }; + let announce = announce_port.map(|port| port.to_string()); + let mut args = vec![ + "run", + "-d", + "--rm", + "-p", + &publish, + image, + server_binary(image), + ]; + args.extend(SERVER_FLAGS); + if let Some(port) = &announce { + args.extend(CLUSTER_FLAGS); + args.extend([ + "--cluster-announce-ip", + "127.0.0.1", + "--cluster-announce-port", + port, + ]); + } + let id = docker(&args).expect("start container"); + let container = Container { id }; + let endpoint = published_endpoint(&container); + (container, endpoint) +} + +/// The `host:port` Docker published for the container's Redis port. +fn published_endpoint(container: &Container) -> String { + docker(&["port", &container.id, "6379/tcp"]) + .expect("published port") + .lines() + .next() + .expect("one published endpoint") + .to_string() +} + +/// The value of `name` in an `INFO` or `CLUSTER INFO` reply, or empty when +/// the field is absent. Lines keep their `\r`, which `trim` removes. +fn info_field(info: &str, name: &str) -> String { + info.lines() + .find_map(|line| line.trim().strip_prefix(name)?.strip_prefix(':')) + .unwrap_or("") + .to_string() +} + +/// A node's `CLUSTER INFO` reports a healthy cluster with every slot served. +fn cluster_ready(info: &str) -> bool { + info_field(info, "cluster_state") == "ok" + && info_field(info, "cluster_slots_assigned") == "16384" + && info_field(info, "cluster_slots_ok") == "16384" +} + +async fn wait_for_ping(endpoint: &str) { + let client = Client::open(format!("redis://{endpoint}")).expect("client url"); + let deadline = Instant::now() + READY_BUDGET; + loop { + if let Ok(mut connection) = client.get_multiplexed_async_connection().await { + if redis::cmd("PING") + .query_async::(&mut connection) + .await + .is_ok() + { + return; + } + } + assert!(Instant::now() < deadline, "{endpoint} did not become ready"); + tokio::time::sleep(Duration::from_millis(25)).await; + } +} + +async fn standalone(image: &str) -> (Container, ConnectionManager) { + let (container, endpoint) = start_server(image, false); + wait_for_ping(&endpoint).await; + let client = Client::open(format!("redis://{endpoint}")).expect("client url"); + let config = ConnectionManagerConfig::new() + .set_connection_timeout(Some(COMMAND_BUDGET)) + .set_response_timeout(Some(COMMAND_BUDGET)) + .set_number_of_retries(1); + let connection = client + .get_connection_manager_with_config(config) + .await + .expect("connection manager"); + (container, connection) +} + +/// One cluster-enabled node owning every slot; readiness waits for the +/// node's own slot view exactly as the Go test waits for six nodes. +async fn single_node_cluster(image: &str) -> (Container, ClusterConnection) { + let (container, endpoint) = start_server(image, true); + wait_for_ping(&endpoint).await; + docker(&[ + "exec", + &container.id, + "redis-cli", + "CLUSTER", + "ADDSLOTSRANGE", + "0", + "16383", + ]) + .expect("assign every slot"); + let deadline = Instant::now() + READY_BUDGET; + loop { + let info = + docker(&["exec", &container.id, "redis-cli", "CLUSTER", "INFO"]).unwrap_or_default(); + if cluster_ready(&info) { + break; + } + assert!( + Instant::now() < deadline, + "cluster did not become ready:\n{info}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let connection = ClusterClientBuilder::new([format!("redis://{endpoint}")]) + .connection_timeout(COMMAND_BUDGET) + .response_timeout(COMMAND_BUDGET) + .build() + .expect("cluster client") + .get_async_connection() + .await + .expect("cluster connection"); + (container, connection) +} + +fn context() -> ReadContext { + ReadContext { + timeout_ms: 50, + cancel: CancelToken::new(), + } +} + +fn tracked(value_key: &str, watermark_key: &str) -> ReadRequest { + ReadRequest { + value_key: value_key.to_string(), + watermark_key: Some(watermark_key.to_string()), + } +} + +async fn primary_integer(connection: &C, key: &str, cmd: redis::Cmd) -> i64 { + match connection + .run_on_primary(key, cmd) + .await + .expect("fixture command") + { + Value::Int(value) => value, + other => panic!("expected integer, got {other:?}"), + } +} + +/// `go/redis_integration_test.go` `testPrimaryRead` plus untracked reads: +/// stored bytes are the exact frame, a value write never extends the +/// watermark, the fence is observed on the primary, and `SCRIPT FLUSH` +/// forces the `EVALSHA` recovery through the real server. +async fn round_trip(adapter: &RedisAdapter, connection: &C, label: &str) { + let key = format!("{{primary-rust-{label}}}:value"); + let watermark = format!("{{primary-rust-{label}}}:watermark"); + let untracked = format!("{{primary-rust-{label}}}:untracked"); + + let frame = Frame { + created_at_ms: 2_000, + payload: Payload::binary(vec![0, 1, 255]), + }; + let text_frame = Frame { + created_at_ms: 3_000, + payload: Payload::text("DialCache é"), + }; + adapter + .write(WriteRequest { + value_key: untracked.clone(), + frame: text_frame.clone(), + ttl_ms: 10_000, + }) + .await + .expect("untracked write"); + let got = adapter + .read( + ReadRequest { + value_key: untracked.clone(), + watermark_key: None, + }, + context(), + ) + .await + .expect("untracked read"); + assert_eq!( + got, + ReadResult::Hit(text_frame), + "{label}: untracked round trip" + ); + let got = adapter + .read( + ReadRequest { + value_key: format!("{untracked}:missing"), + watermark_key: None, + }, + context(), + ) + .await + .expect("absent read"); + assert_eq!(got, ReadResult::miss(MissReason::ValueAbsent), "{label}"); + + // Tracked frame before any watermark: served. + adapter + .write(WriteRequest { + value_key: key.clone(), + frame: frame.clone(), + ttl_ms: 10_000, + }) + .await + .expect("tracked write"); + let got = adapter + .read(tracked(&key, &watermark), context()) + .await + .expect("tracked read"); + assert_eq!( + got, + ReadResult::Hit(frame.clone()), + "{label}: tracked round trip" + ); + + adapter + .invalidate(InvalidateRequest { + watermark_key: watermark.clone(), + invalidated_at_ms: 2_000, + future_buffer_ms: 0, + }) + .await + .expect("invalidate"); + let mut pttl = redis::cmd("PTTL"); + pttl.arg(&watermark); + let before = primary_integer(connection, &key, pttl.clone()).await; + adapter + .write(WriteRequest { + value_key: key.clone(), + frame: frame.clone(), + ttl_ms: 10_000, + }) + .await + .expect("tracked write"); + let mut get = redis::cmd("GET"); + get.arg(&key); + let raw = connection.run_on_primary(&key, get).await.expect("GET"); + assert_eq!( + raw, + Value::BulkString(encode_frame(&frame).unwrap()), + "{label}: stored frame differs" + ); + let got = adapter + .read(tracked(&key, &watermark), context()) + .await + .expect("tracked read"); + assert_eq!( + got, + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(2_000), + }, + "{label}: tracked read missed primary fence" + ); + let after = primary_integer(connection, &key, pttl).await; + assert!( + after <= before, + "{label}: value write extended watermark {before} -> {after}" + ); + + // A newer frame passes the fence. + let newer = Frame { + created_at_ms: 2_001, + payload: Payload::text("1"), + }; + adapter + .write(WriteRequest { + value_key: key.clone(), + frame: newer.clone(), + ttl_ms: 10_000, + }) + .await + .expect("newer write"); + let got = adapter + .read(tracked(&key, &watermark), context()) + .await + .expect("tracked read"); + assert_eq!(got, ReadResult::Hit(newer), "{label}: newer frame fenced"); + + // SCRIPT FLUSH forces the adapter's EVALSHA recovery through the real server. + let mut flush = redis::cmd("SCRIPT"); + flush.arg("FLUSH"); + connection + .run_on_primary(&key, flush) + .await + .expect("SCRIPT FLUSH"); + let mut evalsha = redis::cmd("EVALSHA"); + evalsha + .arg(dialcache::redis::invalidation_script_sha1()) + .arg(1) + .arg(&watermark) + .arg("0") + .arg("3000"); + let noscript = connection + .run_on_primary(&key, evalsha) + .await + .expect_err("flushed script cache rejects EVALSHA"); + assert_eq!(noscript.code(), Some("NOSCRIPT"), "{label}: {noscript}"); + adapter + .invalidate(InvalidateRequest { + watermark_key: watermark.clone(), + invalidated_at_ms: 3_000, + future_buffer_ms: 0, + }) + .await + .expect("NOSCRIPT recovery"); + let got = adapter + .read(tracked(&key, &watermark), context()) + .await + .expect("tracked read"); + assert_eq!( + got, + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(3_000), + }, + "{label}: recovered invalidation did not fence" + ); +} + +async fn exercise(connection: C, label: &str, endpoint: &str, cluster: bool) { + let adapter = RedisAdapter::new(connection); + round_trip(&adapter, adapter.connection(), label).await; + redis_interop::exercise(&adapter, endpoint, cluster, label).await; + let vectors = invalidation_vectors::load_corpus(&repo_root(), sha256_hex); + let prefix = format!("{{rust-invalidation-{label}}}:"); + match invalidation_vectors::replay_all(&adapter, adapter.connection(), &prefix, &vectors).await + { + Ok(count) => println!("{label}: replayed {count} invalidation vectors"), + Err(failures) => panic!( + "{label}: {} of {} invalidation vectors failed:\n{}", + failures.len(), + vectors.len(), + failures.join("\n") + ), + } +} + +fn run_standalone(test: &str, image: &str) { + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); + runtime.block_on(async { + let (container, connection) = standalone(image).await; + exercise(connection, test, &published_endpoint(&container), false).await; + }); +} + +#[test] +#[ignore = "Docker-backed; run through make integration-rust"] +fn redis_6_2_standalone() { + run_standalone("redis6.2", "redis:6.2-alpine"); +} + +#[test] +#[ignore = "Docker-backed; run through make integration-rust"] +fn redis_7_standalone() { + run_standalone("redis7", "redis:7-alpine"); +} + +#[test] +#[ignore = "Docker-backed; run through make integration-rust"] +fn valkey_8_standalone() { + run_standalone("valkey8", "valkey/valkey:8-alpine"); +} + +#[test] +#[ignore = "Docker-backed; run through make integration-rust"] +fn redis_7_single_node_cluster() { + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); + runtime.block_on(async { + let (container, connection) = single_node_cluster("redis:7-alpine").await; + exercise(connection, "cluster", &published_endpoint(&container), true).await; + }); +} + +/// A Docker bridge network, removed when dropped. The daemon detaches a +/// force-removed container's endpoint asynchronously, so removal retries +/// while the network still reports active endpoints. +struct Network { + name: String, +} + +impl Drop for Network { + fn drop(&mut self) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let output = Command::new("docker") + .args(["network", "rm", &self.name]) + .output(); + let retry = match &output { + Ok(output) if output.status.success() => false, + Ok(output) => String::from_utf8_lossy(&output.stderr).contains("active endpoints"), + Err(_) => false, + }; + if !retry || Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + } +} + +/// One node of the replicated cluster: its container, the loopback +/// endpoint Docker published for host clients, and the bridge address the +/// node announces to its peers (and therefore in `CLUSTER SLOTS`). +struct ClusterNode { + container: Container, + endpoint: String, + bridge_ip: String, +} + +impl ClusterNode { + fn host_port(&self) -> u16 { + self.endpoint + .rsplit(':') + .next() + .and_then(|port| port.parse().ok()) + .expect("published endpoint has a port") + } + + /// Run `redis-cli` inside the container for cluster setup commands. + fn redis_cli(&self, args: &[&str]) -> Result { + let mut command = vec!["exec", self.container.id.as_str(), "redis-cli"]; + command.extend(args); + docker(&command) + } +} + +/// One primary owning every slot and one replica of it, gossiping over a +/// dedicated bridge network. Field order removes the containers before the +/// network they are attached to. +struct ReplicatedCluster { + primary: ClusterNode, + replica: ClusterNode, + _network: Network, +} + +/// `docker run` a cluster node on `network`, publishing its port on a free +/// loopback port and letting it announce its bridge address to peers. +fn start_cluster_node(image: &str, network: &str) -> ClusterNode { + let mut args = vec![ + "run", + "-d", + "--rm", + "--network", + network, + "-p", + "127.0.0.1::6379", + image, + server_binary(image), + ]; + args.extend(SERVER_FLAGS); + args.extend(CLUSTER_FLAGS); + let id = docker(&args).expect("start cluster node"); + let container = Container { id }; + let endpoint = published_endpoint(&container); + let bridge_ip = docker(&[ + "inspect", + "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + &container.id, + ]) + .expect("bridge address"); + assert!( + !bridge_ip.is_empty(), + "container {} has no bridge address", + container.id + ); + ClusterNode { + container, + endpoint, + bridge_ip, + } +} + +/// Bring up the replicated cluster: assign every slot to the primary, have +/// it `MEET` the replica, wait for gossip to teach the replica the primary's +/// id, `REPLICATE`, then wait until both nodes report `cluster_state:ok` and +/// the replica's link to its primary is up. +async fn replicated_cluster(image: &str) -> ReplicatedCluster { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|since| since.as_nanos()) + .unwrap_or_default(); + let name = format!("dialcache-rust-{}-{nanos}", std::process::id()); + docker(&["network", "create", &name]).expect("create network"); + let network = Network { name: name.clone() }; + let primary = start_cluster_node(image, &name); + let replica = start_cluster_node(image, &name); + let cluster = ReplicatedCluster { + primary, + replica, + _network: network, + }; + wait_for_ping(&cluster.primary.endpoint).await; + wait_for_ping(&cluster.replica.endpoint).await; + cluster + .primary + .redis_cli(&["CLUSTER", "ADDSLOTSRANGE", "0", "16383"]) + .expect("assign every slot"); + cluster + .primary + .redis_cli(&["CLUSTER", "MEET", &cluster.replica.bridge_ip, "6379"]) + .expect("meet the replica"); + let primary_id = cluster + .primary + .redis_cli(&["CLUSTER", "MYID"]) + .expect("primary id"); + + // CLUSTER REPLICATE rejects an id the replica has not yet learned. + let deadline = Instant::now() + READY_BUDGET; + loop { + let nodes = cluster + .replica + .redis_cli(&["CLUSTER", "NODES"]) + .unwrap_or_default(); + if nodes.lines().any(|line| line.starts_with(&primary_id)) { + break; + } + assert!( + Instant::now() < deadline, + "replica never learned the primary {primary_id}:\n{nodes}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + cluster + .replica + .redis_cli(&["CLUSTER", "REPLICATE", &primary_id]) + .expect("replicate the primary"); + + // The client seeds its slot map from the primary's CLUSTER SLOTS, which + // omits a replica until gossip has reported a non-zero replication + // offset for it; wait for that too, or the client would never learn the + // replica exists. + let deadline = Instant::now() + READY_BUDGET; + loop { + let primary_info = cluster + .primary + .redis_cli(&["CLUSTER", "INFO"]) + .unwrap_or_default(); + let replica_info = cluster + .replica + .redis_cli(&["CLUSTER", "INFO"]) + .unwrap_or_default(); + let replication = cluster + .replica + .redis_cli(&["INFO", "replication"]) + .unwrap_or_default(); + let slots = cluster + .primary + .redis_cli(&["CLUSTER", "SLOTS"]) + .unwrap_or_default(); + if cluster_ready(&primary_info) + && cluster_ready(&replica_info) + && info_field(&replication, "role") == "slave" + && info_field(&replication, "master_link_status") == "up" + && slots + .lines() + .any(|line| line.trim() == cluster.replica.bridge_ip) + { + break; + } + assert!( + Instant::now() < deadline, + "replicated cluster did not become ready:\nprimary:\n{primary_info}\nreplica:\n{replica_info}\nreplication:\n{replication}\nslots:\n{slots}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + cluster +} + +/// A cluster connection that prefers replicas for reads +/// (`RandomReplicaStrategy`, what the deprecated `read_from_replicas` +/// installs). The nodes announce their bridge addresses, which the host +/// cannot route to under Docker Desktop; `node_address_map` redirects each +/// to the node's published loopback port when the client connects. +async fn replica_reading_connection(cluster: &ReplicatedCluster) -> ClusterConnection { + let mut addresses = HashMap::new(); + for node in [&cluster.primary, &cluster.replica] { + addresses.insert( + NodeAddress::new(node.bridge_ip.as_str(), 6379), + NodeAddress::new("127.0.0.1", node.host_port()), + ); + } + ClusterClientBuilder::new([format!("redis://{}", cluster.primary.endpoint)]) + .connection_timeout(COMMAND_BUDGET) + .response_timeout(COMMAND_BUDGET) + .read_routing_strategy(RandomReplicaStrategy) + .node_address_map(addresses) + .build() + .expect("cluster client") + .get_async_connection() + .await + .expect("cluster connection") +} + +/// A plain connection to one node's published endpoint, bypassing cluster +/// routing, for per-node observation. +async fn direct_connection(endpoint: &str) -> MultiplexedConnection { + Client::open(format!("redis://{endpoint}")) + .expect("client url") + .get_multiplexed_async_connection() + .await + .expect("direct connection") +} + +/// How many times the node executed `command`, from `INFO commandstats` +/// (`cmdstat_:calls=N,...`). Commands the node refused with +/// `MOVED` count under `rejected_calls`, not `calls`; a command the node +/// never ran has no line at all. +async fn executed_calls(connection: &mut MultiplexedConnection, command: &str) -> u64 { + let info: String = redis::cmd("INFO") + .arg("commandstats") + .query_async(connection) + .await + .expect("INFO commandstats"); + info_field(&info, &format!("cmdstat_{command}")) + .split(',') + .find_map(|field| field.strip_prefix("calls=")) + .and_then(|calls| calls.parse().ok()) + .unwrap_or(0) +} + +/// Route one `GET key` to a replica of the key's slot through the cluster +/// connection. `ReplicaRequired` can only choose a replica when the client's +/// slot map lists one and otherwise falls back to the primary, so this +/// executing on the replica proves the client knows the replica and reached +/// it through the address map: a `ReplicaOptional` route would have had a +/// replica to pick, which makes the tracked-read assertion decisive. +async fn replica_routed_get(connection: &ClusterConnection, key: &str) { + let route = Route::with_slot(Slot::for_key(key), SlotAddr::ReplicaRequired); + let routing = RoutingInfo::SingleNode(SingleNodeRoutingInfo::SpecificNode(route)); + let mut get = redis::cmd("GET"); + get.arg(key); + connection + .clone() + .route_command(get, routing) + .await + .expect("replica-routed GET"); +} + +/// Poll the replica directly (after `READONLY`, so it serves rather than +/// redirects) until the exact frame bytes have replicated. +async fn wait_for_replica_frame(replica: &mut MultiplexedConnection, key: &str, frame: &Frame) { + redis::cmd("READONLY") + .query_async::<()>(replica) + .await + .expect("READONLY"); + let encoded = encode_frame(frame).expect("encode frame"); + let deadline = Instant::now() + READY_BUDGET; + loop { + let stored: Option> = redis::cmd("GET") + .arg(key) + .query_async(replica) + .await + .expect("GET on the replica"); + if stored.as_deref() == Some(encoded.as_slice()) { + return; + } + assert!( + Instant::now() < deadline, + "replica never received {key}: {stored:?}" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +/// `go/redis_integration_test.go` `testPrimaryRead`'s routing assertion on +/// a cluster that has a replica: with replica reads enabled on the client, +/// every tracked `MGET` still executes on the slot primary and none on the +/// replica, so a lagging replica can never hide a watermark. Each node's own +/// `INFO commandstats` is the witness. +/// +/// The `redis` crate's `RandomReplicaStrategy` picks uniformly among +/// replicas only, so a `ReplicaOptional` route would send every one of these +/// reads to the replica and fail on the first. Twelve reads keep the check +/// decisive even against a strategy that chose uniformly between both +/// nodes, where all twelve landing on the primary by chance would be 2^-12, +/// about 0.02%. +#[test] +#[ignore = "Docker-backed; run through make integration-rust"] +fn redis_7_replicated_cluster_primary_read() { + const TRACKED_READS: u64 = 12; + const UNTRACKED_READS: u64 = 3; + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); + runtime.block_on(async { + let cluster = replicated_cluster("redis:7-alpine").await; + let adapter = RedisAdapter::new(replica_reading_connection(&cluster).await); + let mut primary = direct_connection(&cluster.primary.endpoint).await; + let mut replica = direct_connection(&cluster.replica.endpoint).await; + + let key = "{primary-rust-replicated}:value"; + let watermark = "{primary-rust-replicated}:watermark"; + let frame = Frame { + created_at_ms: 2_000, + payload: Payload::binary(vec![0, 1, 255]), + }; + adapter + .write(WriteRequest { + value_key: key.to_string(), + frame: frame.clone(), + ttl_ms: 60_000, + }) + .await + .expect("tracked write"); + wait_for_replica_frame(&mut replica, key, &frame).await; + + // Precondition: the client's slot map holds the replica, so replica + // routing is a real choice the adapter is declining. + let primary_gets = executed_calls(&mut primary, "get").await; + let replica_gets = executed_calls(&mut replica, "get").await; + replica_routed_get(adapter.connection(), key).await; + assert_eq!( + ( + executed_calls(&mut primary, "get").await - primary_gets, + executed_calls(&mut replica, "get").await - replica_gets + ), + (0, 1), + "replica-routed GET executions (primary, replica): the client does not know the replica" + ); + + let primary_before = executed_calls(&mut primary, "mget").await; + let replica_before = executed_calls(&mut replica, "mget").await; + for read in 0..TRACKED_READS { + let got = adapter + .read(tracked(key, watermark), context()) + .await + .expect("tracked read"); + assert_eq!(got, ReadResult::Hit(frame.clone()), "tracked read {read}"); + } + let primary_after = executed_calls(&mut primary, "mget").await; + let replica_after = executed_calls(&mut replica, "mget").await; + assert_eq!( + ( + primary_after - primary_before, + replica_after - replica_before + ), + (TRACKED_READS, 0), + "tracked MGET executions (primary, replica): primary {primary_before} -> \ + {primary_after}, replica {replica_before} -> {replica_after}" + ); + + // Untracked reads keep the client's own routing and may execute on + // either node with replica reads enabled; only the total is pinned + // and the split is reported. + let primary_gets = executed_calls(&mut primary, "get").await; + let replica_gets = executed_calls(&mut replica, "get").await; + for read in 0..UNTRACKED_READS { + let got = adapter + .read( + ReadRequest { + value_key: key.to_string(), + watermark_key: None, + }, + context(), + ) + .await + .expect("untracked read"); + assert_eq!(got, ReadResult::Hit(frame.clone()), "untracked read {read}"); + } + let primary_delta = executed_calls(&mut primary, "get").await - primary_gets; + let replica_delta = executed_calls(&mut replica, "get").await - replica_gets; + assert_eq!( + primary_delta + replica_delta, + UNTRACKED_READS, + "untracked GET executions: primary {primary_delta}, replica {replica_delta}" + ); + println!("replicated: untracked GETs executed on primary={primary_delta} replica={replica_delta}"); + }); +} diff --git a/rust/tests/redis_interop/mod.rs b/rust/tests/redis_interop/mod.rs new file mode 100644 index 00000000..6fe60f2e --- /dev/null +++ b/rust/tests/redis_interop/mod.rs @@ -0,0 +1,429 @@ +//! Bidirectional production TypeScript/Rust reads, writes and invalidation on +//! the same server. Neither language receives the other's derived key/bytes. + +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use dialcache::identity::{normalize_args, ArgValue, Identity, Keys}; +use dialcache::protocol::{ + compress_payload, decompress_payload, escape_raw_payload, CompressionConfig, +}; +use dialcache::redis::{RedisAdapter, RedisConnection}; +use dialcache::{ + Frame, InvalidateRequest, JsonCodec, MissReason, Payload, ReadRequest, ReadResult, Remote, + WriteRequest, +}; +use serde_json::{json, Value}; + +const STAMP: u64 = 1_700_000_000_000; +const MAX_DECOMPRESSED_BYTES: usize = 512 * 1024 * 1024; + +struct TypeScript { + directory: PathBuf, + root: PathBuf, +} + +impl TypeScript { + fn new() -> Self { + let root = std::env::var_os("DIALCACHE_TS_ROOT") + .map(PathBuf::from) + .unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("..")); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "dialcache-rust-interop-{}-{nonce}", + std::process::id() + )); + fs::create_dir(&directory).expect("create TypeScript bundle directory"); + let script = Self { directory, root }; + // Use tsup's declared esbuild dependency, exactly as the Go integration + // does, and import current production TypeScript rather than a fixture codec. + let build = "const {createRequire}=require('node:module');const {buildSync}=createRequire(require.resolve('tsup'))('esbuild');buildSync({entryPoints:[process.argv[1]],outfile:process.argv[2],bundle:true,platform:'node',format:'cjs'});"; + let output = Command::new("node") + .arg("-e") + .arg(build) + .arg(script.root.join("go/redis_interop.ts")) + .arg(script.directory.join("interop.cjs")) + .current_dir(&script.root) + .output() + .expect("bundle TypeScript interop"); + assert!( + output.status.success(), + "TypeScript bundle failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + script + } + + fn run(&self, endpoint: &str, cluster: bool, actions: &[Value]) -> Vec { + let input = self.directory.join("input.json"); + let output = self.directory.join("output.json"); + let error = self.directory.join("stderr.txt"); + fs::write( + &input, + serde_json::to_vec( + &json!({ "endpoint": endpoint, "cluster": cluster, "actions": actions }), + ) + .unwrap(), + ) + .unwrap(); + // Files keep large key-probe output from filling an unread pipe while + // the parent waits, and let the timeout kill and reap a stuck Node child. + let mut child = Command::new("node") + .arg(self.directory.join("interop.cjs")) + .current_dir(&self.root) + .stdin(Stdio::from(File::open(input).unwrap())) + .stdout(Stdio::from(File::create(&output).unwrap())) + .stderr(Stdio::from(File::create(&error).unwrap())) + .spawn() + .expect("start TypeScript interop"); + let deadline = Instant::now() + Duration::from_secs(30); + let status = loop { + if let Some(status) = child.try_wait().expect("poll TypeScript interop") { + break status; + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!( + "TypeScript interop exceeded 30 seconds: {}", + fs::read_to_string(&error).unwrap_or_default() + ); + } + std::thread::sleep(Duration::from_millis(10)); + }; + assert!( + status.success(), + "TypeScript interop failed: {}", + fs::read_to_string(error).unwrap_or_default() + ); + let results: Vec = + serde_json::from_slice(&fs::read(output).unwrap()).expect("TypeScript JSON results"); + assert_eq!( + results.len(), + actions.len(), + "every interop operation must return an observation" + ); + results + } +} + +impl Drop for TypeScript { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.directory); + } +} + +/// Both ports receive the same scalar inputs and independently normalize them. +/// The exact double is transported as bits, not either port's decimal spelling. +fn identity(label: &str, name: &str, tracked: bool, number_bits: u64) -> (Value, Keys) { + let namespace = format!("rust-ts-{label}"); + let args = normalize_args([ + ("case", ArgValue::Str(name.to_string())), + ("number", ArgValue::Number(f64::from_bits(number_bits))), + ("\u{e000}", ArgValue::Bool(false)), + ("😀", ArgValue::Null), + ]) + .unwrap(); + let keys = Identity::new("entity é", "shared/id", "lookup?#") + .namespace(&namespace) + .tracked(tracked) + .args(args) + .keys() + .unwrap(); + let input = json!({ + "namespace": namespace, "keyType": "entity é", "id": "shared/id", "useCase": "lookup?#", + "trackForInvalidation": tracked, + "args": { "case": name, "\u{e000}": false, "😀": null }, + "numberBits": { "number": format!("{number_bits:016x}") }, + }); + (input, keys) +} + +fn key_json(keys: &Keys) -> Value { + json!({ "logical": keys.logical, "value": keys.value, "watermark": keys.watermark }) +} + +fn read_request(keys: &Keys) -> ReadRequest { + ReadRequest { + value_key: keys.value.clone(), + watermark_key: keys.watermark.clone(), + } +} + +fn prepare(payload: Payload, compressed: bool) -> Payload { + if compressed { + compress_payload( + payload, + &CompressionConfig { + threshold_bytes: 1, + level: 3, + }, + MAX_DECOMPRESSED_BYTES, + ) + .unwrap() + .payload + } else { + escape_raw_payload(payload) + } +} + +struct Case { + identity: Value, + keys: Keys, + payload: Payload, + value: Option, + compressed: bool, +} + +pub async fn exercise( + adapter: &RedisAdapter, + endpoint: &str, + cluster: bool, + label: &str, +) { + let script = TypeScript::new(); + // Native Number::toString is explicitly outside Quint's integer model. + // Probe the real TS key builder with regression ties, exponent boundaries, + // special values and deterministic whole-domain IEEE754 samples. + let mut bits: Vec = [ + 0.0_f64, + -0.0, + f64::from_bits(0x430c6bf526340002), + f64::from_bits(0xc30c6bf526340002), + f64::from_bits(0x430c6bf526340006), + f64::from_bits(0xc30c6bf526340006), + f64::from_bits(0x42d6bcc41e900008), + f64::from_bits(0x42d6bcc41e900018), + 1e-6, + 1e-7, + 1e20, + 1e21, + f64::MIN_POSITIVE, + f64::MAX, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + ] + .into_iter() + .map(f64::to_bits) + .collect(); + bits.push(1); // Smallest positive subnormal. + let mut sample = 88_172_645_463_325_252_u64; + for _ in 0..512 { + sample ^= sample << 13; + sample ^= sample >> 7; + sample ^= sample << 17; + bits.push(sample); + } + let probes: Vec<_> = bits + .into_iter() + .enumerate() + .map(|(index, bits)| identity(label, "number-probe", index % 2 == 0, bits)) + .collect(); + let actions: Vec<_> = probes + .iter() + .map(|(identity, _)| json!({ "op": "key", "identity": identity })) + .collect(); + for ((_, expected), actual) in probes.iter().zip(script.run(endpoint, cluster, &actions)) { + assert_eq!( + actual["keys"], + key_json(expected), + "{label}: independently constructed numeric keys" + ); + } + + let values = [ + Value::Null, + json!(false), + json!(0), + json!(""), + json!({"id": "cross-language", "nested": [1, null, false, "é😀"]}), + json!("DialCache é😀".repeat(1000)), + ]; + let binaries = [ + vec![], + vec![0, 1, 2, 255, 0xe2, 0x82], + vec![1, 0xff], + vec![2, 0xff], + vec![3, 0xff], + [0, 1, 2, 255, 0xe2, 0x82].repeat(1000), + ]; + let mut cases = Vec::new(); + for tracked in [false, true] { + for (index, value) in values.iter().enumerate() { + let (identity, keys) = + identity(label, &format!("json-{index}"), tracked, 0x430c6bf526340002); + cases.push(Case { + identity, + keys, + payload: JsonCodec::encode_value(value).unwrap(), + value: Some(value.clone()), + compressed: index == values.len() - 1, + }); + } + for (index, bytes) in binaries.iter().enumerate() { + let (identity, keys) = identity( + label, + &format!("binary-{index}"), + tracked, + 0xc30c6bf526340002, + ); + cases.push(Case { + identity, + keys, + payload: Payload::binary(bytes.clone()), + value: None, + compressed: index == binaries.len() - 1, + }); + } + } + for case in &cases { + adapter + .write(WriteRequest { + value_key: case.keys.value.clone(), + frame: Frame { + created_at_ms: STAMP, + payload: prepare(case.payload.clone(), case.compressed), + }, + ttl_ms: 60_000, + }) + .await + .expect("Rust writes interop frame"); + } + let reads: Vec<_> = cases.iter().map(|case| json!({ "op": "read", "identity": case.identity, "binary": case.value.is_none() })).collect(); + let results = script.run(endpoint, cluster, &reads); + for (case, result) in cases.iter().zip(results) { + assert_eq!(result["kind"], "hit", "{label}: TS reads Rust frame"); + assert_eq!( + result["stamp"], STAMP, + "{label}: TS preserves Rust timestamp" + ); + assert_eq!( + result["keys"], + key_json(&case.keys), + "{label}: TS derived the same keys" + ); + if let Some(value) = &case.value { + assert_eq!(&result["value"], value, "{label}: TS decodes Rust JSON"); + } else { + assert_eq!( + result["binaryHex"], + hex::encode(&case.payload.bytes), + "{label}: TS decodes Rust binary" + ); + } + } + let writes: Vec<_> = cases.iter().map(|case| { + let mut action = json!({ "op": "write", "identity": case.identity, "stamp": STAMP + 1, "compress": case.compressed }); + if let Some(value) = &case.value { action["value"] = value.clone(); } + else { action["binaryHex"] = json!(hex::encode(&case.payload.bytes)); } + action + }).collect(); + for (case, result) in cases.iter().zip(script.run(endpoint, cluster, &writes)) { + assert_eq!(result["kind"], "written"); + assert_eq!(result["keys"], key_json(&case.keys)); + } + for case in &cases { + let result = adapter + .read(read_request(&case.keys), super::context()) + .await + .expect("Rust reads TypeScript frame"); + let ReadResult::Hit(frame) = result else { + panic!("{label}: Rust missed TypeScript frame: {result:?}"); + }; + assert_eq!( + frame.created_at_ms, + STAMP + 1, + "{label}: must read the TypeScript overwrite" + ); + let payload = decompress_payload(frame.payload, MAX_DECOMPRESSED_BYTES).payload; + if let Some(expected) = &case.value { + assert_eq!( + &JsonCodec::decode_value::(&payload).unwrap(), + expected, + "{label}: Rust decodes TypeScript JSON" + ); + } else { + assert_eq!( + payload, case.payload, + "{label}: Rust decodes TypeScript binary" + ); + } + } + + // Rust's documented undefined adaptation is Option::None, distinct from + // writing its own null above (which TypeScript must continue to read as null). + let (undefined, undefined_keys) = identity(label, "undefined", true, 0); + script.run( + endpoint, + cluster, + &[json!({ "op": "write", "identity": undefined, "stamp": STAMP, "absent": true })], + ); + let ReadResult::Hit(frame) = adapter + .read(read_request(&undefined_keys), super::context()) + .await + .unwrap() + else { + panic!("Rust must read the TypeScript undefined sentinel"); + }; + assert_eq!( + JsonCodec::decode_value::>(&frame.payload).unwrap(), + None + ); + + let tracked = cases + .iter() + .find(|case| case.keys.watermark.is_some()) + .unwrap(); + script.run(endpoint, cluster, &[json!({ "op": "invalidate", "identity": tracked.identity, "stamp": STAMP + 1, "futureMs": 100 })]); + assert_eq!( + adapter + .read(read_request(&tracked.keys), super::context()) + .await + .unwrap(), + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(STAMP + 101) + }, + "{label}: TypeScript invalidation fences Rust reads" + ); + adapter + .write(WriteRequest { + value_key: tracked.keys.value.clone(), + frame: Frame { + created_at_ms: STAMP + 102, + payload: JsonCodec::encode_value(&json!("after invalidation")).unwrap(), + }, + ttl_ms: 60_000, + }) + .await + .unwrap(); + let read = json!({ "op": "read", "identity": tracked.identity }); + let result = script.run(endpoint, cluster, std::slice::from_ref(&read)); + assert_eq!( + result[0]["kind"], "hit", + "{label}: TypeScript reads Rust's newer frame" + ); + assert_eq!(result[0]["value"], "after invalidation"); + adapter + .invalidate(InvalidateRequest { + watermark_key: tracked.keys.watermark.clone().unwrap(), + invalidated_at_ms: STAMP + 1, + future_buffer_ms: 200, + }) + .await + .unwrap(); + let result = script.run(endpoint, cluster, &[read]); + assert_eq!( + result[0]["kind"], "miss", + "{label}: Rust invalidation fences TypeScript reads" + ); + assert_eq!(result[0]["reason"], "watermark_fenced"); + assert_eq!(result[0]["observedWatermarkMs"], STAMP + 201); + println!("{label}: {} independent key probes and {} payloads read in both languages, plus both invalidation directions", probes.len(), cases.len()); +} diff --git a/rust/tests/runtime_policy.rs b/rust/tests/runtime_policy.rs new file mode 100644 index 00000000..528a5d7f --- /dev/null +++ b/rust/tests/runtime_policy.rs @@ -0,0 +1,72 @@ +//! Native typed-policy conversion preserves the portable sparse-overlay rules. + +#![cfg(feature = "tokio")] + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use dialcache::{DialCache, Identity, Operation, Policy}; + +fn cache_with_ttl_overlay() -> DialCache { + DialCache::builder() + .local_capacity(0) + .policy_provider(|_| async { Ok(Some(Policy::default().local_ttl_sec(9).into())) }) + .build() + .unwrap() +} + +#[tokio::test] +async fn typed_ttl_overlay_preserves_request_memoization() { + let cache = cache_with_ttl_overlay(); + let request = cache.enable_guard(); + let calls = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + let calls = calls.clone(); + let value = cache + .get_or_load( + request.scope(), + Operation::::new(Identity::new("item", "one", "lookup")) + .policy(Policy::default().request_local(true)), + move |_| { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + async move { Ok(value) } + }, + ) + .await + .unwrap(); + assert_eq!(*value, 1, "the second call must use the request memo"); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn typed_ttl_overlay_preserves_independent_sources() { + let cache = cache_with_ttl_overlay(); + let request = cache.enable_guard(); + let calls = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let operation = Operation::::new(Identity::new("item", "one", "lookup")) + .policy(Policy::default().coalesce(false)); + let load = { + let calls = calls.clone(); + move |_| { + let calls = calls.clone(); + let barrier = barrier.clone(); + async move { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + barrier.wait().await; + Ok(value) + } + } + }; + let first = cache.get_or_load(request.scope(), operation.clone(), load.clone()); + let second = cache.get_or_load(request.scope(), operation, load); + let (first, second) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(first, second) + }) + .await + .expect("a TTL-only overlay must not coalesce independent sources"); + assert_ne!(*first.unwrap(), *second.unwrap()); + assert_eq!(calls.load(Ordering::SeqCst), 2); +} diff --git a/rust/tests/settlement_control.rs b/rust/tests/settlement_control.rs new file mode 100644 index 00000000..a451c512 --- /dev/null +++ b/rust/tests/settlement_control.rs @@ -0,0 +1,110 @@ +//! The no-settle control: skipping the driver drain must fail the settlement +//! contract of every behavior smoke history before observation comparison. + +#[path = "formal/digest.rs"] +mod digest; + +mod formal; + +use formal::driver::{install_panic_hook, Driver}; +use formal::inventory::repo_path; +use formal::transport::Coordinator; +use serde_json::{json, Value}; + +const BEHAVIOR_SMOKE: [&str; 15] = [ + "dark-layers", + "shadow-read-deadlines", + "effects", + "admission", + "independent", + "layers", + "local-failure", + "policy", + "recovery", + "recovery-read", + "runtime-boundaries", + "scope", + "shadow", + "shadow-layers", + "source-budgets", +]; + +fn replay(coordinator: &mut Coordinator, profile: &str, skip_settle: bool) -> Result<(), String> { + let path = repo_path(&format!( + "formal/{}-smoke.itf.json", + if profile == "effects" { + "effects" + } else { + profile + } + )); + let prepared = coordinator.prepare(profile, &path, None)?; + let mut driver = Driver::new(prepared.fixture.clone()); + driver.skip_settle = skip_settle; + let result = { + let cell = std::cell::RefCell::new(&mut driver); + let mut apply = |input: &Value| cell.borrow_mut().apply(input); + let mut observation = || cell.borrow().observation(); + let mut wall = || cell.borrow().observation_wall_ms(); + let mut receipt = || cell.borrow().receipt(); + coordinator.execute( + &prepared, + &mut apply, + &mut observation, + &mut wall, + Some(&mut receipt), + &mut [], + ) + }; + driver.close(); + result +} + +#[test] +fn unsettled_observations_fail_every_behavior_smoke_history() { + install_panic_hook(); + let mut coordinator = Coordinator::spawn().expect("coordinator"); + for profile in BEHAVIOR_SMOKE { + replay(&mut coordinator, profile, false) + .unwrap_or_else(|e| panic!("{profile} settled replay must pass: {e}")); + let unsettled = replay(&mut coordinator, profile, true); + assert!( + unsettled.is_err(), + "{profile}: an unsettled observation was accepted" + ); + let message = unsettled.unwrap_err(); + assert!( + message.contains("Settlement violation:") + && !message.contains("expected:") + && !message.contains("actual:"), + "{profile}: control did not fail its own settlement contract: {message}" + ); + } + coordinator.finish().expect("coordinator exit"); +} + +#[test] +fn receipt_detects_runnable_work_without_observation_changes() { + let mut driver = Driver::new(json!({})); + let before = driver.observation(); + let clock = driver.observation_wall_ms(); + let finished = std::rc::Rc::new(std::cell::Cell::new(false)); + let flag = finished.clone(); + driver.exec.spawn(async move { flag.set(true) }); + driver.skip_settle = true; + driver + .apply(&json!({"op":"faults", "value":{}})) + .expect("command"); + assert!(finished.get(), "verification drain must run ready work"); + assert_eq!(driver.observation(), before, "work has no observed effects"); + assert_eq!( + driver.observation_wall_ms(), + clock, + "draining must consume no time" + ); + assert!( + driver.receipt()["runnable"].as_u64().unwrap() > 0, + "the receipt must detect actual polls even when observations and timers do not change" + ); + driver.close(); +} diff --git a/rust/tests/shadow_write_deadline.rs b/rust/tests/shadow_write_deadline.rs new file mode 100644 index 00000000..33de6875 --- /dev/null +++ b/rust/tests/shadow_write_deadline.rs @@ -0,0 +1,172 @@ +//! A prepared shadow fill may wait in the runtime before its adapter is invoked. +use std::future::ready; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use dialcache::observe::{Layer, ShadowOutcome}; +use dialcache::testing::{StepRuntime, TestExecutor, VirtualClock, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, Clock, DialCache, Event, Identity, InvalidateRequest, MissReason, Observer, + Operation, Policy, ReadContext, ReadRequest, ReadResult, Remote, Runtime, ShadowPolicy, + SourceBudget, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; + +struct GatedRuntime { + step: Arc, + hold: AtomicBool, + pending: Mutex>>, +} + +impl Runtime for GatedRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + if self.hold.load(Ordering::SeqCst) { + self.pending.lock().push(task); + } else { + self.step.spawn(task); + } + } + + fn defer(&self, task: BoxFuture<'static, ()>) { + self.step.defer(task); + } + + fn sleep(&self, delay: Duration) -> BoxFuture<'static, ()> { + self.step.sleep(delay) + } +} + +impl GatedRuntime { + fn release(&self) { + self.hold.store(false, Ordering::SeqCst); + for task in std::mem::take(&mut *self.pending.lock()) { + self.step.spawn(task); + } + } +} + +struct RecordingRemote { + clock: Arc, + writes: Mutex>, +} + +impl Remote for RecordingRemote { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(ready(Ok(ReadResult::miss(MissReason::ValueAbsent)))) + } + + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.writes.lock().push(self.clock.elapsed()); + Box::pin(ready(Ok(()))) + } + + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(ready(Ok(()))) + } +} + +struct Events { + runtime: Arc, + outcomes: Mutex>, +} + +impl Observer for Events { + fn observes_shadow_outcomes(&self) -> bool { + true + } + + fn observe(&self, event: &Event) { + match event { + Event::StoredSize { labels, .. } if labels.layer == Layer::RemoteShadow => { + self.runtime.hold.store(true, Ordering::SeqCst); + } + Event::ShadowValidation { outcome, .. } => self.outcomes.lock().push(*outcome), + _ => {} + } + } +} + +fn invoke(executor: &mut TestExecutor, cache: &DialCache) { + let cache = cache.clone(); + let value = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load( + request.scope(), + Operation::::new(Identity::new("thing", "one", "QueuedWrite")) + .policy( + Policy::default() + .remote_ttl_sec(60) + .remote_ramp(0.0) + .shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: None, + }), + ) + .budget(SourceBudget::Millis(5)), + |_| ready(Ok(7)), + ) + .await + .expect("shadow scheduling must preserve source success") + }); + assert_eq!(*value, 7); +} + +#[test] +fn shadow_writes_start_only_before_the_deadline_even_when_task_starts_are_delayed() { + for (delay, deliver) in [(0, false), (5, false), (5, true), (10, true)] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let runtime = Arc::new(GatedRuntime { + step: executor.runtime.clone(), + hold: AtomicBool::new(false), + pending: Mutex::new(Vec::new()), + }); + let remote = Arc::new(RecordingRemote { + clock: executor.clock.clone(), + writes: Mutex::new(Vec::new()), + }); + let events = Arc::new(Events { + runtime: runtime.clone(), + outcomes: Mutex::new(Vec::new()), + }); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(runtime.clone()) + .remote_arc(remote.clone()) + .observer_arc(events.clone()) + .shadow_max_in_flight(1) + .build() + .unwrap(); + invoke(&mut executor, &cache); + executor.advance(delay, deliver); + runtime.release(); + executor.drain(); + + let writes = remote.writes.lock().clone(); + assert!( + writes.iter().all(|at| *at < Duration::from_millis(5)), + "adapter called after shadow deadline: {writes:?}, delay={delay}, timers={deliver}" + ); + if delay == 0 { + assert_eq!(writes.len(), 1, "control must actually fill"); + } + assert_eq!( + *events.outcomes.lock(), + vec![if writes.is_empty() { + ShadowOutcome::Timeout + } else { + ShadowOutcome::Filled + }] + ); + + // The completed/skipped raw write must release the only shadow slot. + let before = writes.len(); + invoke(&mut executor, &cache); + runtime.release(); + executor.drain(); + assert_eq!(remote.writes.lock().len(), before + 1); + assert_eq!(events.outcomes.lock().last(), Some(&ShadowOutcome::Filled)); + } +} diff --git a/rust/tests/tokio_runtime.rs b/rust/tests/tokio_runtime.rs new file mode 100644 index 00000000..2acaed60 --- /dev/null +++ b/rust/tests/tokio_runtime.rs @@ -0,0 +1,183 @@ +//! The production tokio path: the default runtime captured at build time, +//! coalescing under a multi-thread scheduler, source deadlines on tokio +//! timers, and the cancellation contracts the deterministic harness cannot +//! exercise (dropped `enable` futures, dropped callers). + +#![cfg(feature = "tokio")] + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use dialcache::{DialCache, Identity, Operation, Policy, Scope, SourceBudget, TokioRuntime}; + +fn operation(id: &str) -> Operation { + Operation::::new(Identity::new("thing", id, "TokioSmoke")) + .policy(Policy::default().local_ttl_sec(60)) +} + +fn source( + calls: &Arc, + delay: Duration, + value: u64, +) -> impl Fn(Scope) -> futures::future::BoxFuture<'static, Result> + + Send + + Sync + + 'static { + let calls = calls.clone(); + move |_| { + let calls = calls.clone(); + Box::pin(async move { + tokio::time::sleep(delay).await; + calls.fetch_add(1, Ordering::SeqCst); + Ok(value) + }) + } +} + +#[test] +fn building_outside_a_tokio_context_is_a_configuration_error() { + let error = match DialCache::builder().build() { + Err(error) => error, + Ok(_) => panic!("built a default runtime without a tokio context"), + }; + assert!( + error.to_string().contains("tokio runtime"), + "unexpected error: {error}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_callers_coalesce_and_later_callers_hit() { + let cache = DialCache::builder().build().expect("built inside tokio"); + let calls = Arc::new(AtomicUsize::new(0)); + let request = cache.enable_guard(); + let mut pending = Vec::new(); + for _ in 0..8 { + let cache = cache.clone(); + let scope = request.scope().clone(); + let load = source(&calls, Duration::from_millis(20), 7); + pending.push(tokio::spawn(async move { + cache.get_or_load(&scope, operation("one"), load).await + })); + } + for handle in pending { + let value = handle.await.expect("task").expect("value"); + assert_eq!(*value, 7); + } + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "callers were not coalesced" + ); + let again = cache + .get_or_load( + request.scope(), + operation("one"), + source(&calls, Duration::ZERO, 8), + ) + .await + .expect("hit"); + assert_eq!(*again, 7); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "a local hit called the source" + ); +} + +#[tokio::test] +async fn a_source_deadline_returns_a_timeout_and_leaves_the_source_running() { + let cache = DialCache::builder() + .runtime(TokioRuntime::from_handle(tokio::runtime::Handle::current())) + .build() + .expect("explicit handle"); + let calls = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let settled = Arc::new(tokio::sync::Notify::new()); + let request = cache.enable_guard(); + // The source waits for an explicit release, so the deadline result never + // races a second timer on a slow runner. + let load = { + let (calls, release, settled) = (calls.clone(), release.clone(), settled.clone()); + move |_: Scope| { + let (calls, release, settled) = (calls.clone(), release.clone(), settled.clone()); + Box::pin(async move { + release.notified().await; + calls.fetch_add(1, Ordering::SeqCst); + settled.notify_one(); + Ok::(1) + }) as futures::future::BoxFuture<'static, Result> + } + }; + let error = cache + .get_or_load( + request.scope(), + operation("slow").budget(SourceBudget::Millis(20)), + load, + ) + .await + .expect_err("deadline"); + assert!(error.is_fallback_timeout(), "unexpected error: {error}"); + assert_eq!(calls.load(Ordering::SeqCst), 0); + release.notify_one(); + tokio::time::timeout(Duration::from_secs(5), settled.notified()) + .await + .expect("the deadline cancelled the source"); + assert_eq!(calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn dropping_the_enable_future_closes_its_scope() { + let cache = DialCache::builder().build().expect("built inside tokio"); + let (send, receive) = tokio::sync::oneshot::channel::(); + let inner = cache.clone(); + let outcome = tokio::time::timeout( + Duration::from_millis(10), + cache.enable(|scope| async move { + assert!(inner.is_enabled(&scope)); + let _ = send.send(scope); + std::future::pending::<()>().await; + }), + ) + .await; + assert!(outcome.is_err(), "the callback completed"); + let retained = receive.await.expect("scope handed out"); + assert!( + !cache.is_enabled(&retained), + "a dropped enable future left its scope live" + ); +} + +#[tokio::test] +async fn dropping_the_caller_future_does_not_cancel_the_execution() { + let cache = DialCache::builder().build().expect("built inside tokio"); + let calls = Arc::new(AtomicUsize::new(0)); + let request = cache.enable_guard(); + let abandoned = tokio::time::timeout( + Duration::from_millis(5), + cache.get_or_load( + request.scope(), + operation("kept"), + source(&calls, Duration::from_millis(50), 3), + ), + ) + .await; + assert!(abandoned.is_err(), "the source finished within 5 ms"); + tokio::time::sleep(Duration::from_millis(150)).await; + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "the source did not complete" + ); + let value = cache + .get_or_load( + request.scope(), + operation("kept"), + source(&calls, Duration::ZERO, 4), + ) + .await + .expect("published value"); + assert_eq!(*value, 3, "the abandoned execution did not publish"); + assert_eq!(calls.load(Ordering::SeqCst), 1); +} diff --git a/test/formal-exploration.test.ts b/test/formal-exploration.test.ts index f1ba6ce5..66e6dd39 100644 --- a/test/formal-exploration.test.ts +++ b/test/formal-exploration.test.ts @@ -43,6 +43,16 @@ function tsReport(directory: string, failure?: string) { testResults: [{ name: `${directory}/test/formal-features.test.ts`, status: failure ? "failed" : "passed", startTime: 2, endTime: 3, message: "", assertionResults }] }; } +function rustReport(failure?: string) { + const records: Record[] = [{ schemaVersion: 1, kind: "start", implementation: "rust", startedAt: 10 }]; + for (const [index, entry] of inventory.entries()) { + const failed = entry.id === failure; + records.push({ kind: "case", id: nativeBinding(entry, "rust"), status: failed ? "failed" : "passed", startedAt: 11 + index, finishedAt: 12 + index, + ...(failed ? { message: "Observation mismatch\nexpected: 1\nactual: 2" } : {}) }); + } + records.push({ kind: "finish", status: failure ? "failed" : "passed", finishedAt: 20, cases: inventory.length, failed: failure ? 1 : 0 }); + return records.map(record => JSON.stringify(record)).join("\n"); +} function goReport(failure?: string) { const events: Record[] = []; const event = (Action: string, Test?: string) => events.push({ Action, ...(Test ? { Test } : {}), Package: packageName, Time: new Date(10 + events.length).toISOString() }); @@ -92,7 +102,7 @@ function savedFixture(directory: string) { writeFileSync(options.directory + '/.formal-traces/saved-runner.json', JSON.stringify(plan)); // A saved run's evaluator also has to leave a completed witness report behind. writeFileSync(options.directory + '/.formal-traces/witness-report.json', ${JSON.stringify(JSON.stringify(completedWitnessReport("0x2a", ["effects"])))}); - return [{ language: 'typescript', status: 'passed' }, { language: 'go', status: 'passed' }]; + return [{ language: 'typescript', status: 'passed' }, { language: 'go', status: 'passed' }, { language: 'rust', status: 'passed' }]; }`, "formal/validation.mjs": `import { mkdirSync, writeFileSync } from 'node:fs'; export function checkPrerequisites(target, { directory }) { @@ -142,10 +152,11 @@ describe("isolated exploratory validation", () => { ["formal/check-go-parity.mjs"], ]); const replays = plan.filter(step => step.nativeReport); - expect(replays.map(step => step.nativeReport)).toEqual(["typescript", "go"]); + expect(replays.map(step => step.nativeReport)).toEqual(["typescript", "go", "rust"]); for (const step of replays) expect(step.env?.DIALCACHE_FEATURE_TRACE_DIR).toBe(`${directory}/.formal-traces/features`); - expect(plan.filter(step => step.explorationContext).map(step => step.explorationContext)).toEqual(["typescript", "go"]); - expect(plan.some(step => step.args?.includes("formal/check-go-replay.mjs") || step.args?.includes("formal/conformance-adapters.mjs"))).toBe(false); + expect(plan.filter(step => step.explorationContext).map(step => step.explorationContext)).toEqual(["typescript", "go", "rust"]); + expect(plan.some(step => step.args?.includes("formal/check-go-replay.mjs") || step.args?.includes("formal/check-rust-replay.mjs") + || step.args?.includes("formal/conformance-adapters.mjs"))).toBe(false); expect(plan.some(step => step.args?.[0] === "formal/conformance.mjs" && step.args[1] === "check")).toBe(false); }); @@ -172,8 +183,9 @@ describe("isolated exploratory validation", () => { } finally { rmSync(directory, { recursive: true, force: true }); } }); - it.each(["typescript", "go"])("classifies exact %s witness leaves separately from replay failures", language => { - const native = (failure?: string) => language === "typescript" ? JSON.stringify(tsReport("/snapshot", failure)) : goReport(failure); + it.each(["typescript", "go", "rust"])("classifies exact %s witness leaves separately from replay failures", language => { + const native = (failure?: string) => language === "typescript" ? JSON.stringify(tsReport("/snapshot", failure)) + : language === "go" ? goReport(failure) : rustReport(failure); expect(nativeExplorationResult(language, native(), context(language), "/snapshot", packageName).status).toBe("passed"); expect(nativeExplorationResult(language, native("witness/recovery"), context(language), "/snapshot", packageName)).toMatchObject({ status: "witness-check-failure", witnessFailures: ["witness/recovery"], caseFailures: [], @@ -250,7 +262,7 @@ describe("isolated exploratory validation", () => { await expect(explore("42", { directory, run: async () => { const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); writeFileSync(join(output, "workspace/rule.qnt"), "changed"); - return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }]; + return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }, { language: "rust", status: "passed" }]; } })).rejects.toThrow(/changed/); const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); expect(readFileSync(join(directory, "rule.qnt"), "utf8")).toBe("original"); @@ -267,12 +279,12 @@ describe("isolated exploratory validation", () => { writeFileSync(join(directory, ".gitignore"), ".formal-traces/\n"); writeWitnessInventory(directory, selectedProfiles("all")); await expect(explore("42", { directory, run: async () => [ - { language: "typescript", status: "witness-check-failure" }, { language: "go", status: "witness-check-failure" }, + { language: "typescript", status: "witness-check-failure" }, { language: "go", status: "witness-check-failure" }, { language: "rust", status: "witness-check-failure" }, ] })).rejects.toThrow(/witness-check-failure/); const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); expect(JSON.parse(readFileSync(join(output, "report.json"), "utf8"))).toMatchObject({ kind: "exploration", acceptance: false, status: "witness-check-failure", sourcesUnchanged: true, - native: [{ language: "typescript" }, { language: "go" }], + native: [{ language: "typescript" }, { language: "go" }, { language: "rust" }], }); expect(existsSync(join(output, "workspace/node_modules"))).toBe(false); } finally { rmSync(directory, { recursive: true, force: true }); } @@ -348,7 +360,7 @@ describe("isolated exploratory validation", () => { mkdirSync(join(workspace, ".formal-traces"), { recursive: true }); writeFileSync(join(workspace, ".formal-traces/witness-report.json"), JSON.stringify(witnesses)); replays = 2; - return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }]; + return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }, { language: "rust", status: "passed" }]; } })).rejects.toThrow(/coverage-gate-failure[\s\S]*reply:13 reached by 1 sampled histories/); expect(replays).toBe(2); const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); @@ -374,7 +386,7 @@ describe("isolated exploratory validation", () => { const workspace = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!, "workspace"); mkdirSync(join(workspace, ".formal-traces"), { recursive: true }); if (contents !== undefined) writeFileSync(join(workspace, ".formal-traces/witness-report.json"), contents); - return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }]; + return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }, { language: "rust", status: "passed" }]; } })).rejects.toThrow(message); const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); expect(JSON.parse(readFileSync(join(output, "report.json"), "utf8")).status).toBe("infrastructure-failure"); @@ -401,7 +413,7 @@ describe("isolated exploratory validation", () => { const workspace = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!, "workspace"); mkdirSync(join(workspace, ".formal-traces"), { recursive: true }); writeFileSync(join(workspace, ".formal-traces/witness-report.json"), JSON.stringify(witnesses)); - return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }]; + return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }, { language: "rust", status: "passed" }]; } }); expect(JSON.parse(readFileSync(join(output, "report.json"), "utf8"))).toMatchObject({ status: "passed", witnesses }); } finally { rmSync(directory, { recursive: true, force: true }); } diff --git a/test/formal-mutation-shards.test.ts b/test/formal-mutation-shards.test.ts index 6fea28f3..d940affd 100644 --- a/test/formal-mutation-shards.test.ts +++ b/test/formal-mutation-shards.test.ts @@ -24,7 +24,7 @@ const shared = await import(new URL("../formal/mutation-reports.mjs", import.met partitionMutations(mutations: T[], shard: Shard): T[]; selectMutations(mutations: T[], selection: Selection): T[]; selectionDirectory(output: string, selection: Selection): string; - fingerprintFiles(directory: string, paths: string[]): { files: number; sha256: string }; + fingerprintFiles(directory: string, paths: string[], options?: { exclude?: string[] }): { files: number; sha256: string }; requiredDetectionRegressions(entries: CatalogEntry[], results: Mutation[]): string[]; goDetection(mutations: Mutation[]): Record; typescriptDetection(mutations: Mutation[], cases: { id: string; vectors: unknown[] }[]): Record; @@ -33,7 +33,7 @@ const shared = await import(new URL("../formal/mutation-reports.mjs", import.met portableCohort(generated: Cohort, fixed: Cohort): Cohort; noncompilingResult(mutation: CatalogEntry, cohorts: string[], reason: string): Mutation; gateDetections(language: Language, report: Report, entries: CatalogEntry[], options?: { directory?: string; summarize?: boolean }): void; - languages: { ts: Language; go: Language }; + languages: { ts: Language; go: Language; rust: Language & { exclude: string[] } }; }; const merge = await import(new URL("../formal/merge-mutation-reports.mjs", import.meta.url).href) as { canonical(value: unknown): string; @@ -601,13 +601,69 @@ describe("mutation shard merge over a shard directory", () => { writeShards(lost, join(directory, ".formal-traces/go-semantic/shards")); expect(() => mergeMutationReports("go", { directory })).toThrow(new RegExp(`Lost required detections: ${survivor.id}/generated`)); expect(readReport(".formal-traces/go-semantic/report.json")).toMatchObject({ complete: false, requiredDetectionRegressions: [`${survivor.id}/generated`], error: expect.stringContaining(`${survivor.id}/generated`) }); - expect(() => mergeMutationReports("rust", { directory })).toThrow(/Expected language ts or go/); + expect(() => mergeMutationReports("zig", { directory })).toThrow(/Expected language ts, go or rust/); expect(() => mergeMutationReports("go", { directory, shardsDirectory: "nowhere" })).toThrow(/no shard directory/); expect(readReport(".formal-traces/go-semantic/report.json")).toMatchObject({ complete: false, error: expect.stringContaining("no shard directory") }); // The command line rejects a bad language before touching any report directory. const { spawnSync } = await import("node:child_process"); - const usage = spawnSync(process.execPath, [new URL("../formal/merge-mutation-reports.mjs", import.meta.url).pathname, "rust"], { encoding: "utf8" }); + const usage = spawnSync(process.execPath, [new URL("../formal/merge-mutation-reports.mjs", import.meta.url).pathname, "zig"], { encoding: "utf8" }); expect(usage.status).toBe(2); - expect(usage.stderr).toMatch(/Usage: node formal\/merge-mutation-reports.mjs /); + expect(usage.stderr).toMatch(/Usage: node formal\/merge-mutation-reports.mjs /); }); }); + +describe("Rust mutation language", () => { + let directory: string; + beforeEach(() => { directory = mkdtempSync(join(tmpdir(), "dialcache-rust-mutation-language-")); }); + afterEach(() => rmSync(directory, { recursive: true, force: true })); + const put = (path: string, text: string) => { mkdirSync(join(directory, path, ".."), { recursive: true }); writeFileSync(join(directory, path), text); }; + + it("fingerprints the crate sources and files without the build directory", () => { + for (const path of ["formal/rust-mutations.json", "test/a.test.ts", "src/a.ts", "go/redis_adapter.go", "rust/Cargo.toml", "rust/Cargo.lock", "rust/src/lib.rs", "rust/tests/conformance.rs"]) put(path, path); + const clean = fingerprintFiles(directory, languages.rust.inputs, { exclude: languages.rust.exclude }); + put("rust/target/release/deps/libdialcache.rlib", "build output"); + put("rust/target/semantic/report.json", "{}"); + expect(fingerprintFiles(directory, languages.rust.inputs, { exclude: languages.rust.exclude })).toEqual(clean); + expect(clean.files).toBe(8); + // Without the exclusion the build output would count, so the exclusion is what keeps a checkout and its workspace copy equal. + expect(fingerprintFiles(directory, languages.rust.inputs).files).toBe(10); + expect(fingerprintFiles(directory, ["rust/Cargo.toml"])).toEqual(fingerprintFiles(directory, ["rust/Cargo.toml"], { exclude: ["rust/target"] })); + expect(fingerprintFiles(directory, ["rust/Cargo.toml"]).files).toBe(1); + }); + + it("merges the separate Rust catalog without claiming shared model-boundary evidence", () => { + const catalogBytes = readRepo("formal/rust-mutations.json"); + const catalog = JSON.parse(catalogBytes.toString()) as Catalog; + for (const path of languages.rust.inputs) mkdirSync(join(directory, path), { recursive: true }); + writeFileSync(join(directory, languages.rust.catalog), catalogBytes); + const catalogSha256 = sha256(catalogBytes); + const inputs = fingerprintFiles(directory, languages.rust.inputs, { exclude: languages.rust.exclude }); + const mutations = catalog.mutations.map(entry => { + const { boundary: _boundary, ...result } = goMutation(entry); + return result; + }); + const single = { schemaVersion: 1, complete: true, startedAt: "2026-09-21T00:00:00.000Z", elapsedSeconds: 1, + cargo: "cargo 1.98.1", catalogSha256, inputs, baselines: goBaselines(), mutations, + scope: { catalog: "rust-native", modelBoundaryEvidence: false, unmappedSharedMutations: ["M14"] } } as Report; + const shards = goShardReports(single, 3); + for (const report of shards) { + const path = join(directory, languages.rust.output, "shards", `${report.shard!.index}-of-3`); + mkdirSync(path, { recursive: true }); + writeFileSync(join(path, "report.json"), JSON.stringify(report)); + } + const merged = mergeMutationReports("rust", { directory }); + expect(merged.complete).toBe(true); + expect(merged.detection).toEqual(goDetection(mutations)); + expect(merged.requiredDetectionRegressions).toEqual([]); + expect(merged.scope).toEqual(single.scope); + const markdown = readFileSync(join(directory, languages.rust.output, "report.md"), "utf8"); + expect(markdown).toContain("# Rust semantic mutation measurement"); + expect(markdown).toContain("| M01 | C45.maximum-age-exclusive | survived | detected | detected | detected |"); + expect(markdown).toContain("model-challenge boundary coverage is not measured"); + expect(markdown).toContain("Merged from 3 shards"); + shards[0]!.mutations[0]!.cohorts.generated!.state = "survived"; + writeFileSync(join(directory, languages.rust.output, "shards/1-of-3/report.json"), JSON.stringify(shards[0])); + expect(() => mergeMutationReports("rust", { directory })).toThrow(/M01\/generated/); + }); +}); + diff --git a/test/formal-rust-replay.test.ts b/test/formal-rust-replay.test.ts new file mode 100644 index 00000000..d30447ed --- /dev/null +++ b/test/formal-rust-replay.test.ts @@ -0,0 +1,218 @@ +import { describe, expect, it } from "vitest"; + +type Entry = { id: string; category: string; profile?: string; path?: string; name?: string; feature?: string; group?: string }; +type Summary = { + schemaVersion: number; implementation: string; status: string; generated: Record; generatedTraces: number; + quintRegressions: Record; quintRegressionTraces: number; fixedScenarios: number; protocolVectors: number; + witnessProfiles: string[]; executedCases: number; +}; +type Step = { label: string; command?: string; args?: string[]; cwd?: string; env?: NodeJS.ProcessEnv; stdoutFile?: string; remove?: string[] }; +type Record_ = Record; + +const { checkRustReplay } = await import(new URL("../formal/check-rust-replay.mjs", import.meta.url).href) as { + checkRustReplay(report: string, inventory?: Entry[]): Summary; +}; +const { parseRustReport, adaptReport, nativeBinding } = await import(new URL("../formal/conformance-adapters.mjs", import.meta.url).href) as { + parseRustReport(text: string, inventory: Entry[]): { startedAt: number; finishedAt: number; results: Array<{ id: string; status: string }> }; + adaptReport(language: string, text: string, context: unknown): unknown; + nativeBinding(entry: Entry, language: string): unknown; +}; +const { conformanceInventory, defaultSources } = await import(new URL("../formal/conformance.mjs", import.meta.url).href) as { + conformanceInventory(): Entry[]; + defaultSources(language: string): string[]; +}; +const { validationPlan, checkPrerequisites } = await import(new URL("../formal/validation.mjs", import.meta.url).href) as { + validationPlan(target: string, options?: { directory?: string; environment?: NodeJS.ProcessEnv }): Step[]; + checkPrerequisites(target: string, options?: { directory?: string; environment?: NodeJS.ProcessEnv; nodeVersion?: string }): void; +}; + +const inventory = conformanceInventory(); +const startedAt = 1_800_000_000_000; +const start = (): Record_ => ({ schemaVersion: 1, kind: "start", implementation: "rust", startedAt }); +const caseRecord = (id: string, status = "passed", extra: Record_ = {}): Record_ => ({ kind: "case", id, status, startedAt: startedAt + 1, finishedAt: startedAt + 2, ...extra }); +const finish = (cases: number, failed = 0, status = failed ? "failed" : "passed"): Record_ => ({ kind: "finish", status, finishedAt: startedAt + 3, cases, failed }); +const encode = (records: unknown[]): string => records.map(record => JSON.stringify(record)).join("\n") + "\n"; +const completed = (): Record_[] => [start(), ...inventory.map(entry => caseRecord(entry.id)), finish(inventory.length)]; +const withCases = (mutate: (records: Record_[]) => void): string => { + const records = completed(); + mutate(records); + return encode(records); +}; + +describe("Rust replay report gate", () => { + it("binds every inventory entry to its own id", () => { + for (const entry of inventory) expect(nativeBinding(entry, "rust")).toBe(entry.id); + expect(nativeBinding(inventory[0]!, "go")).not.toBe(inventory[0]!.id); + expect(() => nativeBinding(inventory[0]!, "zig")).toThrow(/TypeScript, Go and Rust/); + }); + + it("accepts a complete report and summarizes it by category and profile", () => { + const summary = checkRustReplay(encode(completed()), inventory); + const count = (category: string) => inventory.filter(entry => entry.category === category).length; + expect(summary).toMatchObject({ schemaVersion: 1, implementation: "rust", status: "pass", executedCases: inventory.length, + generatedTraces: count("sampled"), quintRegressionTraces: count("regression"), fixedScenarios: count("scenario"), protocolVectors: count("protocol") }); + expect(summary.witnessProfiles).toEqual(inventory.filter(entry => entry.category === "witness").map(entry => entry.profile).sort()); + expect(Object.values(summary.generated).reduce((sum, n) => sum + n, 0)).toBe(summary.generatedTraces); + expect(summary.generated.core).toBeGreaterThan(0); + expect(summary.quintRegressions.core).toBe(inventory.filter(entry => entry.category === "regression" && entry.profile === "core").length); + for (const profile of Object.keys(summary.generated)) expect(summary.quintRegressions, profile).toHaveProperty(profile); + }); + + it("requires new profiles and source-derived exported regressions through the default inventory", () => { + for (const profile of ["dark-layers", "shadow-read-deadlines"]) { + for (const category of ["sampled", "regression", "witness"]) { + expect(inventory.some(entry => entry.profile === profile && entry.category === category), `${profile}/${category}`).toBe(true); + } + const oldCases = inventory.filter(entry => entry.profile !== profile); + const oldReport = encode([start(), ...oldCases.map(entry => caseRecord(entry.id)), finish(oldCases.length)]); + expect(() => checkRustReplay(oldReport)).toThrow(`Missing passed Rust replay case: sampled/${profile}/0`); + } + const historiesOnly = inventory.filter(entry => entry.category !== "regression"); + const missingRegressions = encode([start(), ...historiesOnly.map(entry => caseRecord(entry.id)), finish(historiesOnly.length)]); + expect(() => checkRustReplay(missingRegressions)).toThrow(/Missing passed Rust replay case: regression\//); + }); + + it("rejects a missing, duplicate or failed case", () => { + const missing = inventory.at(-1)!.id; + expect(() => checkRustReplay(withCases(records => { records.splice(records.length - 2, 1); (records.at(-1) as { cases: number }).cases--; }), inventory)) + .toThrow(`Missing passed Rust replay case: ${missing}`); + expect(() => checkRustReplay(withCases(records => { records.splice(1, 0, caseRecord(inventory[0]!.id)); (records.at(-1) as { cases: number }).cases++; }), inventory)) + .toThrow(`Duplicate Rust replay case: ${inventory[0]!.id}`); + expect(() => checkRustReplay(withCases(records => { records[1] = caseRecord(inventory[0]!.id, "failed", { message: "observation mismatch at step 3" }); }), inventory)) + .toThrow(`Rust replay failed: ${inventory[0]!.id} (observation mismatch at step 3)`); + expect(() => checkRustReplay(withCases(records => { records[1] = caseRecord(inventory[0]!.id, "skipped"); }), inventory)).toThrow(/Invalid Rust case status/); + }); + + it("rejects inventory drift, smoke histories in a full replay and unknown ids", () => { + const add = (id: string) => withCases(records => { records.splice(1, 0, caseRecord(id)); (records.at(-1) as { cases: number }).cases++; }); + expect(() => checkRustReplay(add("sampled/core/999999"), inventory)).toThrow("Unexpected Rust replay case (inventory drift): sampled/core/999999"); + expect(() => checkRustReplay(add("witness/nonexistent"), inventory)).toThrow(/inventory drift/); + expect(() => checkRustReplay(add("smoke/core/conformance-smoke.itf.json"), inventory)).toThrow("Smoke history in a full Rust replay: smoke/core/conformance-smoke.itf.json"); + expect(() => checkRustReplay(add("bench/unrelated"), inventory)).toThrow("Unknown Rust replay case: bench/unrelated"); + }); + + it("rejects an incomplete report, a non-passed finish, inconsistent totals and malformed lines", () => { + expect(() => checkRustReplay(withCases(records => { records.pop(); }), inventory)).toThrow("Rust replay is incomplete: missing finish record"); + expect(() => checkRustReplay(withCases(records => { records[records.length - 1] = finish(inventory.length, 0, "failed"); }), inventory)).toThrow("Rust replay finished with status failed"); + expect(() => checkRustReplay(withCases(records => { records[records.length - 1] = finish(inventory.length, 1, "passed"); }), inventory)).toThrow(/finish totals disagree/); + expect(() => checkRustReplay(withCases(records => { records[records.length - 1] = finish(inventory.length - 1); }), inventory)).toThrow(/finish totals disagree/); + expect(() => checkRustReplay(withCases(records => { records.push(caseRecord(inventory[0]!.id)); }), inventory)).toThrow(/continues after the finish record/); + expect(() => checkRustReplay(withCases(records => { records.shift(); }), inventory)).toThrow(/precedes the start record/); + expect(() => checkRustReplay(withCases(records => { records.splice(1, 0, start()); }), inventory)).toThrow(/Duplicate or misplaced Rust start record/); + expect(() => checkRustReplay(withCases(records => { records[0] = { ...start(), implementation: "go" }; }), inventory)).toThrow(/Unsupported Rust start record/); + const [firstLine, ...rest] = encode(completed()).split("\n"); + expect(() => checkRustReplay([firstLine, "{not json", ...rest].join("\n"), inventory)).toThrow("Invalid Rust JSON record at line 2"); + expect(() => checkRustReplay(withCases(records => { records[1] = { kind: "note", id: "x" }; }), inventory)).toThrow(/Unsupported Rust replay record kind: note/); + expect(() => checkRustReplay("", inventory)).toThrow(/Empty Rust replay report/); + }); + + it("adapts the execution window and results only from a report that passed the gate", () => { + const parsed = parseRustReport(encode(completed()), inventory); + expect(parsed).toMatchObject({ startedAt, finishedAt: startedAt + 3 }); + expect(parsed.results).toEqual(inventory.map(entry => ({ id: entry.id, status: "passed" }))); + expect(() => parseRustReport(withCases(records => { records.pop(); }), inventory)).toThrow(/missing finish/); + expect(() => adaptReport("rust", encode(completed()), { schemaVersion: 1, language: "go" })).toThrow(/context/); + }); + + it("binds the crate sources, manifests and lockfile with the shared fixtures and witness evidence, never the build directory", () => { + const sources = defaultSources("rust"); + expect(new Set(sources).size).toBe(sources.length); + expect(sources).toContain("rust/Cargo.toml"); + expect(sources).toContain("rust/Cargo.lock"); + expect(sources).toContain("rust/rust-toolchain.toml"); + expect(sources).toContain("rust/src/lib.rs"); + expect(sources).toContain("rust/tests/conformance.rs"); + expect(sources.some(path => path.startsWith("rust/target/"))).toBe(false); + expect(sources.filter(path => path.startsWith("rust/")).every(path => /\.(rs|toml|lock)$/.test(path))).toBe(true); + const go = defaultSources("go"); + for (const path of go.filter(path => path.startsWith(".formal-traces/go-parity-witnesses/") || path.startsWith("src/") || path.startsWith("test/"))) { + expect(sources, path).toContain(path); + } + }); +}); + +describe("Rust validation lanes", () => { + const directory = "/checkout"; + it("checks formatting, clippy and default tests from inside the crate so rustup honors its toolchain pin", () => { + expect(validationPlan("check-rust", { directory }).map(step => [step.command, ...step.args!])).toEqual([ + ["cargo", "fmt", "--check"], + ["cargo", "clippy", "--all-targets", "--all-features", "--", "-D", "warnings"], + ["cargo", "test", "--all-features"], + ]); + for (const step of validationPlan("check-rust", { directory })) { + expect(step.env, step.label).toBeUndefined(); + expect(step.cwd, step.label).toBe("rust"); + } + expect(validationPlan("check", { directory })).toEqual(["check-ts", "check-go", "check-rust", "docs", "audit"].flatMap(target => validationPlan(target, { directory }))); + }); + + it("replays the complete corpus against the shared evidence and adapts the harness report into a completion", () => { + const plan = validationPlan("formal-rust", { directory }); + expect(plan[0]).toEqual({ label: "Invalidate prior rust completion", remove: [".formal-traces/rust-completion.json"] }); + expect(plan[1]!.args).toEqual(["formal/conformance.mjs", "prepare", "rust", ".formal-traces/rust-context.json"]); + const replay = plan[2]!; + expect([replay.command, ...replay.args!]).toEqual(["cargo", "test", "--release", "--all-features", "--test", "conformance"]); + expect(replay.cwd).toBe("rust"); + expect(replay.env).toEqual({ + DIALCACHE_MBT_TRACE_DIR: "/checkout/.formal-traces/conformance", + DIALCACHE_EFFECTS_TRACE_DIR: "/checkout/.formal-traces/effects", + DIALCACHE_FEATURE_TRACE_DIR: "/checkout/.formal-traces/features", + DIALCACHE_WITNESS_EVIDENCE_DIR: "/checkout/.formal-traces/go-parity-witnesses", + DIALCACHE_RUST_REPORT: "/checkout/.formal-traces/rust-replay.jsonl", + }); + expect(replay.stdoutFile).toBeUndefined(); + expect(plan[3]).toMatchObject({ args: ["formal/check-rust-replay.mjs"], stdoutFile: ".formal-traces/rust-replay-summary.json" }); + expect(plan[4]).toMatchObject({ args: ["formal/conformance-adapters.mjs", "rust", ".formal-traces/rust-replay.jsonl", ".formal-traces/rust-context.json"], stdoutFile: ".formal-traces/rust-completion.json" }); + expect(plan.at(-1)!.args).toEqual(["formal/conformance.mjs", "check", ".formal-traces/rust-completion.json", ".formal-traces/rust-context.json"]); + expect(plan).toHaveLength(6); + // Go's parity ledger is Go-only; Rust neither checks it nor touches the other ports' reports. + expect(plan.some(step => step.args?.[0] === "formal/check-go-parity.mjs")).toBe(false); + expect(plan.some(step => step.args?.some(argument => /\.formal-traces\/(ts|go)-/.test(argument)))).toBe(false); + const go = validationPlan("formal-go", { directory }).find(step => step.command === "go" && step.env)!; + for (const key of Object.keys(go.env!)) expect(replay.env![key], key).toBe(go.env![key]); + expect(validationPlan("formal", { directory }).slice(-plan.length)).toEqual(plan); + }); + + it("adds the smoke conformance run in default mode with no corpus selectors", () => { + const smoke = validationPlan("smoke", { directory }); + expect(smoke.at(-1)).toEqual({ label: "Replay committed Rust fixtures", command: "cargo", args: ["test", "--all-features", "--test", "conformance"], cwd: "rust" }); + expect(smoke.filter(step => step.command === "cargo")).toHaveLength(1); + }); + + it("runs the real-server integration binary only through the integration lane, which selects its ignored tests", () => { + const lane = validationPlan("integration-rust", { directory }); + expect(lane).toEqual([{ label: "Run Rust Redis/Valkey/Cluster integrations", command: "cargo", args: ["test", "--all-features", "--test", "redis_integration", "--", "--ignored"], cwd: "rust" }]); + expect(validationPlan("integration", { directory })).toEqual(["integration-ts", "integration-go", "integration-rust"].flatMap(target => validationPlan(target, { directory }))); + for (const target of ["check-rust", "smoke", "formal-rust"]) expect(validationPlan(target, { directory }).some(step => step.args?.includes("--ignored")), target).toBe(false); + }); + + it("probes the pinned cargo exactly for the Rust lanes", async () => { + const { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { delimiter, join } = await import("node:path"); + const temporary = mkdtempSync(join(tmpdir(), "dialcache-rust-prereq-")); + try { + for (const path of ["bin", "formal", "node_modules/typescript", "rust"]) mkdirSync(join(temporary, path), { recursive: true }); + writeFileSync(join(temporary, "package.json"), '{"packageManager":"pnpm@10.33.0"}'); + writeFileSync(join(temporary, "node_modules/typescript/package.json"), "{}"); + writeFileSync(join(temporary, "formal/generated-fixtures.lock.json"), '{"quintVersion":"0.32.0"}'); + const tool = (name: string, body: string) => { const path = join(temporary, "bin", name); writeFileSync(path, `#!${process.execPath}\n${body}\n`); chmodSync(path, 0o755); }; + tool("corepack", 'console.log("10.33.0")'); + tool("go", 'console.log("go version go1.27.1 test/test")'); + tool("quint", 'console.log("0.32.0")'); + const environment = { ...process.env, PATH: `${join(temporary, "bin")}${delimiter}${process.env.PATH ?? ""}` }; + const options = { directory: temporary, environment, nodeVersion: "v24.20.0" }; + tool("cargo", 'console.error("cargo: command not found"); process.exit(127)'); + for (const target of ["check-rust", "formal-rust", "smoke", "check", "integration-rust", "mutations-rust", "mutations", "explore"]) expect(() => checkPrerequisites(target, options), target).toThrow(/Cannot run cargo/); + for (const target of ["check-ts", "check-go", "formal-ts", "formal-go", "mutations-ts", "mutations-go", "mutations-merge-rust", "audit"]) expect(() => checkPrerequisites(target, options), target).not.toThrow(); + tool("cargo", 'console.log("cargo 1.97.0 (abcdef 2026-06-01)")'); + expect(() => checkPrerequisites("check-rust", options)).toThrow(/requires cargo 1\.98\.1; found cargo 1\.97\.0/); + tool("cargo", 'console.log("cargo 1.98.10 (abcdef 2026-06-01)")'); + expect(() => checkPrerequisites("formal-rust", options)).toThrow(/requires cargo 1\.98\.1/); + tool("cargo", 'console.log("cargo 1.98.1 (797e8a9bc 2026-08-05)")'); + for (const target of ["check-rust", "formal-rust", "smoke"]) expect(() => checkPrerequisites(target, options), target).not.toThrow(); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }); +}); diff --git a/test/formal-rust-semantic-runner.test.ts b/test/formal-rust-semantic-runner.test.ts new file mode 100644 index 00000000..c3a3da1e --- /dev/null +++ b/test/formal-rust-semantic-runner.test.ts @@ -0,0 +1,178 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const moduleUrl = new URL("../formal/measure-rust-semantics.mjs", import.meta.url).href; +type Cohort = { state: string; passed: number; failed: number; failingTests: string[]; executedTests: string[]; assertionKinds: Record; assertionEvidence: Record }; +const { evaluateCargoTestOutput, evaluateRustReport, infrastructureTestFile, rustMutationScope, rustTargetDirectory } = await import(moduleUrl) as { + evaluateCargoTestOutput(output: string, exitCode: number, expectedBinaries?: number): Cohort; + evaluateRustReport(text: string, exitCode: number, stderr?: string): Cohort; + infrastructureTestFile: RegExp; + rustMutationScope(catalog: unknown, typescript: unknown[]): { catalog: string; modelBoundaryEvidence: boolean; mappedMutations: string[]; unmappedSharedMutations: string[] }; + rustTargetDirectory(directory: string, selection: { shard: { index: number; count: number }; only?: string[] }): string; +}; + +// libtest output as `cargo test --release --no-fail-fast --lib --test tokio_runtime` prints it. +function cargoOutput(libOutcome: string, integrationOutcome: string): string { + const failed = (outcome: string) => (outcome === "FAILED" ? 1 : 0); + return [ + " Running unittests src/lib.rs (/tmp/target/release/deps/dialcache-0123)", + "", + "running 2 tests", + `test policy::tests::defaults ... ${libOutcome}`, + "test codec::tests::round_trip ... ok", + "", + `test result: ${failed(libOutcome) ? "FAILED" : "ok"}. ${2 - failed(libOutcome)} passed; ${failed(libOutcome)} failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s`, + "", + " Running tests/tokio_runtime.rs (/tmp/target/release/deps/tokio_runtime-4567)", + "", + "running 2 tests", + `test concurrent_callers_coalesce ... ${integrationOutcome}`, + "test policy::tests::defaults ... ignored", + "", + `test result: ${failed(integrationOutcome) ? "FAILED" : "ok"}. ${1 - failed(integrationOutcome)} passed; ${failed(integrationOutcome)} failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.30s`, + "", + ].join("\n"); +} + +describe("Rust ordinary cohort evaluation", () => { + it("qualifies test names by binary, drops ignored tests and survives a clean run", () => { + const cohort = evaluateCargoTestOutput(cargoOutput("ok", "ok"), 0, 2); + expect(cohort).toMatchObject({ state: "survived", passed: 3, failed: 0, failingTests: [] }); + expect(cohort.executedTests).toEqual(["src/lib.rs::policy::tests::defaults", "src/lib.rs::codec::tests::round_trip", "tests/tokio_runtime.rs::concurrent_callers_coalesce"]); + }); + + it("detects a failing test in either binary and names it", () => { + expect(evaluateCargoTestOutput(cargoOutput("FAILED", "ok"), 101, 2)).toMatchObject({ state: "detected", passed: 2, failed: 1, failingTests: ["src/lib.rs::policy::tests::defaults"] }); + expect(evaluateCargoTestOutput(cargoOutput("ok", "FAILED"), 101, 2)).toMatchObject({ state: "detected", failingTests: ["tests/tokio_runtime.rs::concurrent_callers_coalesce"] }); + }); + + it("treats a missing binary, a missing result line or an exit code that disagrees with the outcomes as infrastructure failure", () => { + expect(() => evaluateCargoTestOutput(cargoOutput("ok", "ok"), 0, 3)).toThrow(/expected 3 test binaries/); + expect(() => evaluateCargoTestOutput("running 1 test\ntest a ... ok\n", 0)).toThrow(/no libtest result line/); + expect(() => evaluateCargoTestOutput(cargoOutput("ok", "ok"), 101, 2)).toThrow(/without a failing test/); + expect(() => evaluateCargoTestOutput(cargoOutput("FAILED", "ok"), 0, 2)).toThrow(/exited 0 with failing tests/); + }); + + it("excludes the harness, its controls, infrastructure, protocol vector suites and real servers from ordinary tests", () => { + for (const file of ["conformance.rs", "settlement_control.rs", "harness_infra.rs", "redis_integration.rs", "protocol_keys.rs", "protocol_frames.rs"]) expect(infrastructureTestFile.test(file), file).toBe(true); + for (const file of ["metrics_exporters.rs", "tokio_runtime.rs", "policy_helpers.rs"]) expect(infrastructureTestFile.test(file), file).toBe(false); + }); +}); + +function report(cases: [string, string, string?][], finish = true, extra: Record = {}): string { + const failed = cases.filter(([, status]) => status === "failed").length; + const lines = [ + JSON.stringify({ schemaVersion: 1, kind: "start", implementation: "rust", startedAt: 1 }), + ...cases.map(([id, status, message]) => JSON.stringify({ kind: "case", id, status, startedAt: 2, finishedAt: 3, ...(status === "failed" ? { message: message ?? "expected: 1\nactual: 2" } : {}) })), + ]; + if (finish) lines.push(JSON.stringify({ kind: "finish", status: failed ? "failed" : "passed", finishedAt: 4, cases: cases.length, failed, ...extra })); + return lines.join("\n") + "\n"; +} + +describe("Rust harness report evaluation", () => { + it("survives a complete passing report and lists every executed case", () => { + const cohort = evaluateRustReport(report([["sampled/core/0", "passed"], ["scenario/read/first-fill", "passed"]]), 0); + expect(cohort).toMatchObject({ state: "survived", passed: 2, failed: 0, failingTests: [], executedTests: ["sampled/core/0", "scenario/read/first-fill"] }); + }); + + it("detects a failed case only when the harness also exited nonzero", () => { + const text = report([["sampled/core/0", "failed"], ["scenario/read/first-fill", "passed"]]); + expect(evaluateRustReport(text, 101)).toMatchObject({ state: "detected", passed: 1, failed: 1, failingTests: ["sampled/core/0"] }); + expect(() => evaluateRustReport(text, 0)).toThrow(/exited 0 with failed cases/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]]), 101)).toThrow(/without a failed case/); + }); + + it("retains the validated assertion kind and evidence", () => { + const id = "sampled/effects/0"; + const message = 'CAUSAL_PROPERTY_FAILURE rule=C26 event={"index":0,"atMs":0,"event":"writeDispatch","condition":"publication without accepted source success","authorized":false}'; + expect(evaluateRustReport(report([[id, "failed", message]]), 101)).toMatchObject({ + assertionKinds: { [id]: "causal-property" }, assertionEvidence: { [id]: message }, + }); + const vector = "protocol/frameVectors/example"; + const mismatch = "PROTOCOL_ASSERTION_FAILURE expected: 1\nactual: 2"; + expect(evaluateRustReport(report([[vector, "failed", mismatch]]), 101)).toMatchObject({ + assertionKinds: { [vector]: "protocol-assertion" }, assertionEvidence: { [vector]: mismatch }, + }); + }); + + it.each([ + "Malformed replay observation: behaviorObservation at observed.calls", + "malformed next replay command", + "no pending read 0", + "coordinator response is not UTF-8", + "effects contract event 0 fallbackCompletion: fallback completion has no source", + 'CAUSAL_PROPERTY_FAILURE rule=C26 event={"index":0,"atMs":0,"event":"writeDispatch","condition":"publication without accepted source success","authorized":true}', + ])("rejects infrastructure or invalid property evidence: %s", message => { + expect(() => evaluateRustReport(report([["sampled/effects/0", "failed", message]]), 101)).toThrow(/lacks observation or validated causal property evidence/); + }); + + it("rejects global infrastructure failures even when an assertion also failed", () => { + const text = report([["sampled/effects/0", "failed"]]); + expect(() => evaluateRustReport(text, 101, "conformance harness failed: coordinator exited unexpectedly")).toThrow(/infrastructure or coverage/); + expect(() => evaluateRustReport(text, 101, "coverage: effects corpus omits action begin")).toThrow(/infrastructure or coverage/); + }); + + it("rejects failed witness audits and malformed protocol fixtures", () => { + expect(() => evaluateRustReport(report([["witness/effects", "failed", "expected: hash-a actual: hash-b"]]), 101)).toThrow(/witness audit failure/); + expect(() => evaluateRustReport(report([["protocol/frameVectors/example", "failed", "frameHex is neither null nor a string"]]), 101)).toThrow(/protocol failure lacks assertion evidence/); + }); + + it("rejects crashed, empty, duplicated or inconsistent reports instead of crediting them", () => { + expect(() => evaluateRustReport("", 0)).toThrow(/empty Rust harness report/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]], false), 0)).toThrow(/no finish record/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"], ["sampled/core/0", "passed"]]), 0)).toThrow(/duplicate case/); + expect(() => evaluateRustReport(report([], true), 0)).toThrow(/no cases/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]], true, { cases: 2 }), 0)).toThrow(/finish totals disagree/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]], true, { status: "failed" }), 0)).toThrow(/finish status disagrees/); + }); +}); + +describe("Rust fault catalog", () => { + const readRepo = (path: string) => readFileSync(new URL(`../${path}`, import.meta.url), "utf8"); + type Entry = { id: string; case: string; description: string; typescriptMutation: string; edits: { path: string; before: string; after: string }[]; requiredDetections: string[]; typescriptRequiredDetections: string[] }; + const catalog = JSON.parse(readRepo("formal/rust-mutations.json")) as { schemaVersion: number; mutations: Entry[] }; + const typescript = JSON.parse(readRepo("formal/mutations.json")) as { mutations: { id: string; case: string; description: string; typescript: { requiredDetections: string[] } }[] }; + + it("maps each scoped Rust fault once into the current shared catalog", () => { + expect(catalog.schemaVersion).toBe(1); + const scope = rustMutationScope(catalog, typescript.mutations); + expect(scope).toMatchObject({ catalog: "rust-native", modelBoundaryEvidence: false }); + expect(scope.mappedMutations).toEqual(catalog.mutations.map(m => m.typescriptMutation)); + expect([...scope.mappedMutations, ...scope.unmappedSharedMutations].sort()).toEqual(typescript.mutations.map(m => m.id).sort()); + expect(scope.unmappedSharedMutations).toContain("M14"); + expect(() => rustMutationScope({ ...catalog, mutations: [...catalog.mutations, catalog.mutations[0]] }, typescript.mutations)).toThrow(/duplicate/); + expect(() => rustMutationScope(catalog, typescript.mutations.slice(1))).toThrow(/counterpart/); + for (const entry of catalog.mutations) { + const counterpart = typescript.mutations.find(m => m.id === entry.typescriptMutation)!; + expect(counterpart, entry.id).toBeDefined(); + expect(entry.case, entry.id).toBe(counterpart.case); + expect(entry.description, entry.id).toBe(counterpart.description); + expect(entry.typescriptRequiredDetections, entry.id).toEqual(counterpart.typescript.requiredDetections); + expect(entry.requiredDetections, entry.id).toEqual(expect.arrayContaining(["generated", "portable"])); + expect(entry.requiredDetections.every(cohort => ["generated", "fixed", "portable"].includes(cohort)), entry.id).toBe(true); + } + }); + + it("anchors every edit exactly once in the crate's production sources", () => { + for (const entry of catalog.mutations) { + expect(entry.edits.length, entry.id).toBeGreaterThan(0); + for (const edit of entry.edits) { + expect(edit.path, entry.id).toMatch(/^rust\/src\/[\w/-]+\.rs$/); + expect(edit.before, entry.id).not.toBe(edit.after); + expect(readRepo(edit.path).split(edit.before).length, `${entry.id} ${edit.path}`).toBe(2); + } + } + }); +}); + + +describe("Rust mutation build isolation", () => { + it("gives concurrent shards and partial runs separate Cargo target directories", () => { + const first = rustTargetDirectory("/repo", { shard: { index: 1, count: 6 } }); + const second = rustTargetDirectory("/repo", { shard: { index: 2, count: 6 } }); + const full = rustTargetDirectory("/repo", { shard: { index: 1, count: 1 } }); + const partial = rustTargetDirectory("/repo", { shard: { index: 1, count: 1 }, only: ["M01"] }); + expect(new Set([first, second, full, partial]).size).toBe(4); + expect(first).toBe("/repo/rust/target/semantic/shards/1-of-6"); + }); +}); diff --git a/test/formal-validation.test.ts b/test/formal-validation.test.ts index 6624e4da..01f8706f 100644 --- a/test/formal-validation.test.ts +++ b/test/formal-validation.test.ts @@ -9,6 +9,7 @@ type Step = { label: string; command?: string; args?: string[]; + cwd?: string; env?: NodeJS.ProcessEnv; stdoutFile?: string; requireEmptyStdout?: boolean; @@ -45,7 +46,7 @@ describe("shared validation runner", () => { beforeEach(() => { directory = mkdtempSync(join(tmpdir(), "dialcache-validation-")); - for (const path of ["bin", "formal", "dist", "node_modules/typescript"]) mkdirSync(join(directory, path), { recursive: true }); + for (const path of ["bin", "formal", "dist", "node_modules/typescript", "rust"]) mkdirSync(join(directory, path), { recursive: true }); child = join(directory, "child.mjs"); put("child.mjs", `import { appendFileSync } from 'node:fs'; appendFileSync(process.env.RUNNER_EVENTS, JSON.stringify({ label: process.argv[2], cwd: process.cwd(), @@ -62,6 +63,7 @@ process.exit(Number(process.argv[3] ?? 0));\n`); put("formal/generated-fixtures.lock.json", '{"quintVersion":"0.32.0"}'); fakeTool("corepack", 'console.log("10.33.0")'); fakeTool("go", 'console.log("go version go1.27.1 test/test")'); + fakeTool("cargo", 'console.log("cargo 1.98.1 (test 2026-08-05)")'); fakeTool("quint", 'console.log("0.32.0")'); fakeTool("java", 'console.log("openjdk 21.0.11")'); fakeTool("tar", 'console.log("bsdtar 3.5.3")'); @@ -86,6 +88,16 @@ process.exit(Number(process.argv[3] ?? 0));\n`); }); }); + it("runs a step inside its declared directory and every other step at the checkout root", async () => { + await run([ + { label: "root", command: process.execPath, args: [child, "root"] }, + { label: "crate", command: process.execPath, args: [child, "crate"], cwd: "rust" }, + ]); + const [root, crate] = events(); + expect(root!.cwd).toBe(realpathSync(directory)); + expect(crate!.cwd).toBe(realpathSync(join(directory, "rust"))); + }); + it("excludes only opt-in Go workers from complete replay and exploration", async () => { const { loadGoReplayInventory } = await import(new URL("../formal/check-go-replay.mjs", import.meta.url).href) as { loadGoReplayInventory(): { required: Array<{ name: string }> }; @@ -130,11 +142,14 @@ process.exit(Number(process.argv[3] ?? 0));\n`); .rejects.toThrow(/files requiring formatting:\ngo\/cache.go/); }); - it("orders generation and shared witness evaluation before both port replays without duplicate wire generation", () => { + it("orders generation and shared witness evaluation before every port replay without duplicate wire generation", () => { const plan = validationPlan("formal", { directory }); const position = (script: string, argument: string) => plan.findIndex(step => step.args?.[0] === script && step.args.includes(argument)); const tsPrepare = position("formal/conformance.mjs", "typescript"); const goPrepare = position("formal/conformance.mjs", "go"); + const rustPrepare = position("formal/conformance.mjs", "rust"); + const goCompletion = position("formal/conformance.mjs", ".formal-traces/go-completion.json"); + const rustCompletion = position("formal/conformance.mjs", ".formal-traces/rust-completion.json"); const tsCompletion = position("formal/conformance.mjs", ".formal-traces/ts-completion.json"); const witnesses = position("formal/witnesses.mjs", "evaluate"); expect(position("formal/run-models.mjs", "check")).toBeLessThan(position("formal/run-models.mjs", "generate")); @@ -142,14 +157,17 @@ process.exit(Number(process.argv[3] ?? 0));\n`); expect(witnesses).toBeLessThan(tsPrepare); expect(tsPrepare).toBeLessThan(tsCompletion); expect(tsCompletion).toBeLessThan(goPrepare); + expect(goPrepare).toBeLessThan(goCompletion); + expect(goCompletion).toBeLessThan(rustPrepare); + expect(rustPrepare).toBeLessThan(rustCompletion); expect(plan.filter(step => step.args?.[0] === "formal/run-models.mjs" && step.args[1] === "generate")).toHaveLength(1); expect(plan.filter(step => step.args?.[0] === "formal/witnesses.mjs")).toHaveLength(1); expect(plan.some(step => step.args?.[0] === "formal/generate-artifacts.mjs")).toBe(false); expect(plan[0]!.args).toEqual(["formal/run-models.mjs", "check"]); - expect(plan.find(step => step.remove)!.remove).toEqual([".formal-traces/ts-completion.json", ".formal-traces/go-completion.json"]); + expect(plan.find(step => step.remove)!.remove).toEqual([".formal-traces/ts-completion.json", ".formal-traces/go-completion.json", ".formal-traces/rust-completion.json"]); // The aggregate is exactly these lanes in order, so a CI job running // one lane executes the same steps as the local sequential run. - expect(plan).toEqual(["formal-check", "formal-generate", "formal-ts", "formal-go"].flatMap(target => validationPlan(target, { directory }))); + expect(plan).toEqual(["formal-check", "formal-generate", "formal-ts", "formal-go", "formal-rust"].flatMap(target => validationPlan(target, { directory }))); }); it("keeps the model check as its own lane that produces nothing the port lanes consume", () => { @@ -164,7 +182,7 @@ process.exit(Number(process.argv[3] ?? 0));\n`); const generate = validationPlan("formal-generate", { directory }); expect(generate.some(step => step.args?.[0] === "formal/run-models.mjs" && step.args[1] === "check")).toBe(false); expect(generate.some(step => step.args?.[0] === "formal/check-model-properties.mjs")).toBe(false); - for (const target of ["formal-ts", "formal-go", "mutations"]) { + for (const target of ["formal-ts", "formal-go", "formal-rust", "mutations"]) { expect(validationPlan(target, { directory }).some(step => step.args?.[0] === "formal/run-models.mjs")).toBe(false); } // Every acceptance entry point keeps one complete campaign, after all @@ -215,18 +233,41 @@ process.exit(Number(process.argv[3] ?? 0));\n`); expect(ts.some(step => step.args?.[0] === "formal/witnesses.mjs" || step.args?.[0] === "formal/run-models.mjs")).toBe(false); }); - it("lets Go parity and both mutation measurements run off the generated corpus without completion checks", () => { + it("replays Rust from its crate with full corpus selectors and a dedicated report before the completion gate", () => { + const plan = validationPlan("formal-rust", { directory, environment }); + expect(plan[0]!.remove).toEqual([".formal-traces/rust-completion.json"]); + expect(plan[1]!.args).toEqual(["formal/conformance.mjs", "prepare", "rust", ".formal-traces/rust-context.json"]); + const replay = plan.find(step => step.command === "cargo")!; + expect(replay).toMatchObject({ cwd: "rust", args: ["test", "--release", "--all-features", "--test", "conformance"] }); + expect(replay.env).toEqual({ + DIALCACHE_MBT_TRACE_DIR: join(directory, ".formal-traces/conformance"), + DIALCACHE_EFFECTS_TRACE_DIR: join(directory, ".formal-traces/effects"), + DIALCACHE_FEATURE_TRACE_DIR: join(directory, ".formal-traces/features"), + DIALCACHE_WITNESS_EVIDENCE_DIR: join(directory, ".formal-traces/go-parity-witnesses"), + DIALCACHE_RUST_REPORT: join(directory, ".formal-traces/rust-replay.jsonl"), + }); + expect(plan.slice(plan.indexOf(replay) + 1).map(step => step.args)).toEqual([ + ["formal/check-rust-replay.mjs"], + ["formal/conformance-adapters.mjs", "rust", ".formal-traces/rust-replay.jsonl", ".formal-traces/rust-context.json"], + ["formal/conformance.mjs", "check", ".formal-traces/rust-completion.json", ".formal-traces/rust-context.json"], + ]); + const smoke = validationPlan("smoke", { directory }).find(step => step.command === "cargo")!; + expect(smoke.args).toEqual(["test", "--all-features", "--test", "conformance"]); + expect(smoke.env).toBeUndefined(); + }); + + it("lets Go parity and every mutation measurement run off the generated corpus without completion checks", () => { const isCompletionCheck = (step: Step) => step.args?.[0] === "formal/conformance.mjs" && step.args[1] === "check"; const go = validationPlan("formal-go", { directory }); expect(go[0]!.remove).toEqual([".formal-traces/go-completion.json"]); expect(go.some(step => step.args?.some(argument => argument.startsWith(".formal-traces/ts-")))).toBe(false); expect(go.filter(isCompletionCheck).map(step => step.args)).toEqual([["formal/conformance.mjs", "check", ".formal-traces/go-completion.json", ".formal-traces/go-context.json"]]); expect(go.some(step => step.args?.[0] === "formal/witnesses.mjs" || step.args?.[0] === "formal/run-models.mjs")).toBe(false); - for (const [target, script] of [["mutations-ts", "formal/measure-semantics.mjs"], ["mutations-go", "formal/measure-go-semantics.mjs"]] as const) { + for (const [target, script] of [["mutations-ts", "formal/measure-semantics.mjs"], ["mutations-go", "formal/measure-go-semantics.mjs"], ["mutations-rust", "formal/measure-rust-semantics.mjs"]] as const) { expect(validationPlan(target, { directory }).map(step => step.args)).toEqual([[script]]); } const all = validationPlan("mutations", { directory }); - expect(all.map(step => step.args?.[0])).toEqual(["formal/measure-semantics.mjs", "formal/measure-go-semantics.mjs"]); + expect(all.map(step => step.args?.[0])).toEqual(["formal/measure-semantics.mjs", "formal/measure-go-semantics.mjs", "formal/measure-rust-semantics.mjs"]); expect(all.some(isCompletionCheck)).toBe(false); expect(all.some(step => step.remove || step.args?.[0] === "formal/run-models.mjs")).toBe(false); }); @@ -237,6 +278,7 @@ process.exit(Number(process.argv[3] ?? 0));\n`); { label: "Measure TypeScript semantic mutations", command: process.execPath, args: ["formal/measure-semantics.mjs", "--shard=2/3"] }, ]); expect(validationPlan("mutations-go", { directory, environment: sharded }).map(step => step.args)).toEqual([["formal/measure-go-semantics.mjs", "--shard=2/3"]]); + expect(validationPlan("mutations-rust", { directory, environment: sharded }).map(step => step.args)).toEqual([["formal/measure-rust-semantics.mjs", "--shard=2/3"]]); // Unset, the plan is exactly today's complete measurement. expect(validationPlan("mutations-ts", { directory, environment }).map(step => step.args)).toEqual([["formal/measure-semantics.mjs"]]); for (const value of ["0/3", "4/3", "1/0", "a/b", "1", "01/3", "1/3/", " 1/3", ""]) { @@ -244,9 +286,9 @@ process.exit(Number(process.argv[3] ?? 0));\n`); } // The aggregates stay unsharded and refuse to ignore the variable silently; unrelated targets ignore it. for (const target of ["mutations", "ci"]) { - expect(() => validationPlan(target, { directory, environment: sharded }), target).toThrow(/MUTATION_SHARD=2\/3 applies only to make mutations-ts and make mutations-go/); + expect(() => validationPlan(target, { directory, environment: sharded }), target).toThrow(/MUTATION_SHARD=2\/3 applies only to make mutations-ts, make mutations-go and make mutations-rust/); } - for (const target of ["check", "formal", "formal-ts", "mutations-merge-ts", "mutations-merge-go"]) { + for (const target of ["check", "formal", "formal-ts", "mutations-merge-ts", "mutations-merge-go", "mutations-merge-rust"]) { expect(validationPlan(target, { directory, environment: sharded }), target).toEqual(validationPlan(target, { directory, environment })); } // The runner's environment cleaning removes replay selectors, not the shard. @@ -259,6 +301,7 @@ process.exit(Number(process.argv[3] ?? 0));\n`); { label: "Measure TypeScript semantic mutations", command: process.execPath, args: ["formal/measure-semantics.mjs", "--only=M14,M15"] }, ]); expect(validationPlan("mutations-go", { directory, environment: partial }).map(step => step.args)).toEqual([["formal/measure-go-semantics.mjs", "--only=M14,M15"]]); + expect(validationPlan("mutations-rust", { directory, environment: { ...environment, MUTATION_ONLY: "M01,M02" } }).map(step => step.args)).toEqual([["formal/measure-rust-semantics.mjs", "--only=M01,M02"]]); for (const value of ["", "M14,", "M14,M14", "m14", "14", "M14 M15"]) { expect(() => validationPlan("mutations-ts", { directory, environment: { ...environment, MUTATION_ONLY: value } }), value).toThrow(/MUTATION_ONLY must be , naming distinct mutant ids/); } @@ -266,23 +309,25 @@ process.exit(Number(process.argv[3] ?? 0));\n`); expect(() => validationPlan("mutations-ts", { directory, environment: { ...partial, MUTATION_SHARD: "2/3" } })).toThrow(/MUTATION_SHARD and MUTATION_ONLY exclude each other/); // The aggregates refuse to ignore it silently; unrelated targets ignore it. for (const target of ["mutations", "ci"]) { - expect(() => validationPlan(target, { directory, environment: partial }), target).toThrow(/MUTATION_ONLY=M14,M15 applies only to make mutations-ts and make mutations-go/); + expect(() => validationPlan(target, { directory, environment: partial }), target).toThrow(/MUTATION_ONLY=M14,M15 applies only to make mutations-ts, make mutations-go and make mutations-rust/); } - for (const target of ["check", "formal", "formal-ts", "mutations-merge-ts", "mutations-merge-go"]) { + for (const target of ["check", "formal", "formal-ts", "mutations-merge-ts", "mutations-merge-go", "mutations-merge-rust"]) { expect(validationPlan(target, { directory, environment: partial }), target).toEqual(validationPlan(target, { directory, environment })); } expect(cleanEnvironment({ MUTATION_ONLY: "M14", DIALCACHE_PROTOCOL_CORPUS: "fixed" })).toEqual({ MUTATION_ONLY: "M14" }); - for (const target of ["mutations-ts", "mutations-go"]) expect(targetDescriptions[target]).toMatch(/MUTATION_ONLY=,/); + for (const target of ["mutations-ts", "mutations-go", "mutations-rust"]) expect(targetDescriptions[target]).toMatch(/MUTATION_ONLY=,/); }); - it("merges each language's shards with a plain Node step that needs neither Quint nor Go", () => { + it("merges each language's shards with a plain Node step that needs no native toolchains or Quint", () => { expect(validationPlan("mutations-merge-ts", { directory })).toEqual([ { label: "Merge TypeScript mutation shards", command: process.execPath, args: ["formal/merge-mutation-reports.mjs", "ts"] }, ]); expect(validationPlan("mutations-merge-go", { directory }).map(step => step.args)).toEqual([["formal/merge-mutation-reports.mjs", "go"]]); + expect(validationPlan("mutations-merge-rust", { directory }).map(step => step.args)).toEqual([["formal/merge-mutation-reports.mjs", "rust"]]); fakeTool("quint", 'console.error("quint: not installed"); process.exit(1)'); fakeTool("go", 'console.error("go: not installed"); process.exit(1)'); - for (const target of ["mutations-merge-ts", "mutations-merge-go"]) { + fakeTool("cargo", 'console.error("cargo: not installed"); process.exit(1)'); + for (const target of ["mutations-merge-ts", "mutations-merge-go", "mutations-merge-rust"]) { expect(() => checkPrerequisites(target, { directory, environment, nodeVersion: "v24.20.0" })).not.toThrow(); expect(targetDescriptions[target]).toMatch(/Merge .* mutation shards/); } @@ -344,7 +389,7 @@ process.exit(Number(process.argv[3] ?? 0));\n`); for (const target of ["mutations-ts", "mutations-go"]) { expect(() => checkPrerequisites(target, { directory, environment, nodeVersion: "v24.20.0" })).toThrow(/docker/); } - for (const target of ["mutations-merge-ts", "mutations-merge-go"]) { + for (const target of ["mutations-merge-ts", "mutations-merge-go", "mutations-merge-rust"]) { expect(() => checkPrerequisites(target, { directory, environment, nodeVersion: "v24.20.0" })).not.toThrow(); } }); @@ -379,7 +424,7 @@ else { describe("full formal workflow shape", () => { type Step = { name?: string; run?: string; uses?: string; if?: string; env?: Record; with?: Record }; type Job = { needs?: string | string[]; if?: string; env?: Record; strategy?: { "fail-fast"?: boolean; matrix?: Record }; "timeout-minutes"?: number; steps: Step[] }; - const lanes = ["typescript-parity", "go-parity", "typescript-mutations", "go-mutations"]; + const lanes = ["typescript-parity", "go-parity", "rust-parity", "typescript-mutations", "go-mutations", "rust-mutations"]; const needsOf = (job: Job) => (job.needs === undefined ? [] : [job.needs].flat()); let jobs: Record; @@ -465,6 +510,32 @@ describe("full formal workflow shape", () => { expect(gate.run).toMatch(/test "\$GO_MUTATIONS_MERGE_RESULT" = success/); }); + it("requires Rust replay and merged mutations and retains Rust completion in the summary", () => { + const parity = jobs["rust-parity"]!; + expect(parity.steps.find(step => step.uses === "./.github/actions/setup-validation")!.with).toEqual({ rust: "true" }); + expect(parity.steps.map(step => step.run).filter(Boolean)).toEqual(["make formal-rust"]); + const mutations = jobs["rust-mutations"]!; + const shards = mutations.strategy!.matrix!.shard as number[]; + expect(mutations.env).toEqual({ MUTATION_SHARD: "$" + "{{ matrix.shard }}/" + shards.length }); + expect(mutations.steps.map(step => step.run).filter(Boolean)).toEqual(["make mutations-rust"]); + const merge = jobs["rust-mutations-merge"]!; + expect(needsOf(merge)).toEqual(["rust-mutations"]); + expect(merge.if).toBe("always() && needs.rust-mutations.result != 'skipped'"); + expect(merge.steps.map(step => step.run).filter(Boolean)).toEqual(["make mutations-merge-rust"]); + const aggregate = jobs["formal-full"]!; + expect(needsOf(aggregate)).toEqual(expect.arrayContaining(["rust-parity", "rust-mutations-merge"])); + expect(needsOf(aggregate)).not.toContain("rust-mutations"); + const gate = aggregate.steps.find(step => step.run?.includes("_RESULT"))!; + expect(gate.env).toMatchObject({ RUST_RESULT: "$" + "{{ needs.rust-parity.result }}", + RUST_MUTATIONS_MERGE_RESULT: "$" + "{{ needs.rust-mutations-merge.result }}" }); + expect(gate.run).toMatch(/test "\$RUST_RESULT" = success/); + expect(gate.run).toMatch(/test "\$RUST_MUTATIONS_MERGE_RESULT" = success/); + const summary = aggregate.steps.find(step => step.uses?.startsWith("actions/upload-artifact"))!.with!; + expect(summary.path).toContain("formal-summary/rust/rust-completion.json"); + expect(summary.path).toContain("formal-summary/rust/rust-context.json"); + expect(summary.path).toContain("formal-summary/rust/rust-replay-summary.json"); + }); + it("requires the model check in the aggregate and retains its report in the long-lived summary", () => { const aggregate = jobs["formal-full"]!; expect(needsOf(aggregate)).toEqual(expect.arrayContaining(["check-models", "generate", "typescript-parity", "go-parity"]));