test: exercise engine snapshot updates under real parallelism

.github/workflows/nightly-tsan.yml

@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) 2026 Praxis Contributors
+
+name: Nightly ThreadSanitizer
+
+# The engine is shared across threads behind `Arc` and mutated while
+# requests are in flight. PR CI runs the seeded stress test on a
+# multi-threaded runtime; this job rebuilds it under ThreadSanitizer so
+# a data race is a red build rather than a wrong plugin count.
+#
+# Nightly only: TSan needs a nightly compiler, and the rebuild is too
+# slow for every pull request. `workflow_dispatch` is here so a race
+# report can be reproduced without waiting for the cron.
+
+on:
+  schedule:
+    - cron: "0 6 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions: {}
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  CARGO_INCREMENTAL: 0
+  RUSTFLAGS: -Zsanitizer=thread
+  TSAN_OPTIONS: halt_on_error=1
+
+jobs:
+  tsan-engine-stress:
+    name: engine concurrency under TSan
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+        with:
+          persist-credentials: false
+      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cargo/bin
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git/db
+            target
+          key: ${{ runner.os }}-cargo-tsan-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}
+          restore-keys: ${{ runner.os }}-cargo-tsan-
+      - run: rustup toolchain install nightly --component rust-src --profile minimal
+      - run: rustup target add x86_64-unknown-linux-gnu --toolchain nightly
+      - run: make test-tsan

CONTRIBUTING.md

@@ -134,3 +134,27 @@ Enforcing one of the allowed groups is welcome as a focused change, one lint at
 time, separate from feature work. `docs/lints.md` is worth reading first: it
 records which lints clippy reports as machine-fixable but cannot actually fix, and
 where a lint's suggested rewrite is worse than the code it replaces.
+
+## Multi-threaded Tokio tests
+
+`#[tokio::test]` defaults to `current_thread`. Tasks yield at `.await` but never
+run on two OS threads at the same time, so a load and a store that have no await
+between them cannot overlap.
+
+Use a multi-threaded runtime when the test shares a `PolicyEngine` (or any other
+`Arc` handle) across tasks **and** mutates it while invokes are in flight:
+
+```rust
+#[tokio::test(flavor = "multi_thread")]
+async fn register_while_other_tasks_invoke() { /* ... */ }
+```
+
+That is the right flavor for registration, unregister, hot reload, and
+route-cache fill or invalidation. Sequential logic — one engine, one task, no
+shared mutation — stays on `current_thread`.
+
+A seeded stress test lives in `crates/ppe-core/tests/engine_concurrency.rs`.
+Replay a failure with `PPE_STRESS_SEED`. Nightly CI runs that test under
+ThreadSanitizer (`make test-tsan`). The `Release` / `Acquire` pairing between
+the snapshot and `config_generation` is checked by the loom model in
+`crates/ppe-core/tests/loom_generation_snapshot.rs`.

Cargo.toml

@@ -108,6 +108,10 @@ paste = "1"
 futures = "0.3"
 hashbrown = "0.17"
 arc-swap = "1.9"
+# Exhaustive scheduler for a tiny model of the generation / snapshot
+# pairing. Dev-only: the model lives in ppe-core's loom test, not in
+# production code.
+loom = "0.7"
 wildmatch = "2"
 rmp-serde = "1"
 serde_bytes = "0.11"

Makefile

@@ -40,6 +40,7 @@ help:
 	@echo ""
 	@echo "Test:"
 	@echo "  test              Run all workspace tests"
+	@echo "  test-tsan         Engine concurrency stress under ThreadSanitizer (nightly)"
 	@echo ""
 	@echo "Supply chain & coverage:"
 	@echo "  audit             cargo deny check (advisories, licenses, bans, sources)"
@@ -157,6 +158,17 @@ test:
 	@$(CARGO) test --workspace
 	@$(CARGO) test --workspace --all-features
 
+# ThreadSanitizer on the engine concurrency stress test. Needs nightly and a
+# Linux target; the sanitizer does not run on the pinned stable toolchain.
+# `--test-threads=1` keeps TSan's own reports from overlapping.
+.PHONY: test-tsan
+test-tsan:
+	@echo "ThreadSanitizer: praxis-policy-core engine concurrency ..."
+	@RUSTFLAGS="-Zsanitizer=thread" CARGO_INCREMENTAL=0 \
+		$(CARGO) +$(NIGHTLY) test -p praxis-policy-core --test engine_concurrency \
+		--target x86_64-unknown-linux-gnu -- --test-threads=1
+	@echo "test-tsan passed"
+
 # =============================================================================
 # Supply chain & coverage
 # =============================================================================

crates/ppe-core/Cargo.toml

@@ -63,5 +63,8 @@ zeroize = { version = "1.9", features = ["zeroize_derive"] }
 # (and praxis-policy-apl-core's `Effect::Parallel`). Leaf crate, no cycles back here.
 praxis-policy-orchestration = { workspace = true }
 
+[dev-dependencies]
+loom = { workspace = true }
+
 [lints]
 workspace = true